coverage — ~bigbes/sr-ht-spec — 64cae3af — search/lang.go

Coverage
100.0% 26/26 statements
Δ
+0.0
Blob
0f9d040
Uncovered nothing — every instrumented line ran
1 package search
2
3 import (
4 "strings"
5 "unicode"
6 )
7
8 // Lang is a language the index has a stemming analyzer for. Every indexed
9 // document is labelled with exactly one — its dominant language — and its text
10 // is routed block by block into the matching analyzed field.
11 type Lang string
12
13 const (
14 LangEN Lang = "en"
15 LangRU Lang = "ru"
16 )
17
18 // DefaultLang is the label a document with too little text to classify gets,
19 // and the language a block falls back to when it is too short to classify on
20 // its own. English rather than Russian because the machine-generated half of
21 // this corpus — frontmatter keys, paths, identifiers, fenced code — is English
22 // whatever language the prose around it is written in.
23 const DefaultLang = LangEN
24
25 // ruLetterRatio is the share of a text's letters that must be Cyrillic for it
26 // to count as Russian.
27 //
28 // The threshold is deliberately far below one half. The two error directions
29 // are not symmetric in likelihood: an English block essentially never contains
30 // Cyrillic at all, while a Russian block in this corpus routinely carries a
31 // third or more Latin letters — identifiers, product names, and untranslated
32 // technical terms are written in Latin inside Russian prose. A 0.5 threshold
33 // would therefore misfile real Russian paragraphs as English, and misfile
34 // almost no English ones as Russian.
35 const ruLetterRatio = 0.35
36
37 // minDetectLetters is the least number of letters a text needs before its
38 // script mix is treated as evidence. Below it, a single Latin acronym in a
39 // Russian heading (or one Russian word in an English one) would decide the
40 // whole block, so the caller's fallback is used instead.
41 const minDetectLetters = 12
42
43 // Detect classifies a text by script. ok is false when the text carries too
44 // few letters to classify, in which case the caller supplies the fallback —
45 // Detect never guesses.
46 67028 func Detect(text string) (lang Lang, ok bool) {
47 67028 var cyrillic, letters int
48 6829191 for _, r := range text {
49 6829191 if !unicode.IsLetter(r) {
50 1342646 continue
51 }
52 5486545 letters++
53 5486545 if unicode.Is(unicode.Cyrillic, r) {
54 2334656 cyrillic++
55 2334656 }
56 }
57 67028 if letters < minDetectLetters {
58 9879 return "", false
59 9879 }
60 57149 if float64(cyrillic)/float64(letters) >= ruLetterRatio {
61 22826 return LangRU, true
62 22826 }
63 34323 return LangEN, true
64 }
65
66 // DetectIn classifies a text, falling back to a language when it is too short
67 // to classify on its own.
68 67021 func DetectIn(text string, fallback Lang) Lang {
69 67021 if l, ok := Detect(text); ok {
70 57145 return l
71 57145 }
72 9876 return fallback
73 }
74
75 // route splits a text into its Russian and its English part, block by block,
76 // so that each half is stemmed by the analyzer that understands it.
77 //
78 // This is the part the design left open, and per-*document* routing — the
79 // obvious reading of "detect the language and write the matching field" — is
80 // not sufficient. The two analyzers pass each other's script through
81 // untouched: bleve's `ru` analyzer leaves "indexes" as "indexes" and its `en`
82 // analyzer leaves "индексы" as "индексы". Foreign-script terms therefore still
83 // match literally (which is why a single-analyzer index is not catastrophic),
84 // but they match *unstemmed*, so "index" does not find "indexes" and
85 // "документы" does not find "документ". A specification whose prose is Russian
86 // and whose examples, headings and quoted requirements are English is one
87 // document, and one label for it necessarily mangles one of its two halves.
88 //
89 // Routing per block costs nothing extra — the document is being walked anyway —
90 // and removes the failure entirely: each block lands in the field whose
91 // analyzer stems it, and a query is run against both fields. Duplicating the
92 // whole text into both fields would also fix the stemming, but it doubles the
93 // index and double-counts every document that matches in both fields, which
94 // biases ranking toward mixed documents for no reason related to relevance.
95 //
96 // The unit is a line, because that is the unit the text arrives in: doc's
97 // plain-text projection emits a newline after every block-level node and at
98 // every soft line break, and frontmatter search text is one "key: value" line
99 // per key. So a line here is a paragraph, a wrapped fragment of one, a heading,
100 // a table row, a list item or a line of code. A line too short to classify
101 // takes fallback, which is the document's dominant language — the language of
102 // the prose it sits inside.
103 1453 func route(text string, fallback Lang) (ru, en string) {
104 1453 var ruB, enB strings.Builder
105 68465 for _, line := range strings.Split(text, "\n") {
106 68465 if strings.TrimSpace(line) == "" {
107 2899 continue
108 }
109 65566 b := &enB
110 65566 if DetectIn(line, fallback) == LangRU {
111 31226 b = &ruB
112 31226 }
113 65566 if b.Len() > 0 {
114 62911 b.WriteByte('\n')
115 62911 }
116 65566 b.WriteString(line)
117 }
118 1453 return ruB.String(), enB.String()
119 }