coverage — ~bigbes/sr-ht-spec — 64cae3af — search/mapping.go

Coverage
92.4% 61/66 statements
Δ
+0.0
Blob
ee85c62
Uncovered L115-L116L154-L155L157-L158L160-L161L163-L164
1 package search
2
3 import (
4 "fmt"
5
6 "github.com/blevesearch/bleve/v2"
7 "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
8 "github.com/blevesearch/bleve/v2/analysis/lang/en"
9 "github.com/blevesearch/bleve/v2/analysis/lang/ru"
10 "github.com/blevesearch/bleve/v2/mapping"
11
12 "sourcecraft.dev/bigbes/sr-ht-spec/core"
13 )
14
15 // Field names in the bleve index. Three groups, and the difference between them
16 // is what the whole mapping is:
17 //
18 // - filters (space, section, lang) are keyword-analyzed, so they are matched
19 // as whole strings by a term query and never tokenized. Filtering by a
20 // space through an analyzed field — which is what warren did for sections —
21 // means "~bigbes/b-tree" matches the space "~someone/tree", and a project
22 // is precisely a filter over spaces, so it has to be exact.
23 // - analyzed text (title_en/title_ru, body_en/body_ru) carries the
24 // searchable words, one field pair per language. See lang.go.
25 // - metadata (rev, path, anchor, title) is stored and not indexed: it is what
26 // turns a hit into a URL, not something to search on.
27 const (
28 fieldSpace = "space"
29 fieldSection = "section"
30 fieldLang = "lang"
31 fieldRev = "rev"
32 fieldPath = "path"
33 fieldAnchor = "anchor"
34 fieldTitle = "title"
35 fieldTitleEN = "title_en"
36 fieldTitleRU = "title_ru"
37 fieldBodyEN = "body_en"
38 fieldBodyRU = "body_ru"
39 )
40
41 // titleBoost is how much more a title match is worth than a body match. Carried
42 // over from warren unchanged: a query that names a document should return that
43 // document, not the twenty documents that mention it.
44 const titleBoost = 3.0
45
46 // buildMapping is the index mapping of the one global index. There is exactly
47 // one, and it is not parameterized by language: the design's earlier
48 // per-index-language choice is what this replaces.
49 34 func buildMapping() mapping.IndexMapping {
50 136 text := func(analyzer string, termVectors bool) *mapping.FieldMapping {
51 136 f := bleve.NewTextFieldMapping()
52 136 f.Analyzer = analyzer
53 136 f.Store = true
54 136 f.IncludeTermVectors = termVectors
55 136 f.IncludeInAll = false
56 136 return f
57 136 }
58 102 exact := func() *mapping.FieldMapping {
59 102 f := bleve.NewTextFieldMapping()
60 102 f.Analyzer = keyword.Name
61 102 f.Store = true
62 102 f.IncludeInAll = false
63 102 return f
64 102 }
65 136 meta := func() *mapping.FieldMapping {
66 136 f := bleve.NewTextFieldMapping()
67 136 f.Index = false
68 136 f.Store = true
69 136 f.IncludeInAll = false
70 136 return f
71 136 }
72
73 34 d := bleve.NewDocumentMapping()
74 34 // Nothing is indexed that this file does not name. A dynamic mapping would
75 34 // silently index whatever a future field happens to be called, with the
76 34 // default analyzer, which is how an index acquires fields nobody meant.
77 34 d.Dynamic = false
78 34 d.AddFieldMappingsAt(fieldSpace, exact())
79 34 d.AddFieldMappingsAt(fieldSection, exact())
80 34 d.AddFieldMappingsAt(fieldLang, exact())
81 34 d.AddFieldMappingsAt(fieldRev, meta())
82 34 d.AddFieldMappingsAt(fieldPath, meta())
83 34 d.AddFieldMappingsAt(fieldAnchor, meta())
84 34 d.AddFieldMappingsAt(fieldTitle, meta())
85 34 d.AddFieldMappingsAt(fieldTitleEN, text(en.AnalyzerName, false))
86 34 d.AddFieldMappingsAt(fieldTitleRU, text(ru.AnalyzerName, false))
87 34 // Body fields carry term vectors because they are the highlighted ones: a
88 34 // snippet is reconstructed from stored text plus term locations.
89 34 d.AddFieldMappingsAt(fieldBodyEN, text(en.AnalyzerName, true))
90 34 d.AddFieldMappingsAt(fieldBodyRU, text(ru.AnalyzerName, true))
91 34
92 34 m := bleve.NewIndexMapping()
93 34 m.DefaultAnalyzer = en.AnalyzerName
94 34 m.DefaultMapping = d
95 34 return m
96 }
97
98 // Key is the id a document is stored under in the global index. The space is
99 // part of it because the index is global: two spaces may each hold a document
100 // whose id fell back to the path "specs/storage", and in a single index those
101 // are two documents, not one overwriting the other.
102 //
103 // It is deliberately not parsed back. Space, path and the rest are stored
104 // fields; the key is an opaque identity.
105 1451 func Key(sp core.SpaceRef, id string) string {
106 1451 return sp.String() + ":" + id
107 1451 }
108
109 // bleveDoc projects a Document into the field map bleve indexes, doing the
110 // language detection and routing on the way. A map rather than a struct so
111 // that a field a document has nothing for is absent from the index instead of
112 // present and empty.
113 1451 func bleveDoc(d Document) (map[string]any, error) {
114 1451 if err := d.validate(); err != nil {
115 0 return nil, err
116 0 }
117 // Dominant language of the document as a whole, used to label it and as the
118 // fallback for lines too short to classify on their own. Title first: it is
119 // the most reliably prose-like text a document has.
120 1451 lang := DetectIn(d.Title+"\n"+d.Text, DefaultLang)
121 1451
122 1451 m := map[string]any{
123 1451 fieldSpace: d.Space.String(),
124 1451 fieldLang: string(lang),
125 1451 }
126 11608 put := func(field, value string) {
127 11608 if value != "" {
128 9904 m[field] = value
129 9904 }
130 }
131 1451 put(fieldSection, d.Section)
132 1451 put(fieldRev, d.Rev)
133 1451 put(fieldPath, d.Path)
134 1451 put(fieldAnchor, d.Anchor)
135 1451 put(fieldTitle, d.Title)
136 1451
137 1451 // A title is one line and takes the document's language; splitting it would
138 1451 // only ever misfile the shorter half of a name.
139 1451 if d.Title != "" {
140 1451 if lang == LangRU {
141 1205 put(fieldTitleRU, d.Title)
142 1205 } else {
143 246 put(fieldTitleEN, d.Title)
144 246 }
145 }
146 1451 bodyRU, bodyEN := route(d.Text, lang)
147 1451 put(fieldBodyRU, bodyRU)
148 1451 put(fieldBodyEN, bodyEN)
149 1451 return m, nil
150 }
151
152 1451 func (d Document) validate() error {
153 1451 if d.Space.Owner == "" || d.Space.Name == "" {
154 0 return fmt.Errorf("search: document %q has no space", d.ID)
155 0 }
156 1451 if err := core.ValidateOwner(d.Space.Owner); err != nil {
157 0 return fmt.Errorf("search: document %q space owner: %w", d.ID, err)
158 0 }
159 1451 if err := core.ValidateSpaceName(d.Space.Name); err != nil {
160 0 return fmt.Errorf("search: document %q space name: %w", d.ID, err)
161 0 }
162 1451 if d.ID == "" {
163 0 return fmt.Errorf("search: document in space %s has no id", d.Space)
164 0 }
165 1451 return nil
166 }