coverage — ~bigbes/sr-ht-spec — 3cb1c03d — search/search.go

Coverage
94.2% 81/86 statements
Δ
—
Blob
1f20095
Uncovered L161-L162L168-L169L237-L238L241-L242L282-L283
1 // Package search is spec.sr.ht's keyword search: one global bleve index over
2 // every document of every space, queried through a filter.
3 //
4 // It is warren's index/ + search/ packages absorbed, with three structural
5 // changes the design calls for.
6 //
7 // # One index, filtered at query time
8 //
9 // warren indexed one vault, so the question never came up. Here it is the
10 // central decision: there is exactly one bleve index, every document in it
11 // carries its space, and a project — a named set of spaces — is a term filter
12 // over that field, not an index of its own. Per-project indexes were specified
13 // in an earlier draft and retracted: with them, every merge fans out to N
14 // rebuilds, adding a space to a project forces one, and the "everything"
15 // project is a second full copy of the corpus. As a filter, the meta-project is
16 // genuinely degenerate — a filter that excludes nothing — and a merge touches
17 // one index.
18 //
19 // # Rebuilds, not incremental updates
20 //
21 // At tens of documents a day, a batch rebuild is cheap and a per-document
22 // upsert/delete path is machinery bought against a cost nobody has measured.
23 // The unit of a rebuild is therefore a space at a revision (RebuildSpace) or
24 // the whole corpus (RebuildAll), never a document. Both report their duration
25 // in Stats so the decision to revisit is made against a measurement.
26 //
27 // # Keyword only
28 //
29 // warren also had a sqlite-vec semantic index and fused the two rankings with
30 // reciprocal rank fusion. Vector search is Phase 5 here, so none of it is
31 // ported — not even as unreachable code. What is left in its place is a seam,
32 // not a stub: Search returns ranked Hits, and a later hybrid ranker fuses two
33 // such lists. Nothing in this package assumes it is the only ranker.
34 //
35 // The package depends on core/, doc/ and gitx/ document types and on nothing
36 // else of this service. In particular it does not touch db/: index staleness
37 // stamps live in Postgres and service/ writes them, while this package is
38 // handed a revision's worth of documents and returns hits.
39 package search
40
41 import (
42 "context"
43 "errors"
44 "fmt"
45 "strings"
46 "time"
47
48 "github.com/blevesearch/bleve/v2"
49 bsearch "github.com/blevesearch/bleve/v2/search"
50 "github.com/blevesearch/bleve/v2/search/query"
51
52 "sourcecraft.dev/bigbes/sr-ht-spec/core"
53 "sourcecraft.dev/bigbes/sr-ht-spec/doc"
54 )
55
56 // DefaultLimit is how many hits a Query with no Limit returns.
57 const DefaultLimit = 20
58
59 // Query is one search over the global index.
60 //
61 // The zero value searches nothing: an empty Text returns no hits rather than
62 // every document, because "search for nothing" is a caller that has not
63 // collected its input yet, not a request to list the corpus.
64 type Query struct {
65 // Text is the user's query, analyzed with the same analyzers the documents
66 // were indexed with, per field.
67 Text string
68 // Spaces restricts results to a set of spaces. This is what a project is:
69 // the design's "a project is a saved filter over one global index, not a
70 // container" is this field and nothing else — a project resolves to a
71 // core.SpaceFilter and it is handed over whole.
72 //
73 // It is a filter rather than a []core.SpaceRef because the empty slice had
74 // two defensible meanings and the two are opposites: here it read as "no
75 // restriction", while a project's empty membership means "no space". An
76 // empty project passed into a query therefore used to return the whole
77 // corpus. The filter carries its own polarity, and the zero value is
78 // neither answer — Search refuses it rather than guessing, since a scope
79 // nobody set is a caller bug and both defaults are wrong for one of them.
80 // core.EverythingFilter() is how a caller says "every space".
81 Spaces core.SpaceFilter
82 // Sections restricts results to top-level sections ("specs", "notes",
83 // "reports"). Empty means every section except the activity log — see
84 // doc.LogSection: log entries summarise other documents, so left in they
85 // compete with the documents they describe for the same queries. Naming
86 // "log" here is the way back in.
87 Sections []string
88 Limit int
89 Offset int
90 }
91
92 // Hit is one ranked document. Space, Path, Rev and Anchor together are a
93 // pinned, immutable URL for the result: the read plane serves
94 // `/~owner/space/path?rev=<sha>#<anchor>`.
95 type Hit struct {
96 Space core.SpaceRef `json:"space"`
97 ID string `json:"id"`
98 Rev string `json:"rev,omitempty"`
99 Path string `json:"path,omitempty"`
100 // Anchor is the heading anchor within Path, set for an activity-log entry.
101 Anchor string `json:"anchor,omitempty"`
102 Title string `json:"title,omitempty"`
103 Section string `json:"section,omitempty"`
104 Lang Lang `json:"lang,omitempty"`
105 Score float64 `json:"score"`
106 // Snippet is a highlighted fragment of the matching text, with the matched
107 // terms wrapped in <mark>. Everything around them is HTML-escaped by bleve's
108 // formatter, so the fragment is safe to render as HTML and must be, or the
109 // marks show up as literal text.
110 Snippet string `json:"snippet,omitempty"`
111 }
112
113 // Results is one page of ranked hits.
114 type Results struct {
115 Hits []Hit `json:"hits"`
116 // Total is how many documents matched, not how many were returned.
117 Total uint64 `json:"total"`
118 Took time.Duration `json:"took"`
119 }
120
121 // Search runs a query against the global index.
122 204 func (x *Index) Search(ctx context.Context, q Query) (Results, error) {
123 204 text := strings.TrimSpace(q.Text)
124 204 if text == "" {
125 2 return Results{}, nil
126 2 }
127 202 if q.Limit <= 0 {
128 196 q.Limit = DefaultLimit
129 196 }
130 202 if q.Offset < 0 {
131 1 return Results{}, fmt.Errorf("search: negative offset %d", q.Offset)
132 1 }
133 201 if q.Spaces.IsZero() {
134 1 return Results{}, errors.New("search: query names no space scope; " +
135 1 "pass core.EverythingFilter() to search every space, or a project's filter to restrict it")
136 1 }
137 // A filter that selects no space — an empty project — has a known answer,
138 // and it is not "everything". Asking the index would be asking a question
139 // with no terms in it.
140 200 if q.Spaces.MatchesNothing() {
141 1 return Results{}, nil
142 1 }
143 199 bq, err := buildQuery(text, q)
144 199 if err != nil {
145 2 return Results{}, err
146 2 }
147
148 197 req := bleve.NewSearchRequestOptions(bq, q.Limit, q.Offset, false)
149 197 req.Fields = []string{fieldSpace, fieldRev, fieldPath, fieldAnchor, fieldTitle, fieldSection, fieldLang}
150 197 req.Highlight = bleve.NewHighlight()
151 197 req.Highlight.AddField(fieldBodyEN)
152 197 req.Highlight.AddField(fieldBodyRU)
153 197
154 197 x.mu.RLock()
155 197 defer x.mu.RUnlock()
156 197 if x.idx == nil {
157 1 return Results{}, errors.New("search: index is closed")
158 1 }
159 196 res, err := x.idx.SearchInContext(ctx, req)
160 196 if err != nil {
161 0 return Results{}, fmt.Errorf("search: query %q: %w", text, err)
162 0 }
163
164 196 out := Results{Total: res.Total, Took: res.Took, Hits: make([]Hit, 0, len(res.Hits))}
165 3328 for _, h := range res.Hits {
166 3328 hit, err := toHit(h)
167 3328 if err != nil {
168 0 return Results{}, err
169 0 }
170 3328 out.Hits = append(out.Hits, hit)
171 }
172 196 return out, nil
173 }
174
175 // buildQuery assembles the bleve query: the text across both languages' fields,
176 // conjoined with the space and section filters.
177 199 func buildQuery(text string, q Query) (query.Query, error) {
178 199 // The query text is run against all four analyzed fields. Both languages
179 199 // every time, not the detected language of the query: a two-word query is
180 199 // far too short to classify, and an English term inside a Russian document
181 199 // lives in that document's English field.
182 796 match := func(field string, boost float64) query.Query {
183 796 m := bleve.NewMatchQuery(text)
184 796 m.SetField(field)
185 796 m.SetBoost(boost)
186 796 return m
187 796 }
188 199 b := bleve.NewBooleanQuery()
189 199 b.AddMust(bleve.NewDisjunctionQuery(
190 199 match(fieldTitleEN, titleBoost),
191 199 match(fieldTitleRU, titleBoost),
192 199 match(fieldBodyEN, 1),
193 199 match(fieldBodyRU, 1),
194 199 ))
195 199
196 199 // The meta-project adds no term at all: a filter that excludes nothing is
197 199 // the absence of a restriction, not the enumeration of every space.
198 199 if refs := q.Spaces.Refs(); !q.Spaces.Everything() {
199 4 want := make([]query.Query, 0, len(refs))
200 5 for _, sp := range refs {
201 5 if sp.Owner == "" || sp.Name == "" {
202 1 return nil, errors.New("search: query carries an empty space")
203 1 }
204 4 t := bleve.NewTermQuery(sp.String())
205 4 t.SetField(fieldSpace)
206 4 want = append(want, t)
207 }
208 3 b.AddMust(bleve.NewDisjunctionQuery(want...))
209 }
210
211 198 if len(q.Sections) > 0 {
212 6 want := make([]query.Query, 0, len(q.Sections))
213 7 for _, s := range q.Sections {
214 7 if s == "" {
215 1 return nil, errors.New("search: query carries an empty section")
216 1 }
217 6 t := bleve.NewTermQuery(s)
218 6 t.SetField(fieldSection)
219 6 want = append(want, t)
220 }
221 5 b.AddMust(bleve.NewDisjunctionQuery(want...))
222 192 } else {
223 192 t := bleve.NewTermQuery(doc.LogSection)
224 192 t.SetField(fieldSection)
225 192 b.AddMustNot(t)
226 192 }
227 197 return b, nil
228 }
229
230 3328 func toHit(h *bsearch.DocumentMatch) (Hit, error) {
231 23296 str := func(field string) string {
232 23296 s, _ := h.Fields[field].(string)
233 23296 return s
234 23296 }
235 3328 raw := str(fieldSpace)
236 3328 if raw == "" {
237 0 return Hit{}, fmt.Errorf("search: indexed document %q carries no space", h.ID)
238 0 }
239 3328 sp, err := core.ParseSpaceRef(raw)
240 3328 if err != nil {
241 0 return Hit{}, fmt.Errorf("search: indexed document %q carries space %q: %w", h.ID, raw, err)
242 0 }
243 3328 hit := Hit{
244 3328 Space: sp,
245 3328 ID: strings.TrimPrefix(h.ID, raw+":"),
246 3328 Rev: str(fieldRev),
247 3328 Path: str(fieldPath),
248 3328 Anchor: str(fieldAnchor),
249 3328 Title: str(fieldTitle),
250 3328 Section: str(fieldSection),
251 3328 Lang: Lang(str(fieldLang)),
252 3328 Score: h.Score,
253 3328 }
254 3328 hit.Snippet = snippet(h, hit.Lang)
255 3328 return hit, nil
256 }
257
258 // snippet picks the highlighted fragment to show. The two body fields are the
259 // two halves of one document, and bleve highlights every requested field
260 // whether or not it matched — a field with no term locations yields its opening
261 // text, unmarked. Preferring the document's own language would therefore show
262 // the Russian opening of a document that matched on its English half. The
263 // matched field is the one that appears in Locations; the language preference
264 // only breaks a tie between two halves that both matched.
265 3328 func snippet(h *bsearch.DocumentMatch, lang Lang) string {
266 3328 order := []string{fieldBodyEN, fieldBodyRU}
267 3328 if lang == LangRU {
268 36 order = []string{fieldBodyRU, fieldBodyEN}
269 36 }
270 3341 for _, field := range order {
271 3341 if len(h.Locations[field]) == 0 {
272 14 continue
273 }
274 3327 if frags := h.Fragments[field]; len(frags) > 0 {
275 3327 return frags[0]
276 3327 }
277 }
278 // Matched on a title or on nothing highlightable: fall back to whichever
279 // half has text, so a hit is never returned with no context at all.
280 2 for _, field := range order {
281 2 if frags := h.Fragments[field]; len(frags) > 0 {
282 0 return frags[0]
283 0 }
284 }
285 1 return ""
286 }