coverage — ~bigbes/sr-ht-dolt — ede3b0bb — beads/ready.go

Coverage
96.1% 123/128 statements
Δ
—
Blob
c2189e4
Uncovered L422-L423L434-L435L452L480-L481L484-L485
1 package beads
2
3 import (
4 "context"
5 "net/url"
6 "sort"
7 "strings"
8 "time"
9
10 "sourcecraft.dev/bigbes/sr-ht-dolt/browse"
11 )
12
13 // --- the cross-database ready set ---------------------------------------------
14 //
15 // "What is ready to work" is answerable in each beads database on the instance
16 // and, until this, nowhere across them — which is the question the split into a
17 // global tracker plus per-project trackers was supposed to make askable.
18 //
19 // This is one function with two consumers: the /ready page renders it, and the
20 // MCP surface's ready_work with no database named calls it. Two implementations
21 // would answer differently the first time the ready rule moved.
22 //
23 // Authorization is not here. By the time a caller reaches this function it has
24 // already decided which databases this caller may browse (see the package
25 // comment: this package renders nothing and authorizes nothing); handing it a
26 // database is the statement that the caller may read it.
27
28 // The bounds every cross-database reading here is held to. They are named for
29 // /ready because that is the page that needed them first; the prefix index
30 // behind cross-database issue links is bounded by these same three numbers
31 // rather than by a second set of its own (see cache.go).
32 const (
33 // ReadyMaxDatabases bounds how many databases one call opens. Opening N
34 // stores per request is exactly what the per-request browse discipline does
35 // not scale to, and a page that hit this ceiling says so: a silent cap reads
36 // as "that is everything".
37 ReadyMaxDatabases = 64
38
39 // ReadyCacheTTL is how long a cached projection stands regardless of the head
40 // hash. The head-hash gate is what makes the cache correct; the TTL is what
41 // makes it impossible for the cache to be the reason a reader sees yesterday's
42 // answer.
43 ReadyCacheTTL = 60 * time.Second
44
45 // readyCacheMaxEntries bounds the cache by entry count. A cache is not a
46 // store: over the ceiling it drops what has expired and, failing that, starts
47 // again, rather than growing with the instance.
48 readyCacheMaxEntries = 256
49 )
50
51 // ReadyDatabase names one database to consider. ID is the identity the cache is
52 // keyed on — the repository row id, which no two databases share and which
53 // survives a rename.
54 //
55 // The field names are OwnerName and Name deliberately: the freshness partial the
56 // beads views render is handed a database here rather than a repository, and a
57 // partial that reads .OwnerName must find it on both.
58 type ReadyDatabase struct {
59 ID int
60 OwnerName string
61 Name string
62 }
63
64 // Slug is the database's "owner/name" address — what ?db= names and what the
65 // group headers show.
66 372 func (d ReadyDatabase) Slug() string { return d.OwnerName + "/" + d.Name }
67
68 // ReadySession is the read-only surface one database is read through. It is
69 // BrowseSession (the rows) plus the three things this aggregation needs that a
70 // projection of a single, already-opened database does not: the branch list (for
71 // the head hash the cache gates on), the table list (for the fingerprint), the
72 // log (for the group's own freshness line) — and Close, because here the
73 // aggregation owns the session's lifetime.
74 //
75 // web's BrowseSession and *browse.DB satisfy it structurally.
76 type ReadySession interface {
77 BrowseSession
78 Branches(ctx context.Context) ([]browse.Branch, error)
79 Tables(ctx context.Context, refStr string) ([]browse.TableInfo, error)
80 Log(ctx context.Context, refStr, fromHash string, limit int) ([]browse.CommitInfo, string, error)
81 Close() error
82 }
83
84 // ReadyOpener opens one database. The returned session is closed by this
85 // package — a caller that kept it would be hoarding a file handle and a memory
86 // mapping, which is what per-request opening exists to avoid.
87 type ReadyOpener func(ctx context.Context, db ReadyDatabase) (ReadySession, error)
88
89 // ReadyGroup is one database's ready work.
90 type ReadyGroup struct {
91 Database ReadyDatabase
92 Ref string // the branch the ready set was read from
93 Head *browse.CommitInfo // that branch's head, or nil when it cannot be read
94 Cards []Card // ready issues, priority then id
95
96 // Truncated says one of the tables this group was projected from — issues,
97 // dependencies, labels, custom_statuses — exceeded Max and came back clipped.
98 // The cards below are then the ready work among the rows that were read,
99 // which is not the same claim as this database's ready work.
100 //
101 // It is a row clip and has nothing to do with ReadyView.Capped, which counts
102 // databases.
103 Truncated bool
104 // ShownOf is this database's issues total, clipped or not: what exists,
105 // against the at most Max rows the projection read. IssuesClipped is the
106 // comparison callers usually want.
107 ShownOf int
108 }
109
110 // IssuesClipped reports that this database's issues table itself exceeded Max,
111 // so the ready rule was applied to its first Max rows only. Truncated is the
112 // wider fact (any input table was clipped); this is the one that says ready work
113 // may be missing from the group rather than merely mislabelled.
114 3 func (g ReadyGroup) IssuesClipped() bool { return g.ShownOf > Max }
115
116 // ReadyTruncation is one database whose ready set was projected from a clipped
117 // read. It exists separately from ReadyGroup because a database with no group is
118 // exactly the case that needs saying: a tracker whose ready work sits past Max
119 // is absent from Groups for the same reason a tracker with no ready work is, and
120 // without this the two are indistinguishable.
121 type ReadyTruncation struct {
122 Database ReadyDatabase
123 // ShownOf is that database's issues total, clipped or not. A truncation whose
124 // ShownOf is within Max was caused by one of the other three tables.
125 ShownOf int
126 }
127
128 // IssuesClipped reports that this database's issues table itself exceeded Max.
129 2 func (t ReadyTruncation) IssuesClipped() bool { return t.ShownOf > Max }
130
131 // ReadyFailure is one database that could not be read. It carries the error for
132 // the caller's log and nothing for the reader: an error string on a page is how
133 // a store path and a dolt internal end up in a browser (sr-ht-dolt-7ta).
134 type ReadyFailure struct {
135 Database ReadyDatabase
136 Err error
137 }
138
139 // ReadyView is the whole answer: the groups, what was considered, and the three
140 // facts a reader needs in order not to over-read it (the ceiling, that some
141 // databases could not be read, and that some were read only in part).
142 type ReadyView struct {
143 Groups []ReadyGroup
144 Total int // ready cards across every group, after filtering
145 Considered int // databases actually opened (or served from cache)
146 Capped bool // there were more candidates than Max
147 Max int // ReadyMaxDatabases, so the page can name the number it hit
148 Failed []ReadyFailure
149 Filter ReadyFilter
150 Options ReadyOptions
151
152 // Truncated lists every database considered whose rows came back clipped at
153 // beads.Max, in the order the caller listed them. It is the complete set, and
154 // deliberately wider than the groups: a database whose ready work sits past
155 // the cap produces no group at all, and that is the case a per-group flag
156 // cannot report.
157 //
158 // Capped and this are two different bounds and neither implies the other.
159 // Capped counts databases — there were more trackers than one call may open.
160 // This counts rows inside a database that was opened and read.
161 Truncated []ReadyTruncation
162 // Query is the request's query as parsed, carried so a link can rebuild this
163 // exact URL with one key replaced (web's withQuery) instead of re-listing the
164 // parameters it happens to know about.
165 Query url.Values
166 }
167
168 // ReadyOptions lists the distinct values present across the ready set, so the
169 // filter dropdowns offer only real choices. Collected before the card filters
170 // narrow anything, so the options do not shrink as a filter is applied.
171 type ReadyOptions struct {
172 Assignees []string
173 Priorities []string
174 }
175
176 // ReadyFilter is the page's filter state: ?q=, ?assignee=, ?priority= over the
177 // cards, and a repeatable ?db=<owner>/<name> over the databases.
178 type ReadyFilter struct {
179 Query string
180 Assignee string
181 Priority string
182 Databases []string // empty means every database the caller was given
183 }
184
185 // ParseReadyFilter reads the filter out of a request query.
186 2 func ParseReadyFilter(q url.Values) ReadyFilter {
187 2 f := ReadyFilter{
188 2 Query: strings.TrimSpace(q.Get("q")),
189 2 Assignee: strings.TrimSpace(q.Get("assignee")),
190 2 Priority: strings.TrimSpace(q.Get("priority")),
191 2 }
192 3 for _, d := range q["db"] {
193 3 if d = strings.TrimSpace(d); d != "" {
194 2 f.Databases = append(f.Databases, d)
195 2 }
196 }
197 2 return f
198 }
199
200 // Active reports whether any filter is set (drives the "Clear" link and the
201 // empty-page wording).
202 2 func (f ReadyFilter) Active() bool {
203 2 return f.Query != "" || f.Assignee != "" || f.Priority != "" || len(f.Databases) > 0
204 2 }
205
206 // selects reports whether a database is one of the ones asked for. No ?db= at
207 // all means every database the caller was handed.
208 182 func (f ReadyFilter) selects(slug string) bool {
209 182 if len(f.Databases) == 0 {
210 113 return true
211 113 }
212 69 for _, d := range f.Databases {
213 69 if d == slug {
214 2 return true
215 2 }
216 }
217 67 return false
218 }
219
220 // matches reports whether one ready card passes every set card filter.
221 2808 func (f ReadyFilter) matches(c Card) bool {
222 2808 if f.Assignee != "" && c.Assignee != f.Assignee {
223 6 return false
224 6 }
225 2802 if f.Priority != "" && c.Priority != f.Priority {
226 2 return false
227 2 }
228 2800 if f.Query != "" {
229 4 hay := strings.ToLower(c.ID + " " + c.Title)
230 4 if !strings.Contains(hay, strings.ToLower(f.Query)) {
231 3 return false
232 3 }
233 }
234 2797 return true
235 }
236
237 // ReadyCache holds one ready projection per database, keyed by the repository id
238 // and gated on the head hash. The gate, the TTL and the entry ceiling are the
239 // shared projectionCache's (cache.go), which the prefix index is bounded by too:
240 // one set of rules, one lifetime.
241 //
242 // What it holds is the projection — the ready cards — and never an open
243 // session. An open store is a file handle and a memory mapping; caching those is
244 // the thing per-request opening exists to prevent.
245 type ReadyCache struct {
246 projectionCache[readyEntry]
247 }
248
249 // readyEntry is one database's cached ready projection. The head it was read at
250 // and the time it was stored are the cache's, not this struct's.
251 type readyEntry struct {
252 ref string // the branch it was read from
253 beads bool // the fingerprint held — a false entry is a database to skip
254 // commit is the head commit for the group's freshness line. It cannot go
255 // stale under an unmoved head, so it is cached with the cards and a cache hit
256 // reads no log either.
257 commit *browse.CommitInfo
258 cards []Card
259 // read is what the row reads reported about their own completeness. It is
260 // cached beside the cards for the same reason: a projection served from the
261 // cache has to say what the read that produced it said, and a cache hit
262 // reads no row it could learn this from a second time.
263 read readyRead
264 }
265
266 // readyRead is what one database's row reads reported about their own
267 // completeness: whether any table came back clipped at Max, and the issues
268 // table's true total. Every number in it comes from the reads readyCards
269 // already makes.
270 type readyRead struct {
271 truncated bool // some table this projection reads exceeded Max
272 issuesTotal int // the issues table's reported total, clipped or not
273 }
274
275 // NewReadyCache returns an empty cache. The zero value works too — the map is
276 // allocated on first store — and this exists for the callers that hold one by
277 // pointer.
278 24 func NewReadyCache() *ReadyCache {
279 24 return &ReadyCache{}
280 24 }
281
282 // ReadyAcross collects the ready set of every database it is handed, grouped by
283 // database: groups ordered by ready count desc then name, cards inside a group
284 // by priority then id, and a database with no ready work absent rather than
285 // shown empty.
286 //
287 // now is the clock the TTL is measured against, passed in rather than read here
288 // for the reason BuildMemories takes one: this package reads no hidden clock,
289 // and a caller that pins its own gets a deterministic answer.
290 //
291 // It returns no error. One database that cannot be opened or read costs itself
292 // only — it lands in Failed for the caller's log — because a page that 500s
293 // because the seventeenth store is corrupt answers nothing about the other
294 // sixteen.
295 func ReadyAcross(
296 ctx context.Context,
297 dbs []ReadyDatabase,
298 open ReadyOpener,
299 cache *ReadyCache,
300 filter ReadyFilter,
301 now time.Time,
302 29 ) *ReadyView {
303 29 view := &ReadyView{Filter: filter, Max: ReadyMaxDatabases}
304 29
305 29 // ?db= narrows the candidates before the ceiling applies: a database the
306 29 // caller named is the one thing the cap must not be able to drop.
307 29 candidates := make([]ReadyDatabase, 0, len(dbs))
308 182 for _, d := range dbs {
309 182 if filter.selects(d.Slug()) {
310 115 candidates = append(candidates, d)
311 115 }
312 }
313 29 if len(candidates) > ReadyMaxDatabases {
314 1 // The first Max in the order the caller listed them, which is the caller's
315 1 // own ordering (newest first, as the listing produces it) and stable
316 1 // across requests.
317 1 candidates = candidates[:ReadyMaxDatabases]
318 1 view.Capped = true
319 1 }
320 29 view.Considered = len(candidates)
321 29
322 29 assignees, priorities := map[string]bool{}, map[string]bool{}
323 112 for _, d := range candidates {
324 112 entry, err := readyProjection(ctx, d, open, cache, now)
325 112 if err != nil {
326 2 view.Failed = append(view.Failed, ReadyFailure{Database: d, Err: err})
327 2 continue
328 }
329 110 if !entry.beads {
330 3 // Not a beads database (or a store with no branches at all): skipped
331 3 // silently, exactly as the view tabs skip it.
332 3 continue
333 }
334 107 if entry.read.truncated {
335 6 // Recorded before the card filters and before the empty-group skip
336 6 // below: a database whose ready work was left past the cap has no
337 6 // group to carry the fact, and it is that database the reader most
338 6 // needs named.
339 6 view.Truncated = append(view.Truncated, ReadyTruncation{
340 6 Database: d,
341 6 ShownOf: entry.read.issuesTotal,
342 6 })
343 6 }
344 107 cards := make([]Card, 0, len(entry.cards))
345 2808 for _, c := range entry.cards {
346 2808 if c.Assignee != "" {
347 2808 assignees[c.Assignee] = true
348 2808 }
349 2808 if c.Priority != "" {
350 2808 priorities[c.Priority] = true
351 2808 }
352 2808 if filter.matches(c) {
353 2797 cards = append(cards, c)
354 2797 }
355 }
356 107 if len(cards) == 0 {
357 6 // Nothing ready here: absent from the page rather than an empty group.
358 6 continue
359 }
360 101 view.Groups = append(view.Groups, ReadyGroup{
361 101 Database: d,
362 101 Ref: entry.ref,
363 101 Head: entry.commit,
364 101 Cards: cards,
365 101 Truncated: entry.read.truncated,
366 101 ShownOf: entry.read.issuesTotal,
367 101 })
368 101 view.Total += len(cards)
369 }
370
371 92 sort.SliceStable(view.Groups, func(i, j int) bool {
372 92 ni, nj := len(view.Groups[i].Cards), len(view.Groups[j].Cards)
373 92 if ni != nj {
374 12 return ni > nj
375 12 }
376 80 return view.Groups[i].Database.Slug() < view.Groups[j].Database.Slug()
377 })
378 29 view.Options = ReadyOptions{
379 29 Assignees: sortedKeys(assignees),
380 29 Priorities: sortedKeys(priorities), // single digits sort numerically as strings
381 29 }
382 29 return view
383 }
384
385 // readyProjection returns one database's ready cards, from the cache when the
386 // head has not moved and by reading the store otherwise.
387 //
388 // The order is the whole point of the gate: open, list branches, and only then
389 // consult the cache. Opening a session and reading the branch list is cheap;
390 // reading and projecting issues + dependencies is not, and on a hit neither the
391 // tables, nor the rows, nor the log are touched.
392 func readyProjection(
393 ctx context.Context,
394 d ReadyDatabase,
395 open ReadyOpener,
396 cache *ReadyCache,
397 now time.Time,
398 112 ) (readyEntry, error) {
399 112 sess, err := open(ctx, d)
400 112 if err != nil {
401 1 return readyEntry{}, err
402 1 }
403 111 defer sess.Close()
404 111
405 111 branches, err := sess.Branches(ctx)
406 111 if err != nil {
407 1 return readyEntry{}, err
408 1 }
409 110 ref := browse.DefaultBranch(branches)
410 110 if ref == "" {
411 1 // A store with no branches carries no tables either: nothing to skip past
412 1 // and nothing to cache.
413 1 return readyEntry{}, nil
414 1 }
415 109 head := headHashOf(branches, ref)
416 109 if e, ok := cache.lookup(d.ID, head, now); ok {
417 9 return e, nil
418 9 }
419
420 100 tables, err := sess.Tables(ctx, ref)
421 100 if err != nil {
422 0 return readyEntry{}, err
423 0 }
424 100 entry := readyEntry{ref: ref, beads: Applies(tables)}
425 100 if !entry.beads {
426 1 // Cached too: a database that is not a tracker is not one on the next
427 1 // request either, and the fingerprint is worth exactly one table listing.
428 1 cache.store(d.ID, head, now, entry)
429 1 return entry, nil
430 1 }
431
432 99 cards, read, err := readyCards(ctx, sess, ref)
433 99 if err != nil {
434 0 return readyEntry{}, err
435 0 }
436 99 entry.cards = cards
437 99 entry.read = read
438 99 entry.commit = readyHead(ctx, sess, ref)
439 99 cache.store(d.ID, head, now, entry)
440 99 return entry, nil
441 }
442
443 // headHashOf returns the head hash of the named branch, or "" when the branch
444 // list does not carry it. An empty hash disables the cache for that database
445 // rather than letting an entry stand un-gated.
446 209 func headHashOf(branches []browse.Branch, ref string) string {
447 209 for _, b := range branches {
448 209 if b.Name == ref {
449 209 return b.Head
450 209 }
451 }
452 0 return ""
453 }
454
455 // readyHead reads the head commit of ref for the group's freshness line, and
456 // returns nil rather than an error on both failure arms. "Ready" read from a
457 // store that stopped receiving pushes is exactly the claim this page must not
458 // make silently — but a log that cannot be read may not be the reason the ready
459 // set is withheld either, so the line is simply absent (the partial renders
460 // nothing for nil).
461 99 func readyHead(ctx context.Context, sess ReadySession, ref string) *browse.CommitInfo {
462 99 commits, _, err := sess.Log(ctx, ref, "", 1)
463 99 if err != nil || len(commits) == 0 {
464 1 return nil
465 1 }
466 98 return &commits[0]
467 }
468
469 // readyCards projects one database's ready set: the same tables the board reads,
470 // through the same ready rule (readyRow), sorted priority then id.
471 //
472 // It also returns what those reads said about their own completeness. All four
473 // tables count towards it: a clipped issues table leaves ready work unread, and
474 // a clipped dependencies, labels or custom_statuses table changes the verdict on
475 // the issues that were read — a blocking edge past the cap is a card called
476 // ready that is not.
477 99 func readyCards(ctx context.Context, sess BrowseSession, ref string) ([]Card, readyRead, error) {
478 99 issues, issuesTotal, err := readRows(ctx, sess, ref, "issues")
479 99 if err != nil {
480 0 return nil, readyRead{}, err
481 0 }
482 99 deps, depsTotal, err := readRows(ctx, sess, ref, "dependencies")
483 99 if err != nil {
484 0 return nil, readyRead{}, err
485 0 }
486 99 labels, labelsTotal, _ := readRowsOptional(ctx, sess, ref, "labels")
487 99 statuses, statusesTotal, _ := readRowsOptional(ctx, sess, ref, "custom_statuses")
488 99
489 99 read := readyRead{
490 99 truncated: issuesTotal > Max || depsTotal > Max ||
491 99 labelsTotal > Max || statusesTotal > Max,
492 99 issuesTotal: issuesTotal,
493 99 }
494 99
495 99 catByStatus := indexStatusCategories(statuses)
496 99 issueCols := indexCols(issues.Columns)
497 99 catByIssue := indexIssueCategories(issues, issueCols, catByStatus)
498 99 depIdx := indexDeps(deps, catByIssue)
499 99 labelsByIssue := indexLabels(labels)
500 99
501 99 var cards []Card
502 8209 for _, r := range rowsOf(issues) {
503 8209 id := cell(issueCols, r, "id")
504 8209 cat := catByIssue[id]
505 8209 blocked := truthy(cell(issueCols, r, "is_blocked")) || depIdx.blockedOpen[id]
506 8209 if !readyRow(cat, blocked, r, issueCols) {
507 6081 continue
508 }
509 2128 cards = append(cards, Card{
510 2128 ID: id,
511 2128 Title: cell(issueCols, r, "title"),
512 2128 Type: cell(issueCols, r, "issue_type"),
513 2128 Priority: cell(issueCols, r, "priority"),
514 2128 Assignee: cell(issueCols, r, "assignee"),
515 2128 Labels: labelsByIssue[id],
516 2128 BlockedBy: depIdx.blockedByCount[id],
517 2128 Blocks: depIdx.blocksCount[id],
518 2128 Ready: true,
519 2128 Category: cat,
520 2128 })
521 }
522 99 sortReadyCards(cards)
523 99 return cards, read, nil
524 }
525
526 // sortReadyCards orders a group by priority (0 = highest first), then id. The
527 // board sorts by created_at between the two; here the id is the tie-break,
528 // because across databases the created_at of one tracker says nothing about the
529 // order of another's.
530 99 func sortReadyCards(cards []Card) {
531 2471 sort.SliceStable(cards, func(i, j int) bool {
532 2471 pi, pj := priorityRank(cards[i].Priority), priorityRank(cards[j].Priority)
533 2471 if pi != pj {
534 12 return pi < pj
535 12 }
536 2459 return cards[i].ID < cards[j].ID
537 })
538 }