coverage — ~bigbes/sr-ht-dolt — 3523280c — beads/memory.go

Coverage
92.5% 124/134 statements
Δ
+0.0
Blob
98951f1
Uncovered L171-L172L192L291-L292L306-L307L312-L313L320-L321L360-L361L378-L379L436-L439
1 package beads
2
3 import (
4 "context"
5 "fmt"
6 "net/url"
7 "regexp"
8 "sort"
9 "strings"
10 "time"
11
12 "sourcecraft.dev/bigbes/sr-ht-dolt/browse"
13 )
14
15 // --- the memories bd remember writes ------------------------------------------
16
17 // memoryTable is where bd keeps them: the beads config table, as ordinary
18 // key/value rows. memoryPrefix marks the keys that are memories; the rest of the
19 // table is the tracker's settings (issue_prefix, compact_tier2_days, …).
20 const (
21 memoryTable = "config"
22 memoryPrefix = "kv.memory."
23 )
24
25 // memoryWalkMax bounds the revision walk (see walkMemories): at most this many
26 // commits back from the head of the ref. The most active tracker on this
27 // instance had 225 commits three weeks in, so this is roughly two months of
28 // headroom at that rate. A memory not attributed inside the walk carries no date
29 // at all rather than one the walk cannot support.
30 const memoryWalkMax = 500
31
32 // MemoryStaleAfter is how old a memory has to be before the page questions it.
33 // It is one constant and not a per-request knob, and what it produces is a
34 // question ("stale?") rather than a verdict: some memories are meant to be
35 // permanent, and only the reader knows which.
36 const MemoryStaleAfter = 60 * 24 * time.Hour
37
38 // MemorySession is the seam the memory projection reads through. It is wider
39 // than BrowseSession because a memory has no timestamp — the config row is
40 // (key, value) and nothing else — so the date has to come out of the history:
41 // Log for the commits and TableHash to skip the ones that did not touch config.
42 // It is still only what this projection calls, and web's BrowseSession and
43 // mcpsrv's (and *browse.DB itself) satisfy it structurally.
44 type MemorySession interface {
45 BrowseSession
46 // Log lists commits from the head of refStr, newest first.
47 Log(ctx context.Context, refStr, fromHash string, limit int) ([]browse.CommitInfo, string, error)
48 // TableHash is the content hash of a table at a ref, ok=false when the table
49 // does not exist there. It reads no rows, which is what makes the walk cheap.
50 TableHash(ctx context.Context, refStr, table string) (string, bool, error)
51 }
52
53 // MemoryRevision is when a memory's value last changed: the commit that wrote
54 // it, taken from the history rather than from the row.
55 type MemoryRevision struct {
56 Commit string
57 Date time.Time
58 Author string
59 }
60
61 // Memory is one `bd remember` entry.
62 type Memory struct {
63 Slug string // the config key with the "kv.memory." prefix stripped
64 // Text is the value with both newline spellings normalised. It is markdown —
65 // `bd remember` stores whatever was written, and what is written is the same
66 // markdown the memory files carry — and it is left as text here: this package
67 // renders nothing, and the split into paragraphs, lists and code blocks is
68 // the renderer's, not a projection's.
69 Text string
70 // Revision is the commit that last changed this key's value, or nil when the
71 // walk could not reach it (see MemoryView.WalkTruncated).
72 Revision *MemoryRevision
73 // Stale reports that the memory is older than MemoryStaleAfter. For a memory
74 // with no Revision it is set only when the walk's own oldest commit is
75 // already past the threshold — that much is known even without a date.
76 Stale bool
77 }
78
79 // MemoryView is the opaque .Data handed to memory.html.
80 type MemoryView struct {
81 Memories []Memory
82 Total int // memories in the tracker, before ?q= / ?key= narrowing
83 Search string // ?q=, sticky form state
84 Key string // ?key=<slug>, a single memory
85 Sort string // MemorySortSlug (default) | MemorySortAge
86 // Query is the request's query as parsed, carried so the sort toggle can
87 // rebuild this exact URL with one key replaced (web's withQuery) instead of
88 // re-listing the parameters it happens to know about.
89 Query url.Values
90 // WalkTruncated says the revision walk stopped at WalkMax commits with the
91 // history still going. It is the only way a Memory can carry no Revision, and
92 // it is what the page says instead of a date.
93 WalkTruncated bool
94 WalkMax int // memoryWalkMax, so the page can name the number it hit
95
96 // ConfigTruncated says a read of the config table came back clipped at
97 // beads.Max. That is a different clip from WalkTruncated, which bounds the
98 // history in commits; this one bounds a single read in rows, and it covers
99 // both reads this view makes: the one at ref that the memories themselves
100 // come from, and the per-commit ones the walk dates them by. A clipped read
101 // in the walk leaves the dates unsafe rather than the list short — a key
102 // whose row fell past the cap reads as absent there, which is
103 // indistinguishable from a key that had not been written yet.
104 ConfigTruncated bool
105 // ConfigShownOf is the config table's reported total at ref, clipped or not:
106 // rows that exist, against the at most Max that were read. It counts the
107 // whole table, memory keys and tracker settings alike, because that is what
108 // the cap applies to.
109 ConfigShownOf int
110 }
111
112 // ConfigClipped reports that the config read the memories themselves come from
113 // exceeded Max, so Memories and Total cover its first Max rows only and a
114 // memory may be missing from the list entirely. ConfigTruncated is the wider
115 // fact (any config read, here or in the walk, was clipped).
116 3 func (v *MemoryView) ConfigClipped() bool { return v.ConfigShownOf > Max }
117
118 // Memory sort orders. Slug is the default; Age is the review queue.
119 const (
120 MemorySortSlug = "slug"
121 MemorySortAge = "age"
122 )
123
124 // AppliesMemories fingerprints a tracker that can carry memories: the beads
125 // fingerprint plus a config table with key and value columns. Like every
126 // Applies, it sees table shapes and never rows, so a tracker whose config holds
127 // no memory at all still gets the tab and renders an empty state.
128 4 func AppliesMemories(tables []browse.TableInfo) bool {
129 4 if !Applies(tables) {
130 1 return false
131 1 }
132 11 for _, t := range tables {
133 11 if t.Name != memoryTable {
134 9 continue
135 }
136 2 var haveKey, haveValue bool
137 3 for _, c := range t.Columns {
138 3 switch c.Name {
139 2 case "key":
140 2 haveKey = true
141 1 case "value":
142 1 haveValue = true
143 }
144 }
145 2 return haveKey && haveValue
146 }
147 1 return false
148 }
149
150 // BuildMemories reads the memories at ref and dates each one from the history.
151 // now is the clock staleness is measured against, passed in rather than read
152 // here: this package renders nothing and reads no hidden clock, and a caller
153 // that pins its own clock (the web view, its tests) gets a deterministic answer.
154 //
155 // A missing config table degrades to no memories — the same treatment the other
156 // optional tables get — but a history that cannot be read is an error, not an
157 // empty answer: the date is what this view is for, and a page that silently
158 // dropped every date would look exactly like a tracker whose memories are all
159 // older than the walk.
160 21 func BuildMemories(ctx context.Context, sess MemorySession, ref string, query url.Values, now time.Time) (*MemoryView, error) {
161 21 view := &MemoryView{
162 21 Search: strings.TrimSpace(query.Get("q")),
163 21 Key: strings.TrimSpace(query.Get("key")),
164 21 Sort: parseMemorySort(query.Get("sort")),
165 21 Query: query,
166 21 WalkMax: memoryWalkMax,
167 21 }
168 21
169 21 rows, configTotal, err := readRowsOptional(ctx, sess, ref, memoryTable)
170 21 if err != nil {
171 0 return nil, err
172 0 }
173 21 view.ConfigShownOf = configTotal
174 21 view.ConfigTruncated = configTotal > Max
175 21 if rows == nil {
176 1 return view, nil
177 1 }
178
179 // The whole config table, narrowed to the memory keys. raw keeps the stored
180 // value under its full key: the walk compares values as they are stored, and
181 // normalising first would make two spellings of the same text look equal.
182 20 cols := indexCols(rows.Columns)
183 20 raw := map[string]string{}
184 20 texts := map[string]string{}
185 2060 for _, r := range rowsOf(rows) {
186 2060 key := cell(cols, r, "key")
187 2060 if !strings.HasPrefix(key, memoryPrefix) {
188 14 continue
189 }
190 2046 slug := strings.TrimPrefix(key, memoryPrefix)
191 2046 if slug == "" {
192 0 continue
193 }
194 2046 raw[key] = cell(cols, r, "value")
195 2046 texts[key] = normalizeMemoryText(raw[key])
196 }
197 20 view.Total = len(raw)
198 20
199 20 // Narrow before walking: the walk is per-key work, and a ?key= page has no
200 20 // business dating the other eight memories.
201 20 tracked := map[string]string{}
202 2046 for key, value := range raw {
203 2046 slug := strings.TrimPrefix(key, memoryPrefix)
204 2046 if view.Key != "" && slug != view.Key {
205 5 continue
206 }
207 2041 if !matchesMemorySearch(view.Search, slug, texts[key]) {
208 7 continue
209 }
210 2034 tracked[key] = value
211 }
212 20 if len(tracked) == 0 {
213 3 return view, nil
214 3 }
215
216 17 walk, err := walkMemories(ctx, sess, ref, tracked)
217 17 if err != nil {
218 1 return nil, err
219 1 }
220 16 view.WalkTruncated = walk.truncated
221 16 view.ConfigTruncated = view.ConfigTruncated || walk.configClipped
222 16
223 16 view.Memories = make([]Memory, 0, len(tracked))
224 2031 for key := range tracked {
225 2031 m := Memory{
226 2031 Slug: strings.TrimPrefix(key, memoryPrefix),
227 2031 Text: texts[key],
228 2031 }
229 2031 if rev, ok := walk.revisions[key]; ok {
230 2029 m.Revision = &rev
231 2029 m.Stale = now.Sub(rev.Date) > MemoryStaleAfter
232 2029 } else if !walk.oldest.IsZero() {
233 2 // No date, but a floor: the memory is at least as old as the oldest
234 2 // commit the walk examined. When that alone is past the threshold the
235 2 // question is supportable; otherwise it is not asked.
236 2 m.Stale = now.Sub(walk.oldest) > MemoryStaleAfter
237 2 }
238 2031 view.Memories = append(view.Memories, m)
239 }
240 16 sortMemories(view.Memories, view.Sort)
241 16
242 16 return view, nil
243 }
244
245 // memoryWalk is what walkMemories learned: the commit each key was last written
246 // by, whether the walk ran out of budget before the history ran out, and the
247 // date of the oldest commit it examined.
248 type memoryWalk struct {
249 revisions map[string]MemoryRevision
250 truncated bool
251 oldest time.Time
252 // configClipped says one of the per-commit config reads exceeded Max. It is
253 // a different bound from truncated — rows rather than commits — and it costs
254 // the attribution its footing: a key the read never reached looks like a key
255 // that commit had not written yet, which is precisely what the comparison
256 // below treats as a write.
257 configClipped bool
258 }
259
260 // walkMemories attributes each tracked key to the commit that last changed its
261 // value, walking the history of ref newest to oldest, at most memoryWalkMax
262 // commits.
263 //
264 // The trick that makes it affordable is the table hash. At each commit the
265 // content hash of config is O(1) and reads no rows; when it equals the hash at
266 // the newer neighbour, config is byte-identical across that step and the newer
267 // commit wrote no memory — skip, read nothing. Only a commit that actually
268 // touched config costs a row read, and then each still-unresolved key whose
269 // value differs from the newer neighbour's was written *by that newer commit*.
270 //
271 // The commit message is not the signal. `bd remember` does write
272 // "bd: remember (auto-commit) by <author>", but that is a claim by whoever wrote
273 // it; the table hash is the fact.
274 //
275 // A key still unresolved when the history itself runs out was present with this
276 // value at the root commit, so the root is what wrote it — that is a date the
277 // walk supports. A key unresolved because the walk hit its budget gets no date
278 // at all, and truncated says so.
279 //
280 // The walk reads the log's linearization (Log is reverse-topological). Beads
281 // histories are linear chains of auto-commits, which this is exact for; across a
282 // merge, attribution is to the nearest commit in that order.
283 17 func walkMemories(ctx context.Context, sess MemorySession, ref string, tracked map[string]string) (memoryWalk, error) {
284 17 out := memoryWalk{revisions: map[string]MemoryRevision{}}
285 17
286 17 commits, next, err := sess.Log(ctx, ref, "", memoryWalkMax)
287 17 if err != nil {
288 1 return memoryWalk{}, fmt.Errorf("beads: read history of %q: %w", ref, err)
289 1 }
290 16 if len(commits) == 0 {
291 0 return out, nil
292 0 }
293 16 out.oldest = commits[len(commits)-1].Date
294 16 out.truncated = next != ""
295 16
296 16 // unresolved carries each key's value at the newer neighbour of the commit
297 16 // being examined; it starts as the value at the head, which is where the
298 16 // memories themselves were read.
299 16 unresolved := make(map[string]string, len(tracked))
300 2031 for k, v := range tracked {
301 2031 unresolved[k] = v
302 2031 }
303
304 16 newerHash, err := memoryTableHash(ctx, sess, commits[0].Hash)
305 16 if err != nil {
306 0 return memoryWalk{}, err
307 0 }
308
309 1039 for i := 1; i < len(commits) && len(unresolved) > 0; i++ {
310 1039 hash, err := memoryTableHash(ctx, sess, commits[i].Hash)
311 1039 if err != nil {
312 0 return memoryWalk{}, err
313 0 }
314 1039 if hash == newerHash {
315 1018 continue // config unchanged across this step: nothing to read
316 }
317
318 21 older, olderTotal, err := memoryValuesAt(ctx, sess, commits[i].Hash)
319 21 if err != nil {
320 0 return memoryWalk{}, err
321 0 }
322 21 out.configClipped = out.configClipped || olderTotal > Max
323 21 newer := commits[i-1]
324 44 for key, newerValue := range unresolved {
325 44 if older[key] == newerValue {
326 22 continue
327 }
328 22 out.revisions[key] = MemoryRevision{
329 22 Commit: newer.Hash,
330 22 Date: newer.Date,
331 22 Author: newer.Author,
332 22 }
333 22 delete(unresolved, key)
334 }
335 22 for key := range unresolved {
336 22 unresolved[key] = older[key]
337 22 }
338 21 newerHash = hash
339 }
340
341 16 if !out.truncated {
342 14 // The history ended with these keys never changing: the oldest commit
343 14 // walked is the root, and it carries the value we are looking at.
344 14 root := commits[len(commits)-1]
345 2007 for key := range unresolved {
346 2007 out.revisions[key] = MemoryRevision{Commit: root.Hash, Date: root.Date, Author: root.Author}
347 2007 }
348 }
349
350 16 return out, nil
351 }
352
353 // memoryTableHash is the config table's content hash at one commit. A table that
354 // does not exist there is "" — an ordinary answer while walking backwards past
355 // the commit that created it, and one that compares correctly against another
356 // commit where it is equally absent.
357 1055 func memoryTableHash(ctx context.Context, sess MemorySession, at string) (string, error) {
358 1055 hash, ok, err := sess.TableHash(ctx, at, memoryTable)
359 1055 if err != nil {
360 0 return "", fmt.Errorf("beads: hash of %s at %s: %w", memoryTable, at, err)
361 0 }
362 1055 if !ok {
363 1 return "", nil
364 1 }
365 1054 return hash, nil
366 }
367
368 // memoryValuesAt reads the config table at one commit as key → value, and the
369 // total that read reported. A missing table is an empty map, which is what it
370 // means here: no key had a value yet.
371 //
372 // The total is returned rather than dropped because "no key had a value yet"
373 // and "the key sits past Max" arrive at this caller as the same empty slot, and
374 // only the total tells them apart.
375 21 func memoryValuesAt(ctx context.Context, sess MemorySession, at string) (map[string]string, int, error) {
376 21 rows, total, err := readRowsOptional(ctx, sess, at, memoryTable)
377 21 if err != nil {
378 0 return nil, 0, err
379 0 }
380 21 out := map[string]string{}
381 21 if rows == nil {
382 1 return out, total, nil
383 1 }
384 20 cols := indexCols(rows.Columns)
385 2060 for _, r := range rowsOf(rows) {
386 2060 if key := cell(cols, r, "key"); key != "" {
387 2060 out[key] = cell(cols, r, "value")
388 2060 }
389 }
390 20 return out, total, nil
391 }
392
393 // memoryEscapedNewline matches the two-character escapes that reach the value
394 // because the memory was typed into a shell string: "\r\n" and "\n" written out
395 // as backslash sequences rather than as newlines.
396 var memoryEscapedNewline = regexp.MustCompile(`\\r\\n|\\n`)
397
398 // normalizeMemoryText brings a stored value's two newline spellings together.
399 // The same memory arrives with real newlines when it was written from a file or
400 // a heredoc and with literal backslash-n when it was typed into a shell string,
401 // and both spellings turn up in the same tracker.
402 2046 func normalizeMemoryText(v string) string {
403 2046 v = strings.ReplaceAll(v, "\r\n", "\n")
404 2046 v = strings.ReplaceAll(v, "\r", "\n")
405 2046 v = memoryEscapedNewline.ReplaceAllString(v, "\n")
406 2046 return strings.TrimSpace(v)
407 2046 }
408
409 // matchesMemorySearch is the ?q= rule: a case-insensitive substring of the slug
410 // or of the text. An empty query matches everything.
411 2041 func matchesMemorySearch(q, slug, text string) bool {
412 2041 if q == "" {
413 2032 return true
414 2032 }
415 9 q = strings.ToLower(q)
416 9 return strings.Contains(strings.ToLower(slug), q) || strings.Contains(strings.ToLower(text), q)
417 }
418
419 // parseMemorySort reads ?sort=, defaulting (and falling back from an unknown
420 // value) to slug order.
421 21 func parseMemorySort(v string) string {
422 21 if strings.ToLower(strings.TrimSpace(v)) == MemorySortAge {
423 1 return MemorySortAge
424 1 }
425 20 return MemorySortSlug
426 }
427
428 // sortMemories orders the list: by slug, or oldest first for the review queue.
429 // In age order a memory with no revision leads — it is older than anything the
430 // walk could date — and ties fall back to the slug, so the order is total.
431 16 func sortMemories(ms []Memory, order string) {
432 27393 sort.SliceStable(ms, func(i, j int) bool {
433 27393 a, b := ms[i], ms[j]
434 27393 if order == MemorySortAge {
435 3 switch {
436 0 case a.Revision == nil && b.Revision != nil:
437 0 return true
438 0 case a.Revision != nil && b.Revision == nil:
439 0 return false
440 3 case a.Revision != nil && b.Revision != nil && !a.Revision.Date.Equal(b.Revision.Date):
441 3 return a.Revision.Date.Before(b.Revision.Date)
442 }
443 }
444 27390 return a.Slug < b.Slug
445 })
446 }