coverage — ~bigbes/sr-ht-dolt — ede3b0bb — web/markdown.go

Coverage
95.7% 112/117 statements
Δ
—
Blob
5a4d245
Uncovered L101-L106L132-L134L189-L190L259-L260L374-L375
1 package web
2
3 import (
4 "bytes"
5 "html/template"
6 "net/url"
7
8 "github.com/yuin/goldmark"
9 "github.com/yuin/goldmark/ast"
10 "github.com/yuin/goldmark/extension"
11 "github.com/yuin/goldmark/parser"
12 "github.com/yuin/goldmark/renderer"
13 "github.com/yuin/goldmark/renderer/html"
14 "github.com/yuin/goldmark/text"
15 "github.com/yuin/goldmark/util"
16
17 "sourcecraft.dev/bigbes/sr-ht-dolt/beads"
18 )
19
20 // --- memories, rendered as the markdown they are -------------------------------
21 //
22 // A memory's value is markdown and always was: `bd remember` stores what was
23 // typed, and what is typed is the same prose the memory files carry — bold
24 // leaders, code spans, fenced blocks, numbered steps. The view used to print it
25 // as pre-wrapped paragraphs, which is readable but is the source and not the
26 // document: a bullet list stays a line starting with a hyphen, a recipe stays
27 // four spaces of indent, and `**Why:**` keeps its asterisks.
28 //
29 // Three things this rendering does that a stock markdown filter would not:
30 //
31 // 1. `[[slug]]` becomes a link to the memory it names, in whichever database
32 // holds it (see the wikilink parser below). That is the whole point of the
33 // notation, and memories reference each other across trackers constantly:
34 // the mirroring workflow files a memory by its type, so a related memory is
35 // as likely to be in another tracker as in this one.
36 // 2. An issue id in the prose links to the issue, through the same
37 // cross-database index the beads detail pane uses.
38 // 3. Raw HTML is *escaped and shown*, not dropped. goldmark's safe default
39 // omits it, and the memory corpus is full of `<placeholder>` spellings —
40 // `SRHT_<NAME>_VER`, `~/data/home/<repo>` — that CommonMark reads as tags.
41 // Omitting them would silently rewrite `SRHT_<NAME>_VER` to `SRHT__VER`,
42 // which is worse than showing markup: it is showing a different fact.
43 //
44 // Safety is goldmark's default posture, kept: no unsafe HTML, no dangerous URL
45 // schemes in links, everything that came out of the database escaped on its way
46 // to the browser. The one thing marked template.HTML is the finished document
47 // this file produced.
48
49 // memoryLinkIndexKey carries the per-request link index into the parse. A
50 // goldmark.Markdown is stateless and shared; what varies per request is which
51 // databases this caller may browse, and that belongs in the parse context rather
52 // than in a second renderer built per page.
53 var memoryLinkIndexKey = parser.NewContextKey()
54
55 // memoryMarkdown is the shared renderer. goldmark's own parsers and renderers
56 // are safe for concurrent use — all per-conversion state lives in the context —
57 // so this is built once and never rebuilt.
58 var memoryMarkdown = goldmark.New(
59 // GFM for the shapes the memories actually use: tables, strikethrough, task
60 // lists, and linkify — a bare https://dolt.srht.bigb.es/~bigbes/<repo> in the
61 // prose is a URL the reader wants to follow.
62 goldmark.WithExtensions(extension.GFM),
63 goldmark.WithParserOptions(
64 // Ahead of the link parser (200), behind the task-list marker (0): "[[" is
65 // a wikilink before it is a link label. Registered as an inline parser and
66 // not as a text rewrite, so a "[[slug]]" written inside a code span is
67 // left alone by construction — inline parsers do not run in there.
68 parser.WithInlineParsers(util.Prioritized(wikilinkParser{}, 150)),
69 parser.WithASTTransformers(util.Prioritized(issueLinkTransformer{}, 900)),
70 ),
71 goldmark.WithRendererOptions(
72 renderer.WithNodeRenderers(util.Prioritized(memoryNodeRenderer{}, 100)),
73 ),
74 )
75
76 // memoryLinks renders memory bodies for one request. It holds the link index —
77 // prefixes and memory slugs over the databases this caller may browse — and
78 // nothing else; a nil one still renders markdown, with every reference left as
79 // text, which is a whole answer and not a degraded one.
80 type memoryLinks struct {
81 index *beads.PrefixIndex
82 }
83
84 // Body renders one memory's stored text as HTML.
85 //
86 // It is the only function here that produces template.HTML, and what it marks is
87 // the document goldmark built: every leaf that came out of the database is
88 // escaped by the renderer on its way in, including the slug inside a wikilink
89 // and the href built from it.
90 29 func (l *memoryLinks) Body(src string) template.HTML {
91 29 var index *beads.PrefixIndex
92 29 if l != nil {
93 18 index = l.index
94 18 }
95 29 pc := parser.NewContext()
96 29 if index != nil {
97 18 pc.Set(memoryLinkIndexKey, index)
98 18 }
99 29 var buf bytes.Buffer
100 29 if err := memoryMarkdown.Convert([]byte(src), &buf, parser.WithContext(pc)); err != nil {
101 0 // Convert fails only on a write, which this buffer cannot do. The memory is
102 0 // still shown, as the escaped text it was: a rendering that could not run is
103 0 // not a reason to answer with a blank pane.
104 0 return template.HTML(`<p class="mem-raw">` +
105 0 template.HTMLEscapeString(src) + `</p>`)
106 0 }
107 29 return template.HTML(buf.String())
108 }
109
110 // memoryHref is the address of one memory in one database: the memory view
111 // narrowed to a single slug, which is the page a reference wants to land on.
112 12 func memoryHref(owner, name, slug string) string {
113 12 return "/~" + url.PathEscape(owner) + "/" + url.PathEscape(name) +
114 12 "/view/memory?key=" + url.QueryEscape(slug)
115 12 }
116
117 // --- [[slug]] ------------------------------------------------------------------
118
119 // wikilink is a resolved or unresolved memory reference. Href is empty when no
120 // database this caller may browse holds a memory under that slug — which is
121 // deliberately the same node as one nobody ever wrote, so the rendering cannot
122 // disclose the existence of a database the caller may not see.
123 type wikilink struct {
124 ast.BaseInline
125 Href string
126 }
127
128 var kindWikilink = ast.NewNodeKind("MemoryWikilink")
129
130 72 func (n *wikilink) Kind() ast.NodeKind { return kindWikilink }
131
132 0 func (n *wikilink) Dump(source []byte, level int) {
133 0 ast.DumpHelper(n, source, level, map[string]string{"Href": n.Href}, nil)
134 0 }
135
136 // wikilinkParser turns "[[slug]]" into a wikilink node, resolving the slug
137 // against the request's index as it goes.
138 type wikilinkParser struct{}
139
140 1 func (wikilinkParser) Trigger() []byte { return []byte{'['} }
141
142 // Parse reads a wikilink out of the current line, or nothing at all: a "[[" with
143 // no closing "]]" on the same line, and anything whose slug is not slug-shaped,
144 // is left to the ordinary link parser and ends up as the text it was. Memory
145 // slugs are single-token keys — `bd remember --key` takes one — so a reference
146 // never spans a line.
147 33 func (wikilinkParser) Parse(_ ast.Node, block text.Reader, pc parser.Context) ast.Node {
148 33 line, _ := block.PeekLine()
149 33 if len(line) < 5 || line[0] != '[' || line[1] != '[' {
150 4 return nil
151 4 }
152 29 end := bytes.Index(line, []byte("]]"))
153 29 if end < 3 {
154 2 return nil
155 2 }
156 27 slug := string(line[2:end])
157 27 if !isMemorySlug(slug) {
158 1 return nil
159 1 }
160 26 block.Advance(end + 2)
161 26
162 26 node := &wikilink{}
163 26 if index, ok := pc.Get(memoryLinkIndexKey).(*beads.PrefixIndex); ok {
164 20 if d, found := index.LookupMemory(slug); found {
165 12 node.Href = memoryHref(d.OwnerName, d.Name, slug)
166 12 }
167 }
168 // The slug is carried as a string node rather than as a source segment: a
169 // line the reader hands back can be padded — a list item's continuation
170 // indent is synthesised, not sliced — and an offset into it is then not an
171 // offset into the source. The renderer escapes a string node exactly as it
172 // escapes every other leaf.
173 26 node.AppendChild(node, ast.NewString([]byte(slug)))
174 26 return node
175 }
176
177 // memorySlugMax bounds what this will treat as a slug. `bd remember --key` takes
178 // a short name; a "[[" followed by half a paragraph and a "]]" is prose that
179 // happens to contain brackets.
180 const memorySlugMax = 128
181
182 // isMemorySlug is the shape a memory key has: the characters `bd remember --key`
183 // and the memory files' `name:` field use, and no others. It is deliberately
184 // narrower than "anything without brackets" — a bracketed aside is not a
185 // reference, and the difference has to be decidable without asking the index,
186 // since a slug nobody holds must render the same way whether or not it is one.
187 27 func isMemorySlug(s string) bool {
188 27 if s == "" || len(s) > memorySlugMax {
189 0 return false
190 0 }
191 343 for _, r := range s {
192 343 switch {
193 case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
194 case r == '-', r == '_', r == '.', r == '/':
195 1 default:
196 1 return false
197 }
198 }
199 26 return true
200 }
201
202 // --- issue ids in memory prose -------------------------------------------------
203
204 // issueLinkTransformer links the issue ids in a memory's prose to the databases
205 // that own them, reusing the index this render already built for the wikilinks.
206 //
207 // It runs after inline parsing, over the text nodes only, and never descends
208 // into a code span, a link, an autolink or a wikilink: an id inside `code` is
209 // being shown rather than cited, and an id inside a link label would nest an
210 // anchor in an anchor.
211 type issueLinkTransformer struct{}
212
213 29 func (issueLinkTransformer) Transform(doc *ast.Document, reader text.Reader, pc parser.Context) {
214 29 index, ok := pc.Get(memoryLinkIndexKey).(*beads.PrefixIndex)
215 29 if !ok || index == nil {
216 11 return
217 11 }
218 18 source := reader.Source()
219 18
220 18 // Collected first and rewritten after: replacing a node during the walk that
221 18 // found it is how a walk starts stepping over its own edits.
222 18 var texts []*ast.Text
223 478 _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
224 478 if !entering {
225 239 return ast.WalkContinue, nil
226 239 }
227 239 switch n.Kind() {
228 case ast.KindLink, ast.KindImage, ast.KindAutoLink, ast.KindCodeSpan,
229 ast.KindRawHTML, ast.KindHTMLBlock, ast.KindCodeBlock,
230 34 ast.KindFencedCodeBlock, kindWikilink:
231 34 return ast.WalkSkipChildren, nil
232 125 case ast.KindText:
233 125 t := n.(*ast.Text)
234 125 // A raw text node is a code span's content, and a padded one carries a
235 125 // block indent its segment offsets do not describe. Neither can be cut
236 125 // on byte offsets taken from the source.
237 125 if !t.IsRaw() && t.Segment.Padding == 0 {
238 125 texts = append(texts, t)
239 125 }
240 }
241 205 return ast.WalkContinue, nil
242 })
243
244 125 for _, t := range texts {
245 125 linkIssueIDs(t, source, index)
246 125 }
247 }
248
249 // linkIssueIDs replaces one text node with the sequence of text and link nodes
250 // its ids imply. A node with no id in it is left exactly as it was.
251 125 func linkIssueIDs(t *ast.Text, source []byte, index *beads.PrefixIndex) {
252 125 seg := t.Segment
253 125 refs := index.Scan(string(source[seg.Start:seg.Stop]))
254 125 if len(refs) == 0 {
255 117 return
256 117 }
257 8 parent := t.Parent()
258 8 if parent == nil {
259 0 return
260 0 }
261
262 8 var nodes []ast.Node
263 8 last := seg.Start
264 8 for _, ref := range refs {
265 8 start, stop := seg.Start+ref.Start, seg.Start+ref.End
266 8 if start > last {
267 8 nodes = append(nodes, ast.NewTextSegment(text.NewSegment(last, start)))
268 8 }
269 8 link := ast.NewLink()
270 8 link.Destination = []byte(beadIssueHref(
271 8 ref.Database.OwnerName, ref.Database.Name, ref.ID))
272 8 link.AppendChild(link, ast.NewTextSegment(text.NewSegment(start, stop)))
273 8 nodes = append(nodes, link)
274 8 last = stop
275 }
276
277 // The tail carries the original node's line-break flags. When the id ran to
278 // the end of the node the tail is empty and is kept anyway: dropping it drops
279 // the newline, and the next line's first word would be glued to the id.
280 8 tail := ast.NewTextSegment(text.NewSegment(last, seg.Stop))
281 8 tail.SetSoftLineBreak(t.SoftLineBreak())
282 8 tail.SetHardLineBreak(t.HardLineBreak())
283 8 nodes = append(nodes, tail)
284 8
285 24 for _, n := range nodes {
286 24 parent.InsertBefore(parent, t, n)
287 24 }
288 8 parent.RemoveChild(parent, t)
289 }
290
291 // --- the three nodes this rendering does not leave to goldmark -----------------
292
293 // memoryNodeRenderer registers the wikilink renderer and replaces goldmark's
294 // handling of raw HTML.
295 type memoryNodeRenderer struct{}
296
297 1 func (memoryNodeRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
298 1 reg.Register(kindWikilink, renderWikilink)
299 1 reg.Register(ast.KindRawHTML, renderRawHTMLAsText)
300 1 reg.Register(ast.KindHTMLBlock, renderHTMLBlockAsText)
301 1 reg.Register(ast.KindImage, renderImageAsLink)
302 1 }
303
304 // renderWikilink writes the anchor, or the muted marker for a slug no database
305 // this caller may browse holds. The marker says only that the reference does not
306 // resolve *here*; it cannot say more without disclosing what it must not.
307 52 func renderWikilink(w util.BufWriter, _ []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
308 52 n := node.(*wikilink)
309 52 switch {
310 12 case entering && n.Href != "":
311 12 _, _ = w.WriteString(`<a class="mem-link" href="`)
312 12 _, _ = w.Write(util.EscapeHTML(util.URLEscape([]byte(n.Href), true)))
313 12 _, _ = w.WriteString(`">`)
314 14 case entering:
315 14 _, _ = w.WriteString(`<span class="mem-link-out" ` +
316 14 `title="No memory with this slug in a tracker you can browse.">`)
317 12 case n.Href != "":
318 12 _, _ = w.WriteString(`</a>`)
319 14 default:
320 14 _, _ = w.WriteString(`</span>`)
321 }
322 52 return ast.WalkContinue, nil
323 }
324
325 // renderImageAsLink renders an image reference as a link to it rather than as an
326 // <img>.
327 //
328 // An <img> pointing at another host is a request that host makes on behalf of
329 // whoever opened the page, and a memory is prose one account wrote and another
330 // may read: an image in it would report the reader's address to a server the
331 // reader never chose to contact. The reference is kept and stays followable; it
332 // simply is not fetched by opening the page.
333 2 func renderImageAsLink(w util.BufWriter, _ []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
334 2 n := node.(*ast.Image)
335 2 if !entering {
336 1 _, _ = w.WriteString(`</a>`)
337 1 return ast.WalkContinue, nil
338 1 }
339 1 _, _ = w.WriteString(`<a class="mem-img" href="`)
340 1 if dest := util.URLEscape(n.Destination, true); !html.IsDangerousURL(dest) {
341 1 _, _ = w.Write(util.EscapeHTML(dest))
342 1 }
343 // The alt text is the anchor's text, which is what the children already
344 // render as — an image with no alt text is left as a bare link.
345 1 _, _ = w.WriteString(`">`)
346 1 return ast.WalkContinue, nil
347 }
348
349 // renderRawHTMLAsText writes inline raw HTML as the escaped text it reads as.
350 //
351 // goldmark's safe default omits it, which is right for a comment feed and wrong
352 // here: `SRHT_<NAME>_VER` is a placeholder somebody typed, CommonMark sees
353 // `<NAME>` as a tag, and omitting it turns the memory into a different sentence.
354 // Escaping shows what was written and is exactly as safe as omitting it.
355 26 func renderRawHTMLAsText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
356 26 if !entering {
357 13 return ast.WalkSkipChildren, nil
358 13 }
359 13 n := node.(*ast.RawHTML)
360 13 for i := 0; i < n.Segments.Len(); i++ {
361 13 seg := n.Segments.At(i)
362 13 _, _ = w.Write(util.EscapeHTML(seg.Value(source)))
363 13 }
364 13 return ast.WalkSkipChildren, nil
365 }
366
367 // renderHTMLBlockAsText is the same treatment for a block that opened with
368 // something tag-shaped: shown, escaped, in a paragraph of its own rather than
369 // dropped.
370 2 func renderHTMLBlockAsText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
371 2 n := node.(*ast.HTMLBlock)
372 2 if !entering {
373 1 if n.HasClosure() {
374 0 _, _ = w.Write(util.EscapeHTML(n.ClosureLine.Value(source)))
375 0 }
376 1 _, _ = w.WriteString("</p>\n")
377 1 return ast.WalkContinue, nil
378 }
379 1 _, _ = w.WriteString(`<p class="mem-raw">`)
380 1 for i := 0; i < n.Lines().Len(); i++ {
381 1 line := n.Lines().At(i)
382 1 _, _ = w.Write(util.EscapeHTML(line.Value(source)))
383 1 }
384 1 return ast.WalkContinue, nil
385 }