mirror of
https://github.com/PerpetualSoftware/pad.git
synced 2026-09-10 06:55:40 +00:00
fix(links,web): align the JS wiki-link grammar with Go's . (BUG-2834)
The Go and JS wiki-link patterns were byte-identical source text and did not mean the same thing. Both spelled the escape alternative `\\.`, but Go's RE2 `.` excludes only LF while ECMAScript's also excludes CR, U+2028 and U+2029. A body with a backslash immediately before one of those three was INDEXED by the server and NOT RENDERED by the client: the backlink panel claimed a link the document refused to draw. CRLF line endings make the CR case the plausible one. Measured on both sides before deciding anything, over nine code points. Exactly three diverge; VT, FF and U+0085 agree, which bounds the divergence at precisely ECMAScript's LineTerminator set minus LF rather than leaving it open at "some whitespace controls". Aligns JS UP to Go (`\\[^\n]`) rather than narrowing Go, for three independent reasons: the grammar's other alternative already admits RAW CR/LS/PS in both languages, so narrowing Go would make `[[A<CR>B]]` legal and `[[A\<CR>B]]` illegal; narrowing Go would stop ExtractWikiLinks returning rows it currently returns, and the next reconcile would DELETE them, which is BUG-2805's damage shape; and markdown.ts's own splitWikiBody already treats `\<CR>` as an escape pair, so only its regex disagreed. LF stays excluded on both sides — scanBracketBody depends on that and is untouched. Also collapses the two duplicate regex literals in markdown.ts into one exported WIKI_LINK_PATTERN_SOURCE. Exported as source text, not a RegExp object, because a `/g` regex carries lastIndex and sharing one between a replace() and a test's exec() would couple them through it. The harness is the part meant to outlive the fix. testdata/wiki_grammar_corpus.json is read by BOTH languages, and carries expectations derived from the grammar spec rather than from either implementation — comparing the two implementations to each other would have reproduced the exact blind spot that hid this, since looking identical is what they already did. Pure ASCII with every control character as a \uXXXX escape: an early probe typed U+2028/U+2029 into a shell heredoc, silently lost them, and would have "confirmed" the bug on two cases that were actually spaces. JS assertions are split across the node and jsdom projects because renderMarkdown finishes through DOMPurify and returns '' without a DOM, where it would fail for a reason unrelated to the grammar. Same split, same reason, as the existing markdown.shareAttachments pair; the jsdom file carries a leg proving the DOM path is live, since '' satisfies every not.toContain assertion. Negative-controlled both ways: reverting the JS pattern fails exactly 9 assertions (3 corpus + 3 per call site) and nothing else; narrowing the GO pattern to JS semantics fails the same 3 cases from the other side, so both halves are live instruments rather than tests that cannot fail.
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
package links
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The Go half of the cross-language wiki-link grammar harness (BUG-2834).
|
||||
//
|
||||
// The JS half is web/src/lib/utils/markdown.grammarParity.test.ts. Both read
|
||||
// the SAME corpus and assert against the expectations recorded IN it, so
|
||||
// neither language's current behaviour is what the other is measured against.
|
||||
// That indirection is the whole point: the two patterns are byte-identical
|
||||
// source text, so a reviewer comparing them concludes they agree, and BUG-2834
|
||||
// lived entirely in the host languages' disagreement about what `.` means.
|
||||
// Comparing implementation to implementation would have reproduced the same
|
||||
// blind spot in test form.
|
||||
//
|
||||
// If you add a case, add it to the corpus — not to one language's test.
|
||||
|
||||
const grammarCorpusRelPath = "../../testdata/wiki_grammar_corpus.json"
|
||||
|
||||
type grammarExpect struct {
|
||||
Match bool `json:"match"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
type grammarCase struct {
|
||||
Name string `json:"name"`
|
||||
Why string `json:"why"`
|
||||
Content string `json:"content"`
|
||||
Expect grammarExpect `json:"expect"`
|
||||
}
|
||||
|
||||
type grammarCorpus struct {
|
||||
Cases []grammarCase `json:"cases"`
|
||||
}
|
||||
|
||||
func loadGrammarCorpus(t *testing.T) grammarCorpus {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Clean(grammarCorpusRelPath))
|
||||
if err != nil {
|
||||
t.Fatalf("read shared grammar corpus: %v", err)
|
||||
}
|
||||
var corpus grammarCorpus
|
||||
if err := json.Unmarshal(raw, &corpus); err != nil {
|
||||
t.Fatalf("parse shared grammar corpus: %v", err)
|
||||
}
|
||||
// An empty or unparsed corpus would make every assertion below vacuous and
|
||||
// the suite would report PASS having measured nothing. The count is asserted
|
||||
// rather than assumed for the same reason the corpus carries its own
|
||||
// expectations: a harness that cannot fail is not an instrument.
|
||||
if len(corpus.Cases) < 20 {
|
||||
t.Fatalf("shared grammar corpus looks truncated: %d cases (expected >= 20)", len(corpus.Cases))
|
||||
}
|
||||
return corpus
|
||||
}
|
||||
|
||||
// TestWikiLinkGrammarMatchesSharedCorpus drives the REAL wikiLinkPattern — not
|
||||
// a copy of its source text — over the shared corpus.
|
||||
func TestWikiLinkGrammarMatchesSharedCorpus(t *testing.T) {
|
||||
t.Parallel()
|
||||
corpus := loadGrammarCorpus(t)
|
||||
|
||||
for _, tc := range corpus.Cases {
|
||||
t.Run(tc.Name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
m := wikiLinkPattern.FindStringSubmatch(tc.Content)
|
||||
|
||||
if !tc.Expect.Match {
|
||||
if m != nil {
|
||||
t.Fatalf("expected NO match but Go matched, body=%s\ncontent=%s\nwhy this case exists: %s",
|
||||
quoteCodePoints(m[1]), quoteCodePoints(tc.Content), tc.Why)
|
||||
}
|
||||
return
|
||||
}
|
||||
if m == nil {
|
||||
t.Fatalf("expected a match, Go found none\ncontent=%s\nwhy this case exists: %s",
|
||||
quoteCodePoints(tc.Content), tc.Why)
|
||||
}
|
||||
if m[1] != tc.Expect.Body {
|
||||
t.Fatalf("captured body mismatch\n got: %s\nwant: %s\ncontent=%s\nwhy this case exists: %s",
|
||||
quoteCodePoints(m[1]), quoteCodePoints(tc.Expect.Body),
|
||||
quoteCodePoints(tc.Content), tc.Why)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// quoteCodePoints renders a string with every non-printable rune as \uXXXX.
|
||||
//
|
||||
// %q alone is not enough here: this corpus is ENTIRELY about characters that
|
||||
// are invisible or that terminate a line in a terminal, and a failure message
|
||||
// that prints a raw U+2028 is a failure message that lies about what it
|
||||
// compared. The bug being tested is itself a case of an invisible character
|
||||
// being mistaken for something else.
|
||||
func quoteCodePoints(s string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(s) + 8)
|
||||
b.WriteByte('"')
|
||||
for _, r := range s {
|
||||
if r < 0x20 || r == 0x7f || r == 0x85 || r == 0x2028 || r == 0x2029 {
|
||||
fmt.Fprintf(&b, `\u%04X`, r)
|
||||
continue
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
b.WriteByte('"')
|
||||
return b.String()
|
||||
}
|
||||
Vendored
+250
@@ -0,0 +1,250 @@
|
||||
{
|
||||
"_comment": [
|
||||
"SHARED cross-language corpus for the wiki-link bracket grammar (BUG-2834).",
|
||||
"",
|
||||
"GENERATED FILE -- pure ASCII by construction. Every control character and",
|
||||
"non-ASCII code point below is a \\uXXXX escape, never a raw byte, so this",
|
||||
"file survives editors, diffs, terminals and copy-paste intact. An early",
|
||||
"probe for this very bug typed U+2028/U+2029 into a shell heredoc, silently",
|
||||
"lost them, and would have 'confirmed' the divergence on two cases that were",
|
||||
"actually spaces. Do not hand-edit a raw control character into this file.",
|
||||
"",
|
||||
"Consumed by BOTH implementations of the grammar, which are written as the",
|
||||
"same pattern text in two languages that do not agree on what it means:",
|
||||
"",
|
||||
" Go: internal/links/extract.go wikiLinkPattern",
|
||||
" asserted by internal/links/grammar_parity_test.go",
|
||||
" JS: web/src/lib/utils/markdown.ts renderMarkdown + wikiLinksToMarkdown",
|
||||
" asserted by web/src/lib/utils/markdown.grammarParity.test.ts",
|
||||
"",
|
||||
"THE POINT: neither implementation defines truth. The `expect` fields are",
|
||||
"derived from the grammar SPEC and both languages are asserted against them,",
|
||||
"so a divergence introduced on EITHER side fails on that side. This file",
|
||||
"exists because the two patterns are BYTE-IDENTICAL SOURCE TEXT -- a reviewer",
|
||||
"comparing them side by side concludes they agree, and BUG-2834 lived in the",
|
||||
"gap between that reading and the host languages' definition of `.`.",
|
||||
"",
|
||||
"THE SPEC. The body production is `(?:\\\\.|[^\\]\\\\])+` where `.` means ANY",
|
||||
"CHARACTER EXCEPT LINE FEED (U+000A) -- Go's RE2 definition. JavaScript's `.`",
|
||||
"additionally excludes CR, U+2028 and U+2029, which IS the divergence; the JS",
|
||||
"patterns therefore spell it `[^\\n]` explicitly rather than `.`.",
|
||||
"",
|
||||
"`content` is the FULL input string. `expect.match` is whether the grammar",
|
||||
"matches anywhere in it; `expect.body` is capture group 1 of the FIRST match",
|
||||
"and is absent when match is false."
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "plain body",
|
||||
"why": "baseline: the ordinary form must match in both languages",
|
||||
"content": "[[Title]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "Title"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "pipe display segment",
|
||||
"why": "the `|` display separator is an ordinary body byte to the grammar",
|
||||
"content": "[[Title|Display]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "Title|Display"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "escaped close bracket",
|
||||
"why": "the `\\\\.` alternative's primary job: `\\]` must not terminate the body",
|
||||
"content": "[[A\\]B]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\]B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "escaped pipe",
|
||||
"why": "escapeWikiBody emits `\\|` for a literal pipe in a title",
|
||||
"content": "[[A\\|B]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\|B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "escaped backslash",
|
||||
"why": "escapeWikiBody doubles backslashes first; the pair must be one unit",
|
||||
"content": "[[A\\\\B]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\\B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "empty body",
|
||||
"why": "the production is `+`, so an empty body is not a link",
|
||||
"content": "[[]]",
|
||||
"expect": {
|
||||
"match": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "escaped close consumes the first of a bare pair",
|
||||
"why": "`[[A\\]]` has nothing left to close with -- the naive strings.Index scan got this wrong (BUG-2805)",
|
||||
"content": "[[A\\]]",
|
||||
"expect": {
|
||||
"match": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "escaped close then real close",
|
||||
"why": "`[[A\\]]]` closes on the LAST two brackets, body keeps the escape",
|
||||
"content": "[[A\\]]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "trailing backslash with nothing to consume",
|
||||
"why": "`\\\\.` needs a byte after the backslash; there is none",
|
||||
"content": "[[A\\",
|
||||
"expect": {
|
||||
"match": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "surplus closing brackets",
|
||||
"why": "the match ends at the first legal close; the extra `]]` is outside it",
|
||||
"content": "[[A]]]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "inner open brackets are ordinary body bytes",
|
||||
"why": "`[` is not excluded by `[^\\]\\\\]`, so the body swallows it -- checks greediness agrees across RE2 and backtracking",
|
||||
"content": "[[A[[B]]]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A[[B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "raw LF inside body",
|
||||
"why": "a RAW line terminator is matched by the `[^\\]\\\\]` alternative in BOTH languages -- this is why `.` excluding them was never a deliberate rule about titles",
|
||||
"content": "[[A\nB]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\nB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "raw CR inside body",
|
||||
"why": "same as raw LF: the second alternative admits it, so only the ESCAPED form ever diverged",
|
||||
"content": "[[A\rB]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\rB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before LF",
|
||||
"why": "BUG-2834 control: Go's `.` excludes LF, so this is NOT an escape pair and the bracket fails -- both languages already agreed here, and the fix must not change it (scanBracketBody in links.go depends on it)",
|
||||
"content": "[[A\\\nB]]",
|
||||
"expect": {
|
||||
"match": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before CR",
|
||||
"why": "BUG-2834 divergence 1 of 3: Go matched, JS did not. CRLF line endings make this the plausible one -- a body ending in a backslash immediately before a CRLF break",
|
||||
"content": "[[A\\\rB]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\rB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before U+2028 LINE SEPARATOR",
|
||||
"why": "BUG-2834 divergence 2 of 3: excluded by ECMAScript's LineTerminator, not by RE2's",
|
||||
"content": "[[A\\\u2028B]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\u2028B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before U+2029 PARAGRAPH SEPARATOR",
|
||||
"why": "BUG-2834 divergence 3 of 3",
|
||||
"content": "[[A\\\u2029B]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\u2029B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before U+000B VERTICAL TAB",
|
||||
"why": "BOUNDS the divergence: a vertical whitespace control that is NOT an ECMAScript LineTerminator. Agreement here is what makes the population exactly three rather than 'the whitespace controls'",
|
||||
"content": "[[A\\\u000bB]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\u000bB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before U+000C FORM FEED",
|
||||
"why": "BOUNDS the divergence: same role as VT",
|
||||
"content": "[[A\\\fB]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\fB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before U+0085 NEXT LINE",
|
||||
"why": "BOUNDS the divergence: a Unicode line break that ECMAScript does NOT count as a LineTerminator. The 'it is all Unicode line breaks' hypothesis dies here",
|
||||
"content": "[[A\\\u0085B]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\u0085B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before space",
|
||||
"why": "ordinary escape of a non-special byte: legal in both, and the negative control for the divergence cases above",
|
||||
"content": "[[A\\ B]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\ B"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "backslash before tab",
|
||||
"why": "ordinary escape, horizontal whitespace",
|
||||
"content": "[[A\\\tB]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "A\\\tB"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "collection-qualified body",
|
||||
"why": "the `<slug>/<title>` form is ordinary body text to the GRAMMAR -- the ambiguity it creates is BUG-2830's, decided above the grammar, not inside it",
|
||||
"content": "[[tasks/Setup]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "tasks/Setup"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "cross-workspace form",
|
||||
"why": "`::` is likewise ordinary body text to the grammar",
|
||||
"content": "[[team::TASK-5]]",
|
||||
"expect": {
|
||||
"match": true,
|
||||
"body": "team::TASK-5"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { renderMarkdown } from './markdown';
|
||||
import type { Item } from '$lib/types';
|
||||
|
||||
// The `renderMarkdown` half of the BUG-2834 binding assertions.
|
||||
//
|
||||
// Lives in the jsdom project (`*.svelte.test.ts`) rather than beside the rest
|
||||
// of the parity suite because renderMarkdown finishes through DOMPurify and
|
||||
// returns '' with no DOM present — in the node project it fails for a reason
|
||||
// that has nothing to do with the grammar, which is worse than not running.
|
||||
// markdown.shareAttachments{,.svelte}.test.ts splits for the same reason.
|
||||
//
|
||||
// The corpus assertions and the wikiLinksToMarkdown binding live in
|
||||
// markdown.grammarParity.test.ts. This file is only the second call site.
|
||||
|
||||
function show(s: string): string {
|
||||
return JSON.stringify(s).replace(/[\u0000-\u001f\u007f-\uffff]/g, (c) => {
|
||||
return '\\u' + c.charCodeAt(0).toString(16).padStart(4, '0').toUpperCase();
|
||||
});
|
||||
}
|
||||
|
||||
describe('renderMarkdown consumes the parity-fixed grammar (BUG-2834 binding)', () => {
|
||||
const noItems: Item[] = [];
|
||||
|
||||
// The three code points where Go's `.` and JavaScript's `.` disagreed.
|
||||
// Before the fix the renderer left these as literal `[[...]]` text while
|
||||
// the server had already indexed them as links — so the backlink panel and
|
||||
// the rendered document disagreed about whether a link existed.
|
||||
const divergent: Array<[string, number]> = [
|
||||
['CR U+000D', 0x000d],
|
||||
['LS U+2028', 0x2028],
|
||||
['PS U+2029', 0x2029]
|
||||
];
|
||||
|
||||
for (const [label, cp] of divergent) {
|
||||
it(`consumes a bracket whose body has a backslash before ${label}`, () => {
|
||||
const body = 'A\\' + String.fromCodePoint(cp) + 'B';
|
||||
const html = renderMarkdown(`see [[${body}]] here`, noItems, 'ws');
|
||||
|
||||
// With no matching item the bracket resolves to the broken-link
|
||||
// span. That is the honest outcome, and — the point of the test —
|
||||
// it is NOT the literal `[[` passthrough the unfixed renderer
|
||||
// produced. Asserting on the span rather than on a resolved link
|
||||
// keeps this leg independent of item-fixture shape.
|
||||
expect(html, `body=${show(body)}`).toContain('doc-link broken');
|
||||
expect(html, `body=${show(body)}`).not.toContain('[[');
|
||||
});
|
||||
}
|
||||
|
||||
// Negative control. LF is the code point both languages have always
|
||||
// rejected and the fix must not have changed it; `scanBracketBody` in
|
||||
// internal/links/links.go depends on that agreement. Without this leg the
|
||||
// three assertions above would also pass for `[\s\S]`, the other candidate
|
||||
// fix, which would have over-matched.
|
||||
it('still refuses a backslash before LF, like the Go parser', () => {
|
||||
const html = renderMarkdown('see [[A\\\nB]] here', noItems, 'ws');
|
||||
|
||||
expect(html).toContain('[[');
|
||||
expect(html).not.toContain('doc-link broken');
|
||||
});
|
||||
|
||||
// Guards the DOM precondition itself. If DOMPurify ever silently returns
|
||||
// '' here again, every assertion above would still "pass" its not.toContain
|
||||
// leg while measuring nothing — the empty string contains neither '[[' nor
|
||||
// 'doc-link broken'. This is the leg that fails loudly instead.
|
||||
it('renders a plain wiki-link, proving the DOM path is live', () => {
|
||||
const html = renderMarkdown('see [[Anything]] here', noItems, 'ws');
|
||||
expect(html).not.toBe('');
|
||||
expect(html).toContain('doc-link broken');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { WIKI_LINK_PATTERN_SOURCE, wikiLinksToMarkdown } from './markdown';
|
||||
import type { Item } from '$lib/types';
|
||||
|
||||
// The JavaScript half of the cross-language wiki-link grammar harness
|
||||
// (BUG-2834). The Go half is internal/links/grammar_parity_test.go.
|
||||
//
|
||||
// Both halves read the SAME corpus and assert against the expectations
|
||||
// recorded IN it, rather than against each other. That indirection is the
|
||||
// entire point. The two patterns used to be byte-identical source text, so a
|
||||
// reviewer comparing them side by side concluded they agreed — and they did
|
||||
// not, because the divergence was in the host languages' definition of `.`,
|
||||
// which is invisible to inspection. A test that compared one implementation to
|
||||
// the other would have inherited exactly that blind spot; a test that pins
|
||||
// each implementation to an independently-stated spec cannot.
|
||||
//
|
||||
// If you add a case, add it to the corpus — not to one language's test.
|
||||
|
||||
const CORPUS_PATH = fileURLToPath(
|
||||
new URL('../../../../testdata/wiki_grammar_corpus.json', import.meta.url)
|
||||
);
|
||||
|
||||
type GrammarCase = {
|
||||
name: string;
|
||||
why: string;
|
||||
content: string;
|
||||
expect: { match: boolean; body?: string };
|
||||
};
|
||||
|
||||
const corpus: { cases: GrammarCase[] } = JSON.parse(readFileSync(CORPUS_PATH, 'utf8'));
|
||||
|
||||
// A truncated or unparsed corpus would make every assertion below vacuous
|
||||
// while the suite still reported green. Asserted, not assumed — a harness that
|
||||
// cannot fail is not an instrument.
|
||||
if (corpus.cases.length < 20) {
|
||||
throw new Error(`shared grammar corpus looks truncated: ${corpus.cases.length} cases`);
|
||||
}
|
||||
|
||||
// Render invisible code points as \uXXXX. This corpus is ENTIRELY about
|
||||
// characters that are invisible or that break a line in a terminal, so a
|
||||
// failure message printing them raw would misreport what it compared — the
|
||||
// same class of mistake as the bug under test.
|
||||
function show(s: string): string {
|
||||
return JSON.stringify(s).replace(/[\u0000-\u001f\u007f-\uffff]/g, (c) => {
|
||||
return '\\u' + c.charCodeAt(0).toString(16).padStart(4, '0').toUpperCase();
|
||||
});
|
||||
}
|
||||
|
||||
describe('wiki-link grammar parity with the Go server (BUG-2834)', () => {
|
||||
// A fresh RegExp per case: the exported value is SOURCE TEXT precisely so
|
||||
// that no `lastIndex` state is shared between this suite and the renderer.
|
||||
function firstMatch(content: string): RegExpExecArray | null {
|
||||
return new RegExp(WIKI_LINK_PATTERN_SOURCE).exec(content);
|
||||
}
|
||||
|
||||
for (const tc of corpus.cases) {
|
||||
it(tc.name, () => {
|
||||
const m = firstMatch(tc.content);
|
||||
|
||||
if (!tc.expect.match) {
|
||||
expect(
|
||||
m,
|
||||
`expected NO match for ${show(tc.content)}\nwhy this case exists: ${tc.why}`
|
||||
).toBeNull();
|
||||
return;
|
||||
}
|
||||
|
||||
expect(
|
||||
m,
|
||||
`expected a match for ${show(tc.content)}\nwhy this case exists: ${tc.why}`
|
||||
).not.toBeNull();
|
||||
expect(
|
||||
show(m![1]),
|
||||
`captured body mismatch for ${show(tc.content)}\nwhy this case exists: ${tc.why}`
|
||||
).toBe(show(tc.expect.body!));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// The suite above vouches for the exported PATTERN. On its own that is a
|
||||
// direct-call test: it proves the source text is right and says nothing about
|
||||
// whether anything USES it (CONVE-19 — wiring is a claim). The original bug was
|
||||
// in rendering behaviour, not in a constant, so both call sites get asserted
|
||||
// through their public entry points.
|
||||
//
|
||||
// This file covers `wikiLinksToMarkdown` (pure string in, string out).
|
||||
// `renderMarkdown` is covered in markdown.grammarParity.svelte.test.ts, because
|
||||
// it finishes through DOMPurify and returns '' without a DOM — in the node
|
||||
// project it would appear to "fail" for a reason that has nothing to do with
|
||||
// the grammar. Same node/jsdom split, and same reason, as
|
||||
// markdown.shareAttachments{,.svelte}.test.ts.
|
||||
describe('wikiLinksToMarkdown consumes the parity-fixed grammar (BUG-2834 binding)', () => {
|
||||
// An UNRESOLVED body is useless as a discriminator here: on a miss
|
||||
// wikiLinksToMarkdown deliberately returns the match verbatim, so a bracket
|
||||
// the grammar rejected and a bracket it accepted-but-could-not-resolve
|
||||
// produce byte-identical output. The fixture therefore has to RESOLVE, so
|
||||
// that "was the bracket matched at all" becomes observable as a link.
|
||||
//
|
||||
// resolveWikiBody step 3 matches on unescapeWikiBody(key), and that only
|
||||
// unescapes `\\`, `\]` and `\|` — a backslash before CR/LS/PS is none of
|
||||
// those, so it survives into the key and the item title must carry it too.
|
||||
function itemTitled(title: string): Item {
|
||||
return {
|
||||
id: 'id-1',
|
||||
title,
|
||||
collection_slug: 'tasks',
|
||||
slug: 'fixture'
|
||||
} as unknown as Item;
|
||||
}
|
||||
|
||||
const divergent: Array<[string, number]> = [
|
||||
['CR U+000D', 0x000d],
|
||||
['LS U+2028', 0x2028],
|
||||
['PS U+2029', 0x2029]
|
||||
];
|
||||
|
||||
for (const [label, cp] of divergent) {
|
||||
it(`links a body with a backslash before ${label}`, () => {
|
||||
const title = 'A\\' + String.fromCodePoint(cp) + 'B';
|
||||
const out = wikiLinksToMarkdown(`see [[${title}]] here`, [itemTitled(title)], 'ws');
|
||||
|
||||
expect(out, `title=${show(title)}`).toContain('](/ws/tasks/');
|
||||
expect(out, `title=${show(title)}`).not.toContain('[[');
|
||||
});
|
||||
}
|
||||
|
||||
// Negative control for the three legs above. LF is the code point BOTH
|
||||
// languages have always rejected, and the fix must not have changed it.
|
||||
// Without this leg, "the renderer now consumes more brackets" would be
|
||||
// satisfied by a pattern that consumes everything — including by
|
||||
// `[\s\S]`, which was the other candidate fix and is WRONG here.
|
||||
it('still refuses a backslash before LF, like the Go parser', () => {
|
||||
const title = 'A\\\nB';
|
||||
const out = wikiLinksToMarkdown(`see [[${title}]] here`, [itemTitled(title)], 'ws');
|
||||
|
||||
expect(out).toContain('[[');
|
||||
expect(out).not.toContain('](/ws/tasks/');
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,49 @@ import {
|
||||
resolveAttachmentLink
|
||||
} from '$lib/markdown/attachments';
|
||||
|
||||
/**
|
||||
* The wiki-link bracket grammar — the ONE copy on the JS side.
|
||||
*
|
||||
* Body production: a backslash-escaped character, or any character that is
|
||||
* neither `]` nor `\`. Kept deliberately permissive so anything the editor can
|
||||
* save is also indexable; see `wikiLinkPattern` in internal/links/extract.go,
|
||||
* which is the server-side half of the same grammar.
|
||||
*
|
||||
* ## Why `\\[^\n]` and not `\\.` (BUG-2834)
|
||||
*
|
||||
* The Go and JS patterns used to be BYTE-IDENTICAL source text — both spelled
|
||||
* the escape alternative `\\.` — and they still did not mean the same thing,
|
||||
* because the two languages do not agree on `.`:
|
||||
*
|
||||
* - Go (RE2): `.` matches everything except LF (U+000A).
|
||||
* - JavaScript: `.` additionally excludes CR, U+2028 and U+2029 — the full
|
||||
* ECMAScript LineTerminator set.
|
||||
*
|
||||
* So a body containing a backslash immediately before CR / U+2028 / U+2029 was
|
||||
* INDEXED by the server and NOT RENDERED here: the backlink panel claimed a
|
||||
* link the document refused to draw. CRLF line endings make the CR case the
|
||||
* plausible one — a body ending in a backslash right before a CRLF break.
|
||||
*
|
||||
* `[^\n]` states Go's definition explicitly, so the two sides now agree by
|
||||
* construction rather than by looking alike. Measured, not reasoned: the three
|
||||
* divergent code points plus VT / FF / U+0085 (which always agreed, and which
|
||||
* bound the divergence to exactly the LineTerminator set) are pinned in
|
||||
* testdata/wiki_grammar_corpus.json and asserted from both languages.
|
||||
*
|
||||
* LF stays excluded on BOTH sides — that was never the divergence, and
|
||||
* `scanBracketBody` in internal/links/links.go depends on it.
|
||||
*
|
||||
* Exported as SOURCE TEXT rather than as a shared RegExp object: a `/g` regex
|
||||
* carries `lastIndex` state, and handing the same object to both a `replace()`
|
||||
* and a test's `exec()` would couple them through it.
|
||||
*/
|
||||
export const WIKI_LINK_PATTERN_SOURCE = String.raw`\[\[((?:\\[^\n]|[^\]\\])+)\]\]`;
|
||||
|
||||
// The shared instance for this module's two rewrite sites. Safe to reuse
|
||||
// despite the `/g` flag because `String.prototype.replace` resets `lastIndex`
|
||||
// both before and after a global match — unlike `exec`/`test`, which do not.
|
||||
const WIKI_LINK_PATTERN = new RegExp(WIKI_LINK_PATTERN_SOURCE, 'g');
|
||||
|
||||
// Mirror of marked's internal cleanUrl() — percent-encodes the href so the
|
||||
// rendered HTML stays well-formed even when input contains spaces, quotes, or
|
||||
// other URL-unsafe characters. The %25 → % round-trip avoids double-encoding
|
||||
@@ -320,10 +363,11 @@ export function renderMarkdown(
|
||||
attachmentResolver?: AttachmentResolver,
|
||||
attachmentImageVariant: 'thumb-sm' | 'thumb-md' = 'thumb-md'
|
||||
): string {
|
||||
// Body may contain backslash-escaped chars (`\]`, `\\`, `\|`) — same
|
||||
// capture as wikiLinksToMarkdown so the two renderers accept identical
|
||||
// stored syntax (BUG-1744).
|
||||
const withLinks = content.replace(/\[\[((?:\\.|[^\]\\])+)\]\]/g, (_match, body: string) => {
|
||||
// Body may contain backslash-escaped chars (`\]`, `\\`, `\|`) — the SAME
|
||||
// pattern object as wikiLinksToMarkdown, so the two renderers cannot drift
|
||||
// apart in accepted syntax (BUG-1744 aligned them; BUG-2834 removed the
|
||||
// second copy that let them look aligned while diverging).
|
||||
const withLinks = content.replace(WIKI_LINK_PATTERN, (_match, body: string) => {
|
||||
// Cross-workspace form: [[workspace-slug::REF]] or [[workspace-slug::REF|Display]].
|
||||
// `::` is the unambiguous separator. The workspace prefix is recognized
|
||||
// only when both the slug AND the right-hand side match their expected
|
||||
@@ -620,9 +664,10 @@ function resolveWikiBody(body: string, items: Item[]): { item: Item | null; disp
|
||||
*/
|
||||
export function wikiLinksToMarkdown(content: string, items: Item[], workspaceSlug: string, username?: string): string {
|
||||
// Body may contain backslash-escaped chars (`\]`, `\\`, `\|`) so the tokens
|
||||
// we emit can carry arbitrary display text. The capture is (\\.|[^\]\\])+,
|
||||
// i.e. "a backslash-escaped char OR any non-`]`/non-`\` char".
|
||||
return content.replace(/\[\[((?:\\.|[^\]\\])+)\]\]/g, (_match, body: string) => {
|
||||
// we emit can carry arbitrary display text. Shares WIKI_LINK_PATTERN with
|
||||
// renderMarkdown — see its definition for why the escape alternative is
|
||||
// spelled `\\[^\n]` rather than `\\.` (BUG-2834).
|
||||
return content.replace(WIKI_LINK_PATTERN, (_match, body: string) => {
|
||||
const prefix = username ? `/${username}/${workspaceSlug}` : `/${workspaceSlug}`;
|
||||
|
||||
// Cross-workspace form: [[workspace::REF]] / [[workspace::REF|Display]].
|
||||
|
||||
Reference in New Issue
Block a user