From d28666c39616a0e5918a66b0cb48204cb45b2acd Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:16:05 +0100 Subject: [PATCH] Guard shipped docs mirror sync at commit time Docs under frontend-modern/public/docs are byte-for-byte copies of repo docs, but the only guard was a CI vitest the git hooks never run. On 2026-09-01 two commits (f4886c2dfb, f313882a7b) each edited a mirrored doc without its copy, passed the hooks, and broke main's Frontend job. Add scripts/check_docs_mirror.py with an index-blob --staged mode wired into the pre-commit hook: a commit that stages either side of an out-of-sync pair (or an orphan shipped copy) fails with the exact sync command, while pre-existing drift from other commits only warns. The worktree mode runs as a named step in the public-docs workflow, with unit tests in scripts/tests picked up by the existing runner. The docsLinks vitest stays as the CI backstop. Build-time generation of public/docs was considered and rejected: the shipped set is a curated subset (61 of 421 docs), so generation still needs a hand-maintained manifest while adding build, dev-server, and test-order coupling. --- .github/workflows/public-docs.yml | 7 ++ .husky/pre-commit | 9 ++ scripts/check_docs_mirror.py | 199 ++++++++++++++++++++++++++++++ scripts/tests/test_docs_mirror.py | 163 ++++++++++++++++++++++++ 4 files changed, 378 insertions(+) create mode 100755 scripts/check_docs_mirror.py create mode 100644 scripts/tests/test_docs_mirror.py diff --git a/.github/workflows/public-docs.yml b/.github/workflows/public-docs.yml index 4acd50e24..792f371c5 100644 --- a/.github/workflows/public-docs.yml +++ b/.github/workflows/public-docs.yml @@ -5,8 +5,10 @@ on: paths: - "*.md" - "docs/**" + - "frontend-modern/public/docs/**" - ".github/ISSUE_TEMPLATE/**" - "scripts/check_public_docs.py" + - "scripts/check_docs_mirror.py" - ".github/workflows/public-docs.yml" push: branches: @@ -14,8 +16,10 @@ on: paths: - "*.md" - "docs/**" + - "frontend-modern/public/docs/**" - ".github/ISSUE_TEMPLATE/**" - "scripts/check_public_docs.py" + - "scripts/check_docs_mirror.py" - ".github/workflows/public-docs.yml" permissions: @@ -37,3 +41,6 @@ jobs: - name: Validate public documentation run: python3 scripts/check_public_docs.py + + - name: Check shipped docs mirror sync + run: python3 scripts/check_docs_mirror.py diff --git a/.husky/pre-commit b/.husky/pre-commit index ecafc1fe8..53f534a53 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -73,6 +73,15 @@ staged_files_match() { python3 scripts/release_control/format_staged_go.py python3 scripts/release_control/format_staged_frontend.py +# Shipped docs under frontend-modern/public/docs are byte-for-byte copies of +# repo docs, enforced only by a CI vitest the hooks never run. Two 2026-09-01 +# commits (f4886c2dfb, f313882a7b) each broke main's Frontend job by editing a +# mirrored doc without its copy; catch that before the commit exists. +if staged_files_match '^docs/|^frontend-modern/public/docs/|^(SECURITY|TERMS|ARCHITECTURE|CONTRIBUTING)\.md$'; then + echo "Running shipped docs mirror check..." + python3 scripts/check_docs_mirror.py --staged +fi + echo "Running browser verification guard..." python3 scripts/release_control/browser_verification_guard.py diff --git a/scripts/check_docs_mirror.py b/scripts/check_docs_mirror.py new file mode 100755 index 000000000..523d0a2fe --- /dev/null +++ b/scripts/check_docs_mirror.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Keep shipped doc copies under frontend-modern/public/docs in sync. + +Every Markdown file under frontend-modern/public/docs/ is a byte-for-byte +copy of a repo doc: docs/, except a small set shipped +from the repository root. CI enforces this in the docsLinks vitest +("keeps shipped docs content synced with repo docs"), but the git hooks do +not run vitest, so a commit that edits a mirrored doc without its copy +passes the hooks and breaks main's Frontend job. + +Modes: + --staged Compare index blobs; fail only when this commit stages either + side of an out-of-sync pair (pre-existing drift warns). Run + from the pre-commit hook. + (none) Compare working-tree bytes for every shipped doc. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + +SHIPPED_ROOT = "frontend-modern/public/docs" + +# Shipped from the repository root rather than docs/. Mirrors the +# rootSourcedDocs set in frontend-modern/src/utils/__tests__/docsLinks.test.ts. +ROOT_SOURCED_DOCS = frozenset( + {"SECURITY.md", "TERMS.md", "ARCHITECTURE.md", "CONTRIBUTING.md"} +) + +# Shipped docs must not deep-link into the GitHub tree; the docsLinks vitest +# rejects this in CI. +FORBIDDEN_LINK = "https://github.com/rcourtman/Pulse/blob/main/" + + +def source_for(mirror: str) -> str: + relative = mirror[len(SHIPPED_ROOT) + 1 :] + if relative in ROOT_SOURCED_DOCS: + return relative + return f"docs/{relative}" + + +def git_output(root: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(root), *args], + check=True, + capture_output=True, + text=True, + ).stdout + + +def index_blobs(root: Path) -> dict[str, str]: + """Map index path -> blob OID for docs surfaces (equal OID = equal bytes).""" + + records = git_output( + root, + "ls-files", + "-s", + "-z", + "--", + SHIPPED_ROOT, + "docs", + *sorted(ROOT_SOURCED_DOCS), + ) + blobs: dict[str, str] = {} + for record in records.split("\0"): + if not record: + continue + meta, path = record.split("\t", 1) + blobs[path] = meta.split()[1] + return blobs + + +def staged_paths(root: Path) -> set[str]: + return { + path + for path in git_output( + root, "diff", "--cached", "--name-only", "-z" + ).split("\0") + if path + } + + +def sync_command(source: str, mirror: str) -> str: + return f"git show :{source} > {mirror} && git add {mirror}" + + +def check_staged(root: Path) -> tuple[list[str], list[str]]: + blobs = index_blobs(root) + staged = staged_paths(root) + errors: list[str] = [] + warnings: list[str] = [] + + for mirror in sorted(blobs): + if not mirror.startswith(f"{SHIPPED_ROOT}/") or not mirror.endswith(".md"): + continue + source = source_for(mirror) + touched = mirror in staged or source in staged + source_blob = blobs.get(source) + + if source_blob is None: + message = ( + f"{mirror}: shipped copy has no repo source {source}; " + f"remove the copy too (git rm {mirror}) or restore the source" + ) + elif source_blob != blobs[mirror]: + message = ( + f"{source} and {mirror} differ in the staged tree; sync the " + f"shipped copy from the staged source:\n" + f" {sync_command(source, mirror)}\n" + f" (if the edit was made to the shipped copy, apply it to " + f"{source} instead)" + ) + else: + if touched and FORBIDDEN_LINK in git_output( + root, "cat-file", "blob", blobs[mirror] + ): + errors.append( + f"{mirror}: shipped docs must not link to " + f"{FORBIDDEN_LINK} (link the shipped path instead)" + ) + continue + + (errors if touched else warnings).append(message) + + return errors, warnings + + +def check_worktree(root: Path) -> tuple[list[str], int]: + errors: list[str] = [] + shipped_root = root / SHIPPED_ROOT + mirrors = sorted( + path.relative_to(root).as_posix() for path in shipped_root.rglob("*.md") + ) + + for mirror in mirrors: + source = source_for(mirror) + source_path = root / source + if not source_path.exists(): + errors.append(f"{mirror}: shipped copy has no repo source {source}") + continue + mirror_bytes = (root / mirror).read_bytes() + if mirror_bytes != source_path.read_bytes(): + errors.append( + f"{source} and {mirror} differ; sync the shipped copy:\n" + f" cp {source} {mirror}" + ) + elif FORBIDDEN_LINK.encode() in mirror_bytes: + errors.append( + f"{mirror}: shipped docs must not link to {FORBIDDEN_LINK}" + ) + + return errors, len(mirrors) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--staged", + action="store_true", + help="compare index blobs and fail only on pairs this commit touches", + ) + args = parser.parse_args() + + if args.staged: + errors, warnings = check_staged(ROOT) + if warnings: + print( + "Warning: shipped docs already out of sync in HEAD " + "(not touched by this commit; CI is failing on main):", + file=sys.stderr, + ) + for warning in warnings: + print(f"- {warning}", file=sys.stderr) + if errors: + print("Shipped docs mirror check failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("Shipped docs mirror check passed.") + return 0 + + errors, checked = check_worktree(ROOT) + if errors: + print("Shipped docs mirror check failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print(f"Shipped docs mirror check passed ({checked} shipped docs).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_docs_mirror.py b/scripts/tests/test_docs_mirror.py new file mode 100644 index 000000000..8376ad149 --- /dev/null +++ b/scripts/tests/test_docs_mirror.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Guard the shipped-docs mirror check that backs the pre-commit hook.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "check_docs_mirror.py" +SPEC = importlib.util.spec_from_file_location("check_docs_mirror", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +docs_mirror = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = docs_mirror +SPEC.loader.exec_module(docs_mirror) + + +def run_git(root: Path, *args: str) -> None: + subprocess.run( + [ + "git", + "-C", + str(root), + "-c", + "user.email=test@example.invalid", + "-c", + "user.name=test", + *args, + ], + check=True, + capture_output=True, + ) + + +def write(root: Path, relative: str, content: str) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +class DocsMirrorMappingTest(unittest.TestCase): + def test_docs_sourced_mapping(self) -> None: + self.assertEqual( + docs_mirror.source_for("frontend-modern/public/docs/i18n/de/README.md"), + "docs/i18n/de/README.md", + ) + + def test_root_sourced_mapping(self) -> None: + self.assertEqual( + docs_mirror.source_for("frontend-modern/public/docs/SECURITY.md"), + "SECURITY.md", + ) + + +class DocsMirrorStagedTest(unittest.TestCase): + def setUp(self) -> None: + self._temporary = tempfile.TemporaryDirectory() + self.addCleanup(self._temporary.cleanup) + self.root = Path(self._temporary.name) + run_git(self.root, "init", "-q") + + def test_synced_staged_pair_passes(self) -> None: + write(self.root, "docs/GUIDE.md", "# Guide\n") + write(self.root, "frontend-modern/public/docs/GUIDE.md", "# Guide\n") + run_git(self.root, "add", "docs/GUIDE.md", "frontend-modern/public/docs/GUIDE.md") + + errors, warnings = docs_mirror.check_staged(self.root) + + self.assertEqual(errors, []) + self.assertEqual(warnings, []) + + def test_staged_source_with_stale_mirror_fails(self) -> None: + write(self.root, "docs/GUIDE.md", "# Guide v1\n") + write(self.root, "frontend-modern/public/docs/GUIDE.md", "# Guide v1\n") + run_git(self.root, "add", "-A") + run_git(self.root, "commit", "-q", "-m", "seed") + write(self.root, "docs/GUIDE.md", "# Guide v2\n") + run_git(self.root, "add", "docs/GUIDE.md") + + errors, warnings = docs_mirror.check_staged(self.root) + + self.assertEqual(len(errors), 1) + self.assertIn("docs/GUIDE.md and frontend-modern/public/docs/GUIDE.md", errors[0]) + self.assertIn("git show :docs/GUIDE.md", errors[0]) + self.assertEqual(warnings, []) + + def test_staged_root_sourced_doc_with_stale_mirror_fails(self) -> None: + write(self.root, "SECURITY.md", "# Security v1\n") + write(self.root, "frontend-modern/public/docs/SECURITY.md", "# Security v1\n") + run_git(self.root, "add", "-A") + run_git(self.root, "commit", "-q", "-m", "seed") + write(self.root, "SECURITY.md", "# Security v2\n") + run_git(self.root, "add", "SECURITY.md") + + errors, _warnings = docs_mirror.check_staged(self.root) + + self.assertEqual(len(errors), 1) + self.assertIn("SECURITY.md and frontend-modern/public/docs/SECURITY.md", errors[0]) + + def test_staged_orphan_mirror_fails(self) -> None: + write(self.root, "frontend-modern/public/docs/NEW.md", "# New\n") + run_git(self.root, "add", "frontend-modern/public/docs/NEW.md") + + errors, _warnings = docs_mirror.check_staged(self.root) + + self.assertEqual(len(errors), 1) + self.assertIn("no repo source docs/NEW.md", errors[0]) + + def test_preexisting_drift_only_warns_on_unrelated_commit(self) -> None: + write(self.root, "docs/GUIDE.md", "# Guide v2\n") + write(self.root, "frontend-modern/public/docs/GUIDE.md", "# Guide v1\n") + run_git(self.root, "add", "-A") + run_git(self.root, "commit", "-q", "-m", "seed drift") + write(self.root, "unrelated.txt", "x\n") + run_git(self.root, "add", "unrelated.txt") + + errors, warnings = docs_mirror.check_staged(self.root) + + self.assertEqual(errors, []) + self.assertEqual(len(warnings), 1) + self.assertIn("docs/GUIDE.md and frontend-modern/public/docs/GUIDE.md", warnings[0]) + + def test_staged_pair_with_github_tree_link_fails(self) -> None: + content = "See https://github.com/rcourtman/Pulse/blob/main/docs/OTHER.md\n" + write(self.root, "docs/GUIDE.md", content) + write(self.root, "frontend-modern/public/docs/GUIDE.md", content) + run_git(self.root, "add", "-A") + + errors, _warnings = docs_mirror.check_staged(self.root) + + self.assertEqual(len(errors), 1) + self.assertIn("must not link to", errors[0]) + + +class DocsMirrorWorktreeTest(unittest.TestCase): + def test_worktree_drift_and_sync(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + write(root, "docs/GUIDE.md", "# Guide v2\n") + write(root, "frontend-modern/public/docs/GUIDE.md", "# Guide v1\n") + write(root, "SECURITY.md", "# Security\n") + write(root, "frontend-modern/public/docs/SECURITY.md", "# Security\n") + + errors, checked = docs_mirror.check_worktree(root) + + self.assertEqual(checked, 2) + self.assertEqual(len(errors), 1) + self.assertIn("cp docs/GUIDE.md frontend-modern/public/docs/GUIDE.md", errors[0]) + + def test_repo_shipped_docs_are_synced(self) -> None: + errors, checked = docs_mirror.check_worktree(ROOT) + + self.assertEqual(errors, []) + self.assertGreater(checked, 0) + + +if __name__ == "__main__": + unittest.main()