diff --git a/.github/workflows/uuid-layout-diagnostic.yml b/.github/workflows/uuid-layout-diagnostic.yml new file mode 100644 index 000000000..3ad2deb39 --- /dev/null +++ b/.github/workflows/uuid-layout-diagnostic.yml @@ -0,0 +1,49 @@ +name: UUID layout diagnostic (not qualification) + +on: + workflow_dispatch: + inputs: + expected_workflow_sha: + description: Full reviewed main SHA containing this diagnostic + required: true + type: string + +permissions: + contents: read + +concurrency: + group: uuid-layout-diagnostic + cancel-in-progress: false + +jobs: + diagnostic: + if: github.repository == 'rcourtman/Pulse' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Bind execution to reviewed workflow source + env: + EXPECTED_SHA: ${{ inputs.expected_workflow_sha }} + run: | + set -euo pipefail + [[ "$EXPECTED_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$GITHUB_SHA" == "$EXPECTED_SHA" ]] + [[ "$GITHUB_RUN_ATTEMPT" == 1 ]] + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: '1.26.7' + cache: false + - name: Collect fixed four-condition UUID evidence + run: python3 scripts/diagnose-uuid-layout.py --output uuid-layout-evidence + - name: Retain partial or complete diagnostic evidence only + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: uuid-layout-diagnostic-${{ github.run_id }}-${{ github.run_attempt }} + path: uuid-layout-evidence/ + if-no-files-found: warn + retention-days: 30 diff --git a/docs/operations/uuid-layout-diagnostic.md b/docs/operations/uuid-layout-diagnostic.md new file mode 100644 index 000000000..98e4809bc --- /dev/null +++ b/docs/operations/uuid-layout-diagnostic.md @@ -0,0 +1,84 @@ +# Fixed UUID layout diagnostic — not release qualification + +This main-only workflow collects evidence for the failed PR1943 UUID comparison +in [run 34055039318](https://github.com/rcourtman/Pulse/actions/runs/34055039318). +It does not change the benchmark gate, its threshold, product declaration order, +release checks, or the candidate. Green here means collection completed, **not** +that the candidate passed its failed check. + +## Exact experiment + +- Baseline `5845692daf302a1524c2ca8fe6d06339ae7f8a24`, tree + `dedef52efe4fc28d1c05c9d957a0cc439f54db22`. +- Candidate `57f3b6401a6553efaa7646d5f1db446a2725b9dc`, tree + `7c2765cb06f424ce49647b33a5ebf3ade375e828`, identical to the failed + synthetic PR merge `ad6869a29cd0b54c918967d3cf41320914ba5790`. +- Ubuntu 24.04 hosted runner, Go **1.26.7** linux/amd64, no toolchain + auto-upgrade or restored build cache, GOMAXPROCS/test.cpu=4. +- Original baseline/candidate and the same two trees with `normalizeSegment` + moved immediately before `normalizeRoute`, without changing its body. Both + HTTP source and benchmark files must match their recorded SHA256 first. +- Four full API test-package compilations before measurement, identical frontend + embed stubs. No unit tests run. Only + `^BenchmarkNormalizeSegment$/^uuid$` executes, including warm-up. +- Ten rounds, forward/reverse condition order alternating, 300ms per sample, + one invocation per condition per round. No retries, early statistical stopping, + broad benchmark selection, or automatic verdict. A failed invocation stops + collection while preserving partial evidence. + +This reproduces the *design* of Core's retained 6 September controlled experiment, +not its local measurements. Hosted CPU and layout effects are still hypotheses. +Contemporaneous paired/interleaved samples reduce time-drift bias; they do not +eliminate shared-host interference. See [Go performance monitoring guidance](https://go.dev/wiki/PerformanceMonitoring). + +## Execution authority: unresolved dependency + +Source integration alone does not authorise Delivery to dispatch this workflow. +Delivery's standing dispatch permission covers **only Pro license/relay**; +starting the release assessment service is not permission to run arbitrary +Actions. Do not use either mechanism to smuggle this experiment into execution. + +After ordinary source review and main integration, an operator with repository +Actions dispatch authority must explicitly approve and perform **one** execution +of `uuid-layout-diagnostic.yml` on main, supplying its full reviewed main commit +as `expected_workflow_sha`. Alternatively the operator must explicitly grant a +bounded execution route. No such grant is supplied by this document. The workflow +rejects branch/SHA mismatch and Actions reruns (`run_attempt != 1`); a new manual +dispatch is not automatically a justified retry. Record the approval and dispatch +run identity internally. Before any further attempt, reconcile the first run and +its partial artifacts and obtain a reasoned diagnostic decision, not a green-run +search. A new source pair or toolchain requires source review, not free-form inputs. + +## Evidence and disposition + +Download the one `uuid-layout-diagnostic--` artifact. Require +`metadata.json` to say `complete: true`, verify `SHA256SUMS`, and verify all four +conditions have ten UUID-only samples. Retain raw per-invocation samples, +aggregates, chronological load/order records, exact source/tree identities, +workflow SHA/run/attempt, Go environment, CPU/kernel observations, binary hashes, +HTTP-only symbols/disassembly, and the four HTTP source files showing the +intervention. Experimental executables and source archives are temporary and +are never uploaded, packaged, tagged, installed, or proposed as candidates. +Partial artifacts cannot establish a completed experiment. + +Reuse the original hosted artifact (archive SHA256 +`61acc9f4e14fc0672f1eacaa81481e2ec497c71d302fe2bca30bc598c2a5354d`) +and Core's already-retained controlled results; do not overwrite or substitute +them. Compare original and reordered pairs separately with a recorded benchstat +version, retaining all results including adverse ones. The historical result is ++21.04%, ten samples; a smaller later number alone is not causal disposition. + +The release qualification owner must then explain whether the intervention +establishes a defensible disposition of the unchanged candidate or requires a +reviewed repair. Existing adverse evidence and the HOLD remain until that +judgment. This diagnostic neither investigates nor clears the separately excluded +SQLite work. No release assessment is warranted merely because this workflow +exists or completes. + +## Focused harness checks + +`python3 scripts/tests/test_uuid_layout_diagnostic.py` uses fake subprocesses to +check fixed scope, ordering, source/toolchain rejection, partial evidence, +non-overwrite behaviour and receipt completeness without executing benchmarks. +For any real local build/benchmark execution use `pulse-heavy-run -- `; +a further local sample is not the missing hosted evidence. diff --git a/scripts/diagnose-uuid-layout.py b/scripts/diagnose-uuid-layout.py new file mode 100644 index 000000000..8a1a2a1af --- /dev/null +++ b/scripts/diagnose-uuid-layout.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""One fixed diagnostic experiment; never a release qualification verdict.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +from datetime import datetime, timezone + +BASE = '5845692daf302a1524c2ca8fe6d06339ae7f8a24' +CANDIDATE = '57f3b6401a6553efaa7646d5f1db446a2725b9dc' +TREES = {BASE: 'dedef52efe4fc28d1c05c9d957a0cc439f54db22', + CANDIDATE: '7c2765cb06f424ce49647b33a5ebf3ade375e828'} +SOURCE_HASHES = { + 'http_metrics.go': '81fc091698cf7c1b011af70776dad53db3aa735367e51f4cda8a9bbf4b2b8527', + 'http_metrics_bench_test.go': '1562ae806264d8b22aece08fdadb7284760ae2374864479c7225b5051b5ba6b2', +} +LABELS = ('base', 'candidate', 'base-reordered', 'candidate-reordered') +SYMBOLS = r'api\.(normalizeSegment|BenchmarkNormalizeSegment)' +BENCH = '^BenchmarkNormalizeSegment$/^uuid$' +ROUNDS = 10 + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def run(args, cwd=None): + return subprocess.check_output(args, cwd=cwd, text=True, stderr=subprocess.STDOUT) + + +def capture(args, output, cwd=None): + # Preserve partial diagnostics on failure as well as successful output. + with output.open('w') as stream: + subprocess.run(args, cwd=cwd, stdout=stream, stderr=subprocess.STDOUT, check=True) + + +def reorder(source): + """Same declaration-only intervention as Core's retained controlled run.""" + for marker in ('func normalizeSegment(', 'func isNumeric(', 'func normalizeRoute('): + if source.count(marker) != 1: + raise ValueError('unexpected declaration layout') + start = source.index('func normalizeSegment(') + end = source.index('func isNumeric(', start) + block = source[start:end] + remainder = source[:start] + source[end:] + index = remainder.index('func normalizeRoute(') + return remainder[:index] + block + remainder[index:] + + +def order(round_number): + return LABELS if round_number % 2 else tuple(reversed(LABELS)) + + +def bench_args(binary, warmup=False): + return [str(binary), '-test.run=^$', '-test.bench=' + BENCH, + '-test.cpu=4', '-test.benchmem', '-test.count=1', + '-test.benchtime=' + ('1x' if warmup else '300ms'), '-test.timeout=2m'] + + +def validate_sample(sample): + rows = [line for line in sample.splitlines() if line.startswith('Benchmark')] + if len(rows) != 1 or not re.fullmatch( + r'BenchmarkNormalizeSegment/uuid-4\s+[1-9][0-9]*\s+[0-9.]+ ns/op\s+' + r'[0-9]+ B/op\s+[0-9]+ allocs/op', rows[0]): + raise ValueError('missing, duplicate, or out-of-scope benchmark sample') + + +def execute(repo, output): + output.mkdir(parents=True, exist_ok=False) # never overwrite a previous attempt + os.environ.update(GOTOOLCHAIN='local', GOMAXPROCS='4', GOFLAGS='', GOENV='off') + if run(['go', 'version']).strip() != 'go version go1.26.7 linux/amd64': + raise ValueError('requires exact hosted experiment toolchain go1.26.7 linux/amd64') + metadata = { + 'purpose': 'diagnostic-only; no gate disposition or release candidate', + 'rounds': ROUNDS, 'gomaxprocs': 4, 'benchtime': '300ms', + 'workflow_sha': os.environ.get('GITHUB_SHA'), + 'run_id': os.environ.get('GITHUB_RUN_ID'), + 'run_attempt': os.environ.get('GITHUB_RUN_ATTEMPT'), + 'conditions': {}, 'complete': False, + } + + def save(): + (output / 'metadata.json').write_text(json.dumps(metadata, indent=2) + '\n') + + save() + capture(['go', 'version'], output / 'go-version.txt') + capture(['go', 'env', 'GOOS', 'GOARCH', 'GOVERSION', 'GOTOOLCHAIN', 'GOAMD64', + 'CGO_ENABLED', 'GOFLAGS'], output / 'go-env.txt') + capture(['lscpu'], output / 'cpu.txt') + capture(['uname', '-smr'], output / 'kernel.txt') + # All four builds complete before warm-up/measurement; no compilation drift. + with tempfile.TemporaryDirectory(prefix='uuid-layout-') as temporary: + work = Path(temporary) + binaries = {} + for label in LABELS: + revision = CANDIDATE if label.startswith('candidate') else BASE + tree = run(['git', 'rev-parse', revision + '^{tree}'], repo).strip() + if tree != TREES[revision]: + raise ValueError('unexpected source tree') + source = work / label + source.mkdir() + archive = work / (label + '.tar') + subprocess.run(['git', 'archive', '-o', str(archive), revision], cwd=repo, check=True) + subprocess.run(['tar', '-xf', str(archive), '-C', str(source)], check=True) + for filename, expected in SOURCE_HASHES.items(): + if digest(source / 'internal/api' / filename) != expected: + raise ValueError('unexpected HTTP source content') + metrics = source / 'internal/api/http_metrics.go' + original = metrics.read_text() + if label.endswith('-reordered'): + metrics.write_text(reorder(original)) + # Retain the exact intervention, not the experimental executable. + (output / (label + '-http_metrics.go')).write_text(metrics.read_text()) + stub = source / 'internal/api/frontend-modern/dist/index.html' + stub.parent.mkdir(parents=True, exist_ok=True) + stub.write_text('ci embed stub\n') + binary = work / (label + '.test') + capture(['go', 'test', '-c', '-o', str(binary), './internal/api'], + output / (label + '-build.txt'), source) + binaries[label] = binary + metadata['conditions'][label] = { + 'commit': revision, 'tree': tree, 'binary_sha256': digest(binary), + 'experimental': label.endswith('-reordered'), + 'http_metrics_sha256': digest(metrics), 'embed_stub_sha256': digest(stub), + } + save() + capture(['go', 'tool', 'objdump', '-s', SYMBOLS, str(binary)], + output / (label + '-http.asm')) + # Restrict retained symbols to the HTTP normaliser experiment. + symbols = run(['go', 'tool', 'nm', str(binary)]) + (output / (label + '-symbols.txt')).write_text('\n'.join( + line for line in symbols.splitlines() if re.search(SYMBOLS, line)) + '\n') + for label in LABELS: + capture(bench_args(binaries[label], warmup=True), output / (label + '-warmup.txt')) + for round_number in range(1, ROUNDS + 1): + for label in order(round_number): + with (output / 'order.jsonl').open('a') as log: + log.write(json.dumps({'round': round_number, 'condition': label, + 'at': datetime.now(timezone.utc).isoformat(), + 'load': os.getloadavg()}) + '\n') + sample_path = output / f'{label}-{round_number:02}.txt' + capture(bench_args(binaries[label]), sample_path) + sample = sample_path.read_text() + validate_sample(sample) + with (output / (label + '.txt')).open('a') as aggregate: + aggregate.write(sample) + metadata['complete'] = True + save() + files = sorted(path for path in output.iterdir() if path.is_file()) + (output / 'SHA256SUMS').write_text(''.join(f'{digest(path)} {path.name}\n' for path in files)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', required=True, type=Path) + args = parser.parse_args() + execute(Path(__file__).resolve().parents[1], args.output.resolve()) diff --git a/scripts/tests/test_uuid_layout_diagnostic.py b/scripts/tests/test_uuid_layout_diagnostic.py new file mode 100644 index 000000000..c00baeeba --- /dev/null +++ b/scripts/tests/test_uuid_layout_diagnostic.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""No real build or benchmark: verify the diagnostic's bounds and receipts.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('diagnostic', ROOT / 'scripts/diagnose-uuid-layout.py') +diag = importlib.util.module_from_spec(spec) +spec.loader.exec_module(diag) +SOURCE = ('package api\n\nfunc normalizeRoute() {}\n\n' + 'func normalizeSegment() {}\n\nfunc isNumeric() {}\n') +SAMPLE = 'BenchmarkNormalizeSegment/uuid-4 1000 51.64 ns/op 0 B/op 0 allocs/op\nPASS\n' + + +class DiagnosticTests(unittest.TestCase): + def test_reorder_only_moves_declaration(self): + expected = ('package api\n\nfunc normalizeSegment() {}\n\n' + 'func normalizeRoute() {}\n\nfunc isNumeric() {}\n') + self.assertEqual(diag.reorder(SOURCE), expected) + with self.assertRaises(ValueError): + diag.reorder(SOURCE + 'func normalizeSegment() {}\n') + with self.assertRaises(ValueError): + diag.reorder('package api') + + def test_ten_balanced_alternating_rounds(self): + self.assertEqual(diag.ROUNDS, 10) + for label in diag.LABELS: + for position in range(4): + self.assertIn(sum(diag.order(n)[position] == label for n in range(1, 11)), (0, 5)) + self.assertEqual(diag.order(2), tuple(reversed(diag.order(1)))) + + def test_execution_is_uuid_only(self): + for warmup in (True, False): + args = diag.bench_args(Path('/tmp/fixture.test'), warmup) + self.assertIn('-test.run=^$', args) + self.assertIn('-test.bench=^BenchmarkNormalizeSegment$/^uuid$', args) + self.assertIn('-test.cpu=4', args) + self.assertIn('-test.count=1', args) + self.assertIn('-test.timeout=2m', args) + + def test_sample_validation_fails_closed(self): + diag.validate_sample(SAMPLE) + for bad in ('PASS\n', SAMPLE * 2, SAMPLE.replace('uuid', 'numeric'), + SAMPLE.replace('-4', '-8'), SAMPLE.replace('1000', '0')): + with self.subTest(bad=bad), self.assertRaises(ValueError): + diag.validate_sample(bad) + + def exercise(self, output, fail_sample=False, bad_tree=False, bad_go=False, bad_source=False): + captured = [] + hashes = {'http_metrics.go': hashlib.sha256(SOURCE.encode()).hexdigest(), + 'http_metrics_bench_test.go': hashlib.sha256(b'fixture').hexdigest()} + + def fake_run(args, cwd=None): + if args == ['go', 'version']: + return 'go version go1.26.8 linux/amd64' if bad_go else 'go version go1.26.7 linux/amd64' + if args[:2] == ['git', 'rev-parse']: + return 'wrong' if bad_tree else diag.TREES[args[2].split('^')[0]] + if args[:3] == ['go', 'tool', 'nm']: + return '123 T api.normalizeSegment\n456 T api.BenchmarkNormalizeSegment\n789 T other.symbol' + raise AssertionError(args) + + def fake_subprocess(args, **kwargs): + if args[:2] == ['git', 'archive']: + return + if args[:2] == ['tar', '-xf']: + folder = Path(args[-1]) / 'internal/api' + folder.mkdir(parents=True) + (folder / 'http_metrics.go').write_text('changed' if bad_source else SOURCE) + (folder / 'http_metrics_bench_test.go').write_text('fixture') + return + raise AssertionError(args) + + def fake_capture(args, path, cwd=None): + captured.append(args) + if args[:3] == ['go', 'test', '-c']: + Path(args[4]).write_bytes(b'fake binary') + path.write_text('compiled') + elif args[0].endswith('.test'): + path.write_text(SAMPLE) + if fail_sample and '-test.benchtime=300ms' in args: + raise subprocess.CalledProcessError(1, args) + else: + path.write_text('fixture metadata') + + with patch.object(diag, 'run', side_effect=fake_run), \ + patch.object(diag, 'capture', side_effect=fake_capture), \ + patch.object(diag.subprocess, 'run', side_effect=fake_subprocess), \ + patch.object(diag, 'SOURCE_HASHES', hashes), patch.dict(diag.os.environ): + diag.execute(ROOT, output) + return captured + + def test_complete_receipt_and_no_uploaded_executables(self): + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / 'evidence' + calls = self.exercise(output) + metadata = json.loads((output / 'metadata.json').read_text()) + self.assertTrue(metadata['complete']) + self.assertEqual(len(metadata['conditions']), 4) + for label in diag.LABELS: + self.assertEqual((output / (label + '.txt')).read_text(), SAMPLE * 10) + self.assertEqual(metadata['conditions'][label]['experimental'], label.endswith('-reordered')) + orders = [json.loads(line) for line in (output / 'order.jsonl').read_text().splitlines()] + self.assertEqual([row['condition'] for row in orders], + [label for n in range(1, 11) for label in diag.order(n)]) + self.assertEqual(sum(call[:3] == ['go', 'test', '-c'] for call in calls), 4) + self.assertEqual(sum(call[0].endswith('.test') for call in calls), 44) + first_benchmark = next(i for i, call in enumerate(calls) if call[0].endswith('.test')) + self.assertEqual(sum(call[:3] == ['go', 'test', '-c'] + for call in calls[:first_benchmark]), 4) + self.assertFalse(list(output.glob('*.test'))) + for line in (output / 'SHA256SUMS').read_text().splitlines(): + digest, name = line.split(' ') + self.assertEqual(diag.digest(output / name), digest) + + def test_partial_evidence_survives_failure_without_success_claim(self): + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / 'evidence' + with self.assertRaises(subprocess.CalledProcessError): + self.exercise(output, fail_sample=True) + self.assertFalse(json.loads((output / 'metadata.json').read_text())['complete']) + self.assertTrue((output / 'base-01.txt').exists()) + self.assertFalse((output / 'SHA256SUMS').exists()) + + def test_wrong_tree_or_toolchain_rejected(self): + for option in ('bad_tree', 'bad_go', 'bad_source'): + with tempfile.TemporaryDirectory() as tmp, self.assertRaises(ValueError): + self.exercise(Path(tmp) / 'evidence', **{option: True}) + + def test_refuses_overwriting_receipt(self): + with tempfile.TemporaryDirectory() as tmp, self.assertRaises(FileExistsError): + diag.execute(ROOT, Path(tmp)) + + def test_workflow_authority_and_artifact_bounds(self): + workflow = (ROOT / '.github/workflows/uuid-layout-diagnostic.yml').read_text() + self.assertIn('workflow_dispatch:', workflow) + self.assertNotIn('pull_request:', workflow) + self.assertNotIn('push:', workflow) + self.assertIn('contents: read', workflow) + self.assertNotIn('secrets.', workflow) + self.assertIn('github.ref == \'refs/heads/main\'', workflow) + self.assertIn('[[ "$GITHUB_SHA" == "$EXPECTED_SHA" ]]', workflow) + self.assertIn('[[ "$GITHUB_RUN_ATTEMPT" == 1 ]]', workflow) + self.assertIn('path: uuid-layout-evidence/', workflow) + self.assertIn('if: always()', workflow) + self.assertIn("go-version: '1.26.7'", workflow) + self.assertIn('timeout-minutes: 45', workflow) + + +if __name__ == '__main__': + unittest.main()