diff --git a/docs/RELEASE_RESOURCE_EVIDENCE.md b/docs/RELEASE_RESOURCE_EVIDENCE.md new file mode 100644 index 000000000..c61095157 --- /dev/null +++ b/docs/RELEASE_RESOURCE_EVIDENCE.md @@ -0,0 +1,44 @@ +# Backend preflight resource evidence + +`release-preflight-worker.sh` emits one `RELEASE_RESOURCE_SNAPSHOT` JSON line +before and after the backend phase, plus `RELEASE_BACKEND_TOOLCHAIN` and +`RELEASE_BACKEND_EXIT`. The exact-preflight launcher's retained log preserves +these lines even when it removes its disposable worker directory. No success +receipt is produced by this evidence collector. Backend commands, exit status, +parallelism and assertion thresholds are unchanged. + +The collector reads only allowlisted kernel resource counters and numeric Go +runtime settings. It does not dump the environment, process command lines, +credentials or application data. Cgroups are labelled by ancestor distance, +not service/session names. An unmapped cgroup mount, missing file or collector +failure is explicitly unavailable evidence, never a zero counter. The standard +Linux cgroup2 root mount is required for hierarchy collection; other mappings +are deliberately not guessed. + +Compare cumulative counters between boundaries, checking counter availability +and for resets first. Parent counters include other work; host pressure and load +are not exclusive to the worker. Boundary snapshots cannot localise contention +to a test, detect all transient pressure, prove causation or prove exclusivity. +They do not record changes between boundaries. Abrupt termination may leave +only the before record. This instrumentation is not a repair of a product +latency failure and cannot clear prior adverse qualification evidence. + +Rationale: failed rehearsal 20260907T131615Z on +940f788dc29162d110fa7b896f5668262d8e4e7c retained a p95 latency miss but no +contemporaneous resource counters. The subsequent fixed twelve-sample isolated +comparison passed without measured throttling, but omitted preceding full-suite +workload and did not establish the cause of the original miss. Instrument future +otherwise-justified qualification rather than replaying to obtain a pass. + +The [kernel cgroup v2 documentation](https://docs.kernel.org/admin-guide/cgroup-v2.html) +defines ancestor restrictions, `cpu.stat` counters and pressure interfaces. +These interfaces supply context, not release readiness. No excluded crash or +database investigation is part of this collection. + +Focused checks: + +```sh +python3 -m unittest discover -s scripts/release_control/internal -p 'release_resource_snapshot_test.py' -v +python3 -m unittest discover -s scripts/release_control/internal -p 'release_preflight_test.py' -v +bash -n scripts/release-preflight-worker.sh +``` diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index acdbbb0ed..d049fd642 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -15,6 +15,29 @@ ## Purpose +### Backend preflight resource evidence + +The exact-source worker brackets the backend phase with bounded, read-only +resource snapshots in stdout so the launcher's retained log keeps evidence +when the disposable worker directory is removed after failure. The records +include the actual Go toolchain, backend exit status, allowlisted runtime +settings and host/cgroup counters including mapped ancestors. Environment +and process-command dumps, credentials and application data are excluded. +Unmapped hierarchies, missing counters and collection failures are unavailable +evidence, not zero pressure. Collection failure must neither replace a backend +failure nor prevent its ordinary after-boundary observation. + +The backend retains errexit semantics, original exit status, test commands, +concurrency and qualification thresholds. Phase-boundary evidence cannot +localise contention to a test, establish causation or clear historical adverse +qualification; abrupt termination may omit the after record. The executed +worker-function fixtures in +`scripts/release_control/internal/release_preflight_test.py` prove successful +and failed backend exits with successful and failed telemetry for both worker +profiles. Collector hierarchy and confidentiality fixtures remain in +`scripts/release_control/internal/release_resource_snapshot_test.py`. Neither +fixture constitutes full-suite or installed-release qualification. + ### Benchmark qualification evidence The Build and Test benchmark job retains `bench-metadata.txt` together with diff --git a/scripts/release-preflight-worker.sh b/scripts/release-preflight-worker.sh index 07b9a1e96..a19516430 100755 --- a/scripts/release-preflight-worker.sh +++ b/scripts/release-preflight-worker.sh @@ -209,7 +209,17 @@ run_frontend_tests() { phase frontend-tests npm --prefix frontend-modern test } -run_backend() { +# Keep evidence in stdout: the exact preflight launcher retains its log even +# when its disposable worker directory is removed after failure. This subshell +# retains errexit semantics; do not wrap run_backend in an `if` or `||` list. +run_backend() ( + backend_resource_snapshot() { + python3 ./scripts/release-resource-snapshot.py "$1" || + echo "RELEASE_RESOURCE_SNAPSHOT unavailable boundary=$1" >&2 + } + trap 'status=$?; backend_resource_snapshot after; echo "RELEASE_BACKEND_EXIT ${status}"; exit "$status"' EXIT + backend_resource_snapshot before + echo "RELEASE_BACKEND_TOOLCHAIN ${ACTUAL_GO}" rm -rf "$TEST_DATA_DIR" mkdir -p "$TEST_DATA_DIR" if [ "$PROFILE" = "rehearsal" ]; then @@ -219,7 +229,7 @@ run_backend() { --data-root "$TEST_DATA_DIR" \ --api-shards auto fi -} +) run_playwright() { docker run --rm \ diff --git a/scripts/release-resource-snapshot.py b/scripts/release-resource-snapshot.py new file mode 100644 index 000000000..71a408cd3 --- /dev/null +++ b/scripts/release-resource-snapshot.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Bounded, read-only backend phase evidence; never a release-gate verdict. + +Only allowlisted kernel files and numeric runtime knobs are emitted. No full +process environment, command lines, host identity or application data is read. +Snapshots bracket the backend phase, not an individual test. A missing counter +is unavailable evidence, not zero pressure. Ancestors may include other work. +""" +import json +import os +from pathlib import Path +import platform +import re +import sys +import time + +CGROUP_FILES = ( + "cpu.max", "cpu.stat", "cpu.pressure", "cpuset.cpus.effective", + "memory.current", "memory.max", "memory.events", "memory.pressure", + "io.pressure", "pids.current", "pids.max", +) + + +def read(path): + try: + with path.open() as stream: + return stream.read(8192).strip() + except OSError as error: + return {"unavailable_errno": error.errno} + + +def cgroups(proc=Path("/proc"), root=Path("/sys/fs/cgroup")): + membership = read(proc / "self/cgroup") + if not isinstance(membership, str): + return {"unavailable": "cgroup membership unreadable"} + paths = [line[3:] for line in membership.splitlines() if line.startswith("0::/")] + if len(paths) != 1 or ".." in Path(paths[0]).parts: + return {"unavailable": "no safely mapped unified cgroup"} + # Do not guess the mapping for a relocated or subtree cgroup mount. + mounts = read(proc / "self/mountinfo") + if not isinstance(mounts, str) or not any( + len(fields := line.split()) > 6 and fields[3] == "/" + and fields[4] == str(root) and " - cgroup2 " in line + for line in mounts.splitlines() + ): + return {"unavailable": "standard root cgroup2 mount not established"} + current = root / paths[0].lstrip("/") + result = [] + # Label by ancestor distance rather than disclosing service/session names. + while True: + result.append({"ancestor_distance": len(result), "files": { + name: read(current / name) for name in CGROUP_FILES + }}) + if current == root: + break + if len(result) >= 64: + return {"unavailable": "hierarchy exceeds collection bound", "partial": result} + current = current.parent + return result + + +def snapshot(): + knobs = {} + for name in ("GOMAXPROCS", "GOGC", "GOMEMLIMIT"): + value = os.environ.get(name) + knobs[name] = value if value is None or re.fullmatch(r"(?:[0-9]+(?:[KMGTPE]i?B)?|off)", value) else "nonstandard value omitted" + return { + "schema_version": 1, + "unix_time_ns": time.time_ns(), + "monotonic_ns": time.monotonic_ns(), + "kernel": platform.release(), + "architecture": platform.machine(), + "cpu_count": os.cpu_count(), + "cpu_affinity": sorted(os.sched_getaffinity(0)), + "runtime_knobs": knobs, + "host": {name: read(Path("/proc") / name) for name in ( + "loadavg", "pressure/cpu", "pressure/memory", "pressure/io", + )}, + "cgroups": cgroups(), + } + + +if __name__ == "__main__": + if len(sys.argv) != 2 or sys.argv[1] not in ("before", "after"): + raise SystemExit("Usage: release-resource-snapshot.py before|after") + print("RELEASE_RESOURCE_SNAPSHOT " + json.dumps({"boundary": sys.argv[1], **snapshot()}, sort_keys=True), flush=True) diff --git a/scripts/release_control/internal/release_preflight_test.py b/scripts/release_control/internal/release_preflight_test.py index 3a3de052e..8e3a8685b 100755 --- a/scripts/release_control/internal/release_preflight_test.py +++ b/scripts/release_control/internal/release_preflight_test.py @@ -282,6 +282,36 @@ class ReleasePreflightTest(unittest.TestCase): self.assertLess(block.index("done\n"), block.index("run_frontend_tests\n")) self.assertLess(block.index("run_frontend_tests\n"), block.index("run_backend\n")) + def test_backend_preserves_exit_and_collects_after_failure(self): + worker = (ROOT / 'scripts/release-preflight-worker.sh').read_text() + function = re.search(r'^run_backend\(\) \(\n.*?^\)', worker, re.M | re.S).group(0) + for profile in ('release', 'rehearsal'): + for code in (0, 17): + for telemetry_code in (0, 9): + with self.subTest(profile=profile, code=code, telemetry_code=telemetry_code): + # Stub only operations: execute the real backend shell function + # under errexit, without any product tests or data mutation. + script = f'''set -euo pipefail +PROFILE={profile} +TEST_DATA_DIR=unused +ACTUAL_GO=go-test-fixture +rm() {{ :; }} +mkdir() {{ :; }} +python3() {{ echo "snapshot $2"; return {telemetry_code}; }} +phase() {{ echo "phase $1"; return {code}; }} +{function} +run_backend +echo CONTINUED +''' + result = subprocess.run(['bash', '-c', script], text=True, capture_output=True) + self.assertEqual(result.returncode, code, result.stderr) + self.assertIn('snapshot before', result.stdout) + self.assertIn('snapshot after', result.stdout) + self.assertIn(f'RELEASE_BACKEND_EXIT {code}', result.stdout) + self.assertEqual('CONTINUED' in result.stdout, code == 0) + self.assertIn('backend-serial' if profile == 'rehearsal' else 'backend-race-sharded', result.stdout) + + def test_browser_mount_identity_and_locked_cli_follow_the_docker_daemon(self) -> None: worker = (ROOT / "scripts/release-preflight-worker.sh").read_text() functions = [] diff --git a/scripts/release_control/internal/release_resource_snapshot_test.py b/scripts/release_control/internal/release_resource_snapshot_test.py new file mode 100644 index 000000000..431dc285a --- /dev/null +++ b/scripts/release_control/internal/release_resource_snapshot_test.py @@ -0,0 +1,46 @@ +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location('resource_snapshot', ROOT / 'scripts/release-resource-snapshot.py') +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class ResourceSnapshotTest(unittest.TestCase): + def test_ancestors_and_missing_counters(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + proc, root = base / 'proc', base / 'cgroup' + (proc / 'self').mkdir(parents=True) + (root / 'parent/leaf').mkdir(parents=True) + (proc / 'self/cgroup').write_text('0::/parent/leaf\n') + (proc / 'self/mountinfo').write_text(f'1 0 0:1 / {root} rw - cgroup2 cgroup rw\n') + (root / 'parent/cpu.max').write_text('600000 100000\n') + result = module.cgroups(proc, root) + self.assertEqual(len(result), 3) + self.assertEqual(result[1]['files']['cpu.max'], '600000 100000') + self.assertEqual(result[0]['files']['cpu.stat'], {'unavailable_errno': 2}) + self.assertNotIn('parent', json.dumps(result)) + (proc / 'self/cgroup').write_text('0::/../../outside\n') + self.assertIn('unavailable', module.cgroups(proc, root)) + (proc / 'self/cgroup').write_text('0::/parent/leaf\n') + (proc / 'self/mountinfo').write_text('') + self.assertIn('unavailable', module.cgroups(proc, root)) + + def test_environment_allowlist_and_numeric_filter(self): + with patch.dict(os.environ, {'GITHUB_TOKEN': 'synthetic-secret', 'GOGC': 'synthetic-secret', 'GOMEMLIMIT': '512MiB', 'GOMAXPROCS': '2'}): + result = module.snapshot() + self.assertNotIn('synthetic-secret', json.dumps(result)) + self.assertEqual(result['runtime_knobs']['GOMEMLIMIT'], '512MiB') + self.assertEqual(result['runtime_knobs']['GOMAXPROCS'], '2') + + + +if __name__ == '__main__': + unittest.main()