diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 9e7cc1e7ab..c72ec6504a 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -773,11 +773,14 @@ jobs: run: go build ./cmd/pulse chart-latency-slo: - name: Workload chart 90ms SLO + name: Workload chart SLO evidence needs: changes if: github.event_name == 'pull_request' && needs.changes.outputs.go_compile == 'true' runs-on: ubuntu-24.04 timeout-minutes: 25 + outputs: + conclusion: ${{ steps.proof.outputs.conclusion }} + report: ${{ steps.proof.outputs.report }} env: GOMAXPROCS: "2" steps: @@ -822,11 +825,34 @@ jobs: # This additional evidence job keeps the ordinary hosted envelope and # required source/release gates intact. A fresh VM must first demonstrate # idle capacity; the candidate then enforces the existing local 90 ms SLO. + # Competing capacity during the measurement (on hosted runners, the + # runner's own provjobd daemon in every observed run) voids the latency + # verdict: the proof exits 78 and the verdict check is neutral, not red. - name: Verify chart latency on observed idle capacity + id: proof run: | + set +e python3 internal/api/testdata/chart_latency_proof.py \ --candidate "$RUNNER_TEMP/chart-latency/candidate-api.test" \ --output "$RUNNER_TEMP/chart-latency/samples" + status=$? + set -e + case "$status" in + 0) conclusion=success ;; + 78) conclusion=neutral ;; + *) conclusion=failure ;; + esac + report="$(grep -o 'charts/workloads p50=[^ ]* p95=[^ ]* p99=[^ ]* SLO=[^ ]*' \ + "$RUNNER_TEMP/chart-latency/samples/candidate.log" 2>/dev/null | head -n 1 || true)" + { + echo "conclusion=$conclusion" + echo "report=$report" + } >> "$GITHUB_OUTPUT" + # The proof writes its own summary for an inconclusive run. + [ "$conclusion" = neutral ] || + printf 'Workload chart 90ms SLO: %s\n\n%s\n' "$conclusion" \ + "${report:-No SLO report was produced.}" >> "$GITHUB_STEP_SUMMARY" + [ "$status" -eq 0 ] || [ "$status" -eq 78 ] - name: Upload chart latency evidence if: always() @@ -839,6 +865,40 @@ jobs: if-no-files-found: warn retention-days: 7 + # A separate job holds the check-write token, so the evidence job that runs + # PR-built executables stays read-only. Dependabot's read-only token cannot + # create check runs; the evidence job summary still carries the verdict. + # An evidence job that failed before the proof ran publishes a failure. + chart-latency-verdict: + name: Publish chart SLO verdict + needs: chart-latency-slo + if: always() && (needs.chart-latency-slo.result == 'success' || needs.chart-latency-slo.result == 'failure') + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + checks: write + steps: + - name: Publish SLO verdict check + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CONCLUSION: ${{ needs.chart-latency-slo.outputs.conclusion || 'failure' }} + REPORT: ${{ needs.chart-latency-slo.outputs.report }} + DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + case "$CONCLUSION" in + success) title='p95 within 90 ms on an idle runner' ;; + neutral) title='Inconclusive: the runner was not idle during the measurement' ;; + *) title='Failed on an idle runner, or the proof could not run' ;; + esac + gh api "repos/$GITHUB_REPOSITORY/check-runs" \ + -f name='Workload chart 90ms SLO' -f head_sha="$HEAD_SHA" \ + -f status=completed -f conclusion="$CONCLUSION" -f details_url="$DETAILS_URL" \ + -f "output[title]=$title" \ + -f "output[summary]=${REPORT:-No SLO report was produced.} See the evidence job summary and chart-latency artifact." \ + > /dev/null || + echo "::warning title=Chart SLO verdict not published::This token cannot create check runs; the evidence job carries the verdict." + benchmarks: name: Benchmarks needs: changes diff --git a/docs/release-control/v6/internal/records/workload-chart-reporting-completion-2026-10-21.md b/docs/release-control/v6/internal/records/workload-chart-reporting-completion-2026-10-21.md new file mode 100644 index 0000000000..b29ff143cc --- /dev/null +++ b/docs/release-control/v6/internal/records/workload-chart-reporting-completion-2026-10-21.md @@ -0,0 +1,27 @@ +# Completing hosted workload-chart reporting, 21 October 2026 + +Original proposal: [Pulse PR 3095](https://github.com/rcourtman/Pulse/pull/3095), +Richard Courtman's shared `b31fabd778f64f4bc35d1f33438256a2b24ff443`. +The two Python proof files are preserved byte-for-byte; only its chart evidence +and verdict jobs are composed into the assigned maintained workflow. Other +workflow jobs and the current frontend/toolchain remain untouched. The shared +commit is not amended, and its unqualified historical timing conclusions are +not adopted as a regression verdict. + +[Governance job 119307595195](https://github.com/rcourtman/Pulse/actions/runs/39624922365/job/119307595195) +rejects missing dependent contracts and API/deployment verification. This +completion supplies those contracts and executed CLI/workflow regression +controls rather than waiving the guard or changing the latency threshold. +A broad composition was abandoned because it would pull unrelated frontend +and toolchain changes into this bounded repair. The original proposal remains +shared and unchanged; the credited canonical equivalent receives its own +independent review, containing CI and protected publication. + +The classifier distinguishes a valid 90 ms pass/fail from an invalidated +measurement. A neutral hosted check is not latency qualification, native +recovery, released availability or evidence that a regression is absent. +Synthetic child/telemetry and fake-provider controls establish classification, +raw-evidence preservation and permission separation only. No performance test, +database investigation, workflow retry or release operation is part of this +completion. The current hosted artifact, exact source proof, adverse results +and remaining delivery state are retained in Delivery's linked outcome. diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index f173ba07e6..8e34b05d8b 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -10551,3 +10551,15 @@ effective environment and command, not its exit alone. It also consumes the actual XML producers with a plist parser and checks shell-export round trips. These are disposable parser proofs, not a running systemd/launchd agent, installed TLS recovery or a change to the saved-state/transport contract. + +### Hosted chart evidence cannot qualify an agent lifecycle + +The optional hosted workload-chart capacity proof reports an executed but +contended measurement as inconclusive, never as a healthy reading or a passed +90 ms budget. This CI-only classification does not satisfy agent installation, +restart, confinement, credential persistence or signed recovery acceptance. +Skips, missing reports and non-budget child failures still fail the proof. +The actual CLI/receipt control is +`TestContractChartSLOProofCLIClassifiesExecution`; the separate read-only +evidence and check-writer boundary is +`TestChartSLOWorkflowPreservesInconclusiveAndFailure`. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 5be5084834..b861fc3801 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -12726,3 +12726,17 @@ and manual-token recovery without another provisioning request. Its emitted safe HTTP responses bind consuming-editor verification to the producer. This source fixture does not establish installed provisioning, guest liveness or reporter recovery. + +### Workload-chart proof distinguishes an invalid measurement from recovery + +`internal/api/testdata/chart_latency_proof.py` keeps the strict 90 ms budget, +exact one-test execution, executable digest and child result in its receipt. +Only a single complete PASS or budget-only FAIL with competing capacity can +exit 78 and describe the timing as inconclusive. A skipped/missing/duplicate +report, wrong budget, cleanup failure or other child failure remains an error. +A valid-capacity budget failure still fails. No API payload, route, History +sample, incident state or operational-recovery result is changed by this CI +classification. `TestContractChartSLOProofCLIClassifiesExecution` drives the +real CLI and disposable child processes through all ten outcomes, verifies +unaltered logs/result/digest/strict arguments and excludes the synthetic token +from the child environment. Its capacity rows are synthetic, not latency proof. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 82b94b00a1..f6e59f0b0b 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -4555,6 +4555,22 @@ artifact-selection behaviour. the manifest range, the locked `@types/node`/`undici-types` versions and the `ES2022` lib declaration stay in step. +### Hosted chart verdict writer has no proposed executable + +Build and Test's optional workload-chart evidence job retains a read-only +repository token while running the PR-built executable. Its proof maps only +exit 0 to success and 78 to neutral; any other exit fails. The separate check +writer has only `checks: write`, no checkout or proposed executable, uses the +exact PR head and defaults missing proof output to failure. The raw timing and +capacity artifact and evidence summary remain necessary: a neutral check +establishes neither the 90 ms budget nor release or installed acceptance. +Dependabot/fork token limitations remain explicit, not alternate credentials. +`TestChartSLOWorkflowPreservesInconclusiveAndFailure` executes the actual YAML +shells with a synthetic proof exit and a credential-free capturing provider, +covering success, neutral, budget failure and a broken proof. It checks the +read-only evidence/separate-writer admission and missing-output failure path. +No real check is created, workflow dispatched or latency workload executed. + ## Current State ### Unix installer certificate probing has a total deadline diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 6a9c1ed0ac..5d20c62e55 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -4582,3 +4582,18 @@ regression protects the same lazy owner across power/lease changes, and useDiskDetailModel protects stable History through qualification. Browser proof uses the real scoped HTTP reader and confirms native power changes require no extra disk read. No fleet CPU/RSS, installed or native recovery gain is claimed. + +### Hosted workload-chart observations keep three distinct outcomes + +The optional hosted job reports an invalidated measurement as inconclusive: +exit 78 is neutral, not a passed 90 ms budget. Valid capacity still requires a +single executed passing test at that budget. A contended FAIL can be neutral +only when its sole test diagnostic is the budget violation; skips, missing or +duplicate reports, wrong budgets, cleanup and child errors remain failures. +The private disposable-worker route continues to reject every nonzero exit, +including 78. Raw return codes, logs, digests and competing-capacity evidence +remain intact; invalid timings do not demonstrate absence of regression. +The registered API CLI and installtest workflow controls execute synthetic +children/telemetry and the actual evidence/writer shells, not a latency +workload. See `workload-chart-reporting-completion-2026-10-21.md` in `../records/` +for the source proposal, adverse governance result and available hosted sample. diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index a4f4546470..1a664d022b 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -8054,3 +8054,14 @@ Scoped inventory read failures and retry remain authoritative. GuestHardwareObse useDiskDetailModel and the read-only guest-hardware-observation.cjs protocol cover live qualification, source independence and same-open recovery. These synthetic consumer proofs do not establish native disk health, thaw or filesystem writes. + +### Capacity-invalid chart evidence is not storage recovery + +The hosted workload-chart proof may call a complete, contended measurement +inconclusive without clearing its raw budget failure. It must not turn a +cleanup/child error, absent report or skipped test into neutral evidence. +Neither neutrality nor synthetic CLI classification establishes a valid +History latency result, intact stored data, guest liveness or installed +recovery. `TestContractChartSLOProofCLIClassifiesExecution` preserves child +exit, output and executable identity across those distinctions; the 90 ms +valid-capacity requirement and independent native recovery remain unchanged. diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 001d808a66..b6bd7d85dc 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -15,6 +15,7 @@ import ( "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" "reflect" "regexp" @@ -79,6 +80,87 @@ import ( tmock "github.com/stretchr/testify/mock" ) +// Exercise the proof's real CLI, child execution and on-disk receipts without +// running a latency workload. Only capacity telemetry and sampling delays are +// synthetic; a malformed/failed child must never become a neutral budget result. +func TestContractChartSLOProofCLIClassifiesExecution(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "python3", "-c", ` +import hashlib, json, os, shlex, sys, tempfile +from pathlib import Path +sys.path.insert(0, str(Path('testdata').resolve())) +import chart_latency_proof as proof + +sleep = proof.time.sleep +proof.time.sleep = lambda seconds: sleep(0.005) +proof.wait_idle = lambda prefix: None +proof.snapshot = lambda: {} +proof.process_counters = lambda: {} +proof.ticks = lambda pid: 0 +proof.process_io = lambda pid: {} +os.environ['GITHUB_TOKEN'] = 'synthetic-must-not-reach-child' +report = ' slo_bench_test.go:637: charts/workloads p50=75ms p95=107ms p99=112ms SLO=90ms\n' +violation = ' slo_bench_test.go:637: SLO VIOLATION: p95=107ms exceeds target 90ms\n' +passed = report + '--- PASS: TestSLO_WorkloadCharts (0.03s)\nPASS\n' +failed = report + violation + '--- FAIL: TestSLO_WorkloadCharts (0.03s)\nFAIL\n' +cases = [ + ('idle-pass', False, passed, 0, 0), + ('idle-budget-failure', False, failed, 1, 1), + ('contended-pass', True, passed, 0, 78), + ('contended-budget-failure', True, failed, 1, 78), + ('contended-cleanup-error', True, failed.replace(violation, violation + + ' testing.go:1369: TempDir RemoveAll cleanup: directory not empty\n'), 1, 1), + ('contended-skip', True, report + '--- SKIP: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), + ('contended-process-error', True, failed, 2, 1), + ('contended-no-report', True, '--- PASS: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), + ('contended-wrong-budget', True, passed.replace('90ms', '550ms'), 0, 1), + ('contended-duplicate', True, passed + passed, 0, 1), +] +with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, contended, text, child_exit, expected in cases: + proof.interval = lambda before, after: dict(total_ticks=100, + busy_pct=25 if contended else 0, guest_pct=0, steal_pct=0, swap_pages=0) + binary = root / (name + '.sh') + binary.write_text('#!/bin/sh\n' + 'test -z "${GITHUB_TOKEN:-}" || exit 7\n' + 'test "$1" = "-test.run=^TestSLO_WorkloadCharts$" || exit 8\n' + 'sleep 0.03\nprintf %s ' + shlex.quote(text) + '\nexit ' + str(child_exit) + '\n') + binary.chmod(0o700) + output = root / name + summary = root / (name + '.summary') + os.environ['GITHUB_STEP_SUMMARY'] = str(summary) + sys.argv = ['chart_latency_proof.py', '--candidate', str(binary), '--output', str(output)] + observed = 0 + try: + proof.main() + except SystemExit as error: + observed = error.code + except RuntimeError: + observed = 1 + assert observed == expected, (name, observed, expected) + receipt = json.loads((output / 'candidate.result.json').read_text()) + assert receipt['returncode'] == child_exit, name + assert receipt['binary_sha256'] == hashlib.sha256(binary.read_bytes()).hexdigest(), name + assert receipt['environment']['PULSE_STRICT_LOCAL_API_SLOS'] == 'true', name + assert 'GITHUB_TOKEN' not in receipt['environment'], name + assert (output / 'candidate.log').read_text() == text, name + assert bool(receipt['capacity_errors']) == contended, name + assert receipt['command'][1:] == ['-test.run=^TestSLO_WorkloadCharts$', '-test.v', + '-test.count=1', '-test.cpu=2', '-test.timeout=5m'], name + assert summary.exists() == (expected == 78), name + if summary.exists(): + assert 'inconclusive' in summary.read_text() and '(not a verdict)' in summary.read_text(), name + print('CLASSIFIED', name, observed) +`) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("chart proof CLI fixture failed: %v\n%s", err, output) + } else { + t.Logf("synthetic CLI classification, not measured latency:\n%s", output) + } +} + func TestAgentConfigFetchAuditCapacityContract(t *testing.T) { capture := &auditCaptureLogger{} previousLogger, previousManager := audit.GetLogger(), GetTenantAuditManager() diff --git a/internal/api/testdata/chart_latency_proof.py b/internal/api/testdata/chart_latency_proof.py index 9b12bac4c9..188b3855cb 100644 --- a/internal/api/testdata/chart_latency_proof.py +++ b/internal/api/testdata/chart_latency_proof.py @@ -7,8 +7,13 @@ import os from pathlib import Path import re import subprocess +import sys import time +# A capacity-invalid proof is neither a pass nor a failure. The workflow maps +# this exit status to a neutral check conclusion. +INCONCLUSIVE_EXIT = 78 + def snapshot(): cpu = list(map(int, Path('/proc/stat').read_text().splitlines()[0].split()[1:])) @@ -121,15 +126,51 @@ def digest(path): return hashlib.file_digest(source, 'sha256').hexdigest() +def slo_report(text): + reports = re.findall(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=(\S+)', text) + if len(reports) != 1 or reports[0] != '90ms': + raise RuntimeError(f'Expected one explicit SLO report at the selected budget, got {reports}') + return re.search(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=\S+', text).group(0) + + def validate_output(text, returncode): if returncode != 0: raise RuntimeError(f'SLO process exited {returncode}') passes = re.findall(r'^--- PASS: TestSLO_WorkloadCharts \(', text, re.MULTILINE) if len(passes) != 1 or re.search(r'^--- SKIP: TestSLO_WorkloadCharts ', text, re.MULTILINE): raise RuntimeError('SLO did not pass, including a skipped or missing test') - reports = re.findall(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=(\S+)', text) - if len(reports) != 1 or reports[0] != '90ms': - raise RuntimeError(f'Expected one explicit SLO report at the selected budget, got {reports}') + slo_report(text) + + +def validate_contended_output(text, returncode): + # Contention only voids the latency verdict. The test must still have + # executed once and reported, so a crash, skip or broken fixture fails. + report = slo_report(text) + outcome = re.findall(r'^--- (PASS|FAIL|SKIP): TestSLO_WorkloadCharts \(', text, re.MULTILINE) + diagnostics = re.findall(r'^\s+\S+\.go:\d+: (.*)$', text, re.MULTILINE) + violations = [line for line in diagnostics if line.startswith('SLO VIOLATION: p95=')] + others = [line for line in diagnostics + if line not in violations and not line.startswith('charts/workloads p50=')] + budget_only = (outcome == ['FAIL'] and returncode == 1 + and len(violations) == 1 and not others) + if not (outcome == ['PASS'] and returncode == 0) and not budget_only: + raise RuntimeError(f'SLO process exited {returncode} with outcome {outcome}') + return report + + +def report_inconclusive(report, capacity_errors, summary_path): + names = sorted({row['name'] for error in capacity_errors + for row in error['background_processes'] if row['cpu_pct'] > 2}) + peak = max(error['background_busy_pct'] for error in capacity_errors) + lines = ['Workload chart 90ms SLO: inconclusive', + f'{report} (not a verdict)', + f'{len(capacity_errors)} measurement intervals had competing capacity ' + f'(peak {peak:.1f}% background CPU; processes: {", ".join(names) or "none named"}).', + 'The hosted runner was not idle, so this result neither passes nor fails the budget.'] + print('\n'.join(lines), flush=True) + if summary_path: + with open(summary_path, 'a') as summary: + summary.write('\n\n'.join(lines) + '\n') def measure(binary, prefix): @@ -174,17 +215,21 @@ def measure(binary, prefix): after = snapshot() receipt.write(json.dumps(interval(before, after)) + '\n') before = after + verdict = 'inconclusive' if capacity_errors else 'valid' result = dict(command=command, binary_sha256=identity, environment=env, - pid=process.pid, returncode=returncode, capacity_errors=capacity_errors) + pid=process.pid, returncode=returncode, verdict=verdict, + capacity_errors=capacity_errors) prefix.with_suffix('.result.json').write_text(json.dumps(result, indent=2) + '\n') text = prefix.with_suffix('.log').read_text() print(text, flush=True) if digest(binary) != identity: raise RuntimeError('Measured executable changed') if capacity_errors: - raise RuntimeError('Background CPU or guest/steal activity invalidated the idle measurement ' - f'(SLO process exit {returncode}; see raw log)') + report = validate_contended_output(text, returncode) + report_inconclusive(report, capacity_errors, os.environ.get('GITHUB_STEP_SUMMARY')) + return verdict validate_output(text, returncode) + return verdict def main(): @@ -195,7 +240,8 @@ def main(): args.output.mkdir(parents=True, exist_ok=False) # Absolute qualification executes the candidate directly. The separate # paired benchmark job owns relative comparisons and baseline context. - measure(args.candidate.resolve(), args.output / 'candidate') + if measure(args.candidate.resolve(), args.output / 'candidate') == 'inconclusive': + sys.exit(INCONCLUSIVE_EXIT) if __name__ == '__main__': diff --git a/internal/api/testdata/test_chart_latency_proof.py b/internal/api/testdata/test_chart_latency_proof.py index 5998c0cec5..04a1cb9235 100644 --- a/internal/api/testdata/test_chart_latency_proof.py +++ b/internal/api/testdata/test_chart_latency_proof.py @@ -1,4 +1,6 @@ +import tempfile import unittest +from pathlib import Path import chart_latency_proof as proof @@ -34,6 +36,43 @@ class ChartLatencyProofTests(unittest.TestCase): with self.assertRaises(RuntimeError): proof.validate_output(text, status) + def test_contention_voids_only_the_latency_verdict(self): + report = ' slo_bench_test.go:637: charts/workloads p50=75ms p95=107ms p99=112ms SLO=90ms\n' + violation = ' slo_bench_test.go:637: SLO VIOLATION: p95=107ms exceeds target 90ms\n' + passed = report + '--- PASS: TestSLO_WorkloadCharts (1.0s)\n' + failed = report + violation + '--- FAIL: TestSLO_WorkloadCharts (1.0s)\nFAIL\n' + for text, status in [(passed, 0), (failed, 1)]: + with self.subTest(text=text): + self.assertEqual(proof.validate_contended_output(text, status), + report.split(': ', 1)[1].strip()) + cleanup = ' testing.go:1369: TempDir RemoveAll cleanup: directory not empty\n' + for text, status in [(report + '--- SKIP: TestSLO_WorkloadCharts (1.0s)\n', 0), + (failed.replace(violation, ''), 1), (failed, 2), (passed, 1), + (failed.replace(violation, violation + cleanup), 1), + (failed.replace(violation, violation + violation), 1), + ('--- FAIL: TestSLO_WorkloadCharts (1.0s)\n', 1), + (passed.replace('90ms', '550ms'), 0), (passed + passed, 0)]: + with self.subTest(text=text, status=status): + with self.assertRaises(RuntimeError): + proof.validate_contended_output(text, status) + + def test_inconclusive_summary_names_competing_processes(self): + errors = [dict(background_busy_pct=25.2, background_processes=[ + dict(pid=1, name='provjobd312033', cpu_pct=25.2), + dict(pid=2, name='python3', cpu_pct=0.2)]), + dict(background_busy_pct=14.4, background_processes=[ + dict(pid=1, name='provjobd312033', cpu_pct=14.2)])] + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / 'summary.md' + proof.report_inconclusive('charts/workloads p50=1ms p95=2ms p99=3ms SLO=90ms', + errors, summary) + text = summary.read_text() + self.assertIn('inconclusive', text) + self.assertIn('2 measurement intervals', text) + self.assertIn('peak 25.2%', text) + self.assertIn('provjobd312033', text) + self.assertNotIn('python3', text) + def test_measurement_io_stays_in_latency_and_background_cpu_invalidates(self): row = dict(background_busy_pct=0, guest_pct=0, steal_pct=0, swap_pages=0, iowait_pct=20) diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index b43493e60a..4fb293e103 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -22,6 +22,118 @@ import ( "gopkg.in/yaml.v3" ) +// Run the maintained evidence/writer shells with a fake proof process and a +// capturing, credential-free provider. This protects the actual exit mapping +// and token separation without creating a check or executing a timed workload. +func TestChartSLOWorkflowPreservesInconclusiveAndFailure(t *testing.T) { + content, err := os.ReadFile(repoFile(".github", "workflows", "build-and-test.yml")) + if err != nil { + t.Fatal(err) + } + type step struct { + ID, Run, Uses string + Env map[string]string + } + var workflow struct { + Permissions map[string]string + Jobs map[string]struct { + Permissions map[string]string + Needs string + If string + Steps []step + } + } + if err := yaml.Unmarshal(content, &workflow); err != nil { + t.Fatal(err) + } + evidence := workflow.Jobs["chart-latency-slo"] + writer := workflow.Jobs["chart-latency-verdict"] + if len(workflow.Permissions) != 1 || workflow.Permissions["contents"] != "read" || len(evidence.Permissions) != 0 { + t.Fatal("PR-built executables must keep the read-only workflow token") + } + if len(writer.Permissions) != 1 || writer.Permissions["checks"] != "write" || writer.Needs != "chart-latency-slo" || + !strings.Contains(writer.If, "always()") || !strings.Contains(writer.If, "'failure'") || len(writer.Steps) != 1 || writer.Steps[0].Uses != "" { + t.Fatal("verdict writer must remain a separate no-checkout job, including failed evidence") + } + publish := writer.Steps[0] + if publish.Env["CONCLUSION"] != "${{ needs.chart-latency-slo.outputs.conclusion || 'failure' }}" || + publish.Env["HEAD_SHA"] != "${{ github.event.pull_request.head.sha }}" { + t.Fatal("missing proof output must fail at the exact PR head") + } + proofShell := "" + for _, candidate := range evidence.Steps { + if candidate.ID == "proof" { + proofShell = candidate.Run + } + } + if proofShell == "" || publish.Run == "" || !strings.Contains(proofShell, "chart_latency_proof.py") { + t.Fatal("missing actual chart proof/writer shell") + } + for _, tc := range []struct { + name, conclusion string + processExit int + stepPass bool + }{ + {"valid", "success", 0, true}, + {"contended", "neutral", 78, true}, + {"budget failure", "failure", 1, false}, + {"broken proof", "failure", 2, false}, + } { + t.Run(tc.name, func(t *testing.T) { + directory := t.TempDir() + bin := filepath.Join(directory, "bin") + samples := filepath.Join(directory, "chart-latency", "samples") + for _, path := range []string{bin, samples} { + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + } + for name, script := range map[string]string{ + "python3": "#!/bin/sh\nexit \"$CHART_PROOF_STATUS\"\n", + "gh": "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$CAPTURE_ARGS\"\n", + } { + if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0700); err != nil { + t.Fatal(err) + } + } + const report = "charts/workloads p50=40ms p95=82ms p99=86ms SLO=90ms" + if err := os.WriteFile(filepath.Join(samples, "candidate.log"), []byte(report+"\n"), 0600); err != nil { + t.Fatal(err) + } + outputs := filepath.Join(directory, "outputs") + summary := filepath.Join(directory, "summary") + capture := filepath.Join(directory, "provider-args") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", proofShell) + command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), + "RUNNER_TEMP="+directory, "GITHUB_OUTPUT="+outputs, "GITHUB_STEP_SUMMARY="+summary, + "CHART_PROOF_STATUS="+strconv.Itoa(tc.processExit)) + if output, err := command.CombinedOutput(); (err == nil) != tc.stepPass { + t.Fatalf("evidence shell status: %v\n%s", err, output) + } + got, err := os.ReadFile(outputs) + if err != nil || string(got) != "conclusion="+tc.conclusion+"\nreport="+report+"\n" { + t.Fatalf("wrong downstream evidence: %q, %v", got, err) + } + command = exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", publish.Run) + const head = "1234567890123456789012345678901234567890" + command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), + "CONCLUSION="+tc.conclusion, "REPORT="+report, "CAPTURE_ARGS="+capture, "HEAD_SHA="+head, + "GITHUB_REPOSITORY=rcourtman/Pulse", "DETAILS_URL=https://github.com/rcourtman/Pulse/actions/runs/1", "GH_TOKEN=") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("writer shell: %v\n%s", err, output) + } + args, err := os.ReadFile(capture) + if err != nil || !strings.Contains(string(args), "api\nrepos/rcourtman/Pulse/check-runs\n") || + !strings.Contains(string(args), "head_sha="+head+"\n") || + !strings.Contains(string(args), "conclusion="+tc.conclusion+"\n") { + t.Fatalf("wrong captured check identity/verdict: %q, %v", args, err) + } + }) + } +} + func TestBuildReleaseUsesV6InstallScripts(t *testing.T) { content, err := os.ReadFile(repoFile("scripts", "build-release.sh")) if err != nil {