diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index cbd6b1a9d8..755d6eaeee 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -759,11 +759,14 @@ jobs: run: go build ./cmd/pulse chart-latency-slo: - name: Workload chart 90ms SLO + name: Workload chart SLO evidence needs: changes if: github.event_name == 'pull_request' && needs.changes.outputs.go_compile == 'true' runs-on: ubuntu-24.04 timeout-minutes: 25 + outputs: + conclusion: ${{ steps.proof.outputs.conclusion }} + report: ${{ steps.proof.outputs.report }} env: GOMAXPROCS: "2" steps: @@ -808,11 +811,34 @@ jobs: # This additional evidence job keeps the ordinary hosted envelope and # required source/release gates intact. A fresh VM must first demonstrate # idle capacity; the candidate then enforces the existing local 90 ms SLO. + # Competing capacity during the measurement (on hosted runners, the + # runner's own provjobd daemon in every observed run) voids the latency + # verdict: the proof exits 78 and the verdict check is neutral, not red. - name: Verify chart latency on observed idle capacity + id: proof run: | + set +e python3 internal/api/testdata/chart_latency_proof.py \ --candidate "$RUNNER_TEMP/chart-latency/candidate-api.test" \ --output "$RUNNER_TEMP/chart-latency/samples" + status=$? + set -e + case "$status" in + 0) conclusion=success ;; + 78) conclusion=neutral ;; + *) conclusion=failure ;; + esac + report="$(grep -o 'charts/workloads p50=[^ ]* p95=[^ ]* p99=[^ ]* SLO=[^ ]*' \ + "$RUNNER_TEMP/chart-latency/samples/candidate.log" 2>/dev/null | head -n 1 || true)" + { + echo "conclusion=$conclusion" + echo "report=$report" + } >> "$GITHUB_OUTPUT" + # The proof writes its own summary for an inconclusive run. + [ "$conclusion" = neutral ] || + printf 'Workload chart 90ms SLO: %s\n\n%s\n' "$conclusion" \ + "${report:-No SLO report was produced.}" >> "$GITHUB_STEP_SUMMARY" + [ "$status" -eq 0 ] || [ "$status" -eq 78 ] - name: Upload chart latency evidence if: always() @@ -825,6 +851,40 @@ jobs: if-no-files-found: warn retention-days: 7 + # A separate job holds the check-write token, so the evidence job that runs + # PR-built executables stays read-only. Dependabot's read-only token cannot + # create check runs; the evidence job summary still carries the verdict. + # An evidence job that failed before the proof ran publishes a failure. + chart-latency-verdict: + name: Publish chart SLO verdict + needs: chart-latency-slo + if: always() && (needs.chart-latency-slo.result == 'success' || needs.chart-latency-slo.result == 'failure') + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + checks: write + steps: + - name: Publish SLO verdict check + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CONCLUSION: ${{ needs.chart-latency-slo.outputs.conclusion || 'failure' }} + REPORT: ${{ needs.chart-latency-slo.outputs.report }} + DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + case "$CONCLUSION" in + success) title='p95 within 90 ms on an idle runner' ;; + neutral) title='Inconclusive: the runner was not idle during the measurement' ;; + *) title='Failed on an idle runner, or the proof could not run' ;; + esac + gh api "repos/$GITHUB_REPOSITORY/check-runs" \ + -f name='Workload chart 90ms SLO' -f head_sha="$HEAD_SHA" \ + -f status=completed -f conclusion="$CONCLUSION" -f details_url="$DETAILS_URL" \ + -f "output[title]=$title" \ + -f "output[summary]=${REPORT:-No SLO report was produced.} See the evidence job summary and chart-latency artifact." \ + > /dev/null || + echo "::warning title=Chart SLO verdict not published::This token cannot create check runs; the evidence job carries the verdict." + benchmarks: name: Benchmarks needs: changes diff --git a/docs/release-control/v6/internal/records/workload-chart-reporting-completion-2026-10-21.md b/docs/release-control/v6/internal/records/workload-chart-reporting-completion-2026-10-21.md new file mode 100644 index 0000000000..b29ff143cc --- /dev/null +++ b/docs/release-control/v6/internal/records/workload-chart-reporting-completion-2026-10-21.md @@ -0,0 +1,27 @@ +# Completing hosted workload-chart reporting, 21 October 2026 + +Original proposal: [Pulse PR 3095](https://github.com/rcourtman/Pulse/pull/3095), +Richard Courtman's shared `b31fabd778f64f4bc35d1f33438256a2b24ff443`. +The two Python proof files are preserved byte-for-byte; only its chart evidence +and verdict jobs are composed into the assigned maintained workflow. Other +workflow jobs and the current frontend/toolchain remain untouched. The shared +commit is not amended, and its unqualified historical timing conclusions are +not adopted as a regression verdict. + +[Governance job 119307595195](https://github.com/rcourtman/Pulse/actions/runs/39624922365/job/119307595195) +rejects missing dependent contracts and API/deployment verification. This +completion supplies those contracts and executed CLI/workflow regression +controls rather than waiving the guard or changing the latency threshold. +A broad composition was abandoned because it would pull unrelated frontend +and toolchain changes into this bounded repair. The original proposal remains +shared and unchanged; the credited canonical equivalent receives its own +independent review, containing CI and protected publication. + +The classifier distinguishes a valid 90 ms pass/fail from an invalidated +measurement. A neutral hosted check is not latency qualification, native +recovery, released availability or evidence that a regression is absent. +Synthetic child/telemetry and fake-provider controls establish classification, +raw-evidence preservation and permission separation only. No performance test, +database investigation, workflow retry or release operation is part of this +completion. The current hosted artifact, exact source proof, adverse results +and remaining delivery state are retained in Delivery's linked outcome. diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index fabcbb890c..f5088308d1 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -10210,3 +10210,15 @@ runs when they close, failed runs included, and publishes the action plans the inbox expiry sweep expires, enrols, commands, updates or removes no agent and touches no agent identity or credential. Investigations still reach hosts only through the read-only evidence tools and the governed action lifecycle. + +### Hosted chart evidence cannot qualify an agent lifecycle + +The optional hosted workload-chart capacity proof reports an executed but +contended measurement as inconclusive, never as a healthy reading or a passed +90 ms budget. This CI-only classification does not satisfy agent installation, +restart, confinement, credential persistence or signed recovery acceptance. +Skips, missing reports and non-budget child failures still fail the proof. +The actual CLI/receipt control is +`TestContractChartSLOProofCLIClassifiesExecution`; the separate read-only +evidence and check-writer boundary is +`TestChartSLOWorkflowPreservesInconclusiveAndFailure`. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 05293a1f2e..bc80368ff6 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -12563,3 +12563,17 @@ changes. Proofs: `internal/actionlifecycle/service_test.go` `TestRefreshedActionIDNamesTheReplacedAction`), `internal/api/patrol_action_expiry_sweep_test.go` and `internal/api/ai_handlers_investigation_finalize_test.go`. + +### Workload-chart proof distinguishes an invalid measurement from recovery + +`internal/api/testdata/chart_latency_proof.py` keeps the strict 90 ms budget, +exact one-test execution, executable digest and child result in its receipt. +Only a single complete PASS or budget-only FAIL with competing capacity can +exit 78 and describe the timing as inconclusive. A skipped/missing/duplicate +report, wrong budget, cleanup failure or other child failure remains an error. +A valid-capacity budget failure still fails. No API payload, route, History +sample, incident state or operational-recovery result is changed by this CI +classification. `TestContractChartSLOProofCLIClassifiesExecution` drives the +real CLI and disposable child processes through all ten outcomes, verifies +unaltered logs/result/digest/strict arguments and excludes the synthetic token +from the child environment. Its capacity rows are synthetic, not latency proof. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index a8d4d6f5a9..702deeab2b 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -3764,6 +3764,22 @@ artifact-selection behaviour. ## Current State +### Hosted chart verdict writer has no proposed executable + +Build and Test's optional workload-chart evidence job retains a read-only +repository token while running the PR-built executable. Its proof maps only +exit 0 to success and 78 to neutral; any other exit fails. The separate check +writer has only `checks: write`, no checkout or proposed executable, uses the +exact PR head and defaults missing proof output to failure. The raw timing and +capacity artifact and evidence summary remain necessary: a neutral check +establishes neither the 90 ms budget nor release or installed acceptance. +Dependabot/fork token limitations remain explicit, not alternate credentials. +`TestChartSLOWorkflowPreservesInconclusiveAndFailure` executes the actual YAML +shells with a synthetic proof exit and a credential-free capturing provider, +covering success, neutral, budget failure and a broken proof. It checks the +read-only evidence/separate-writer admission and missing-output failure path. +No real check is created, workflow dispatched or latency workload executed. + ### Registered Windows native-exit completion proof (8 October 2026) The workflow's seven immediate native exit checks and the lifecycle harness's diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 4072af7969..45c155a84b 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -4412,3 +4412,18 @@ handler in `useWorkloadsControlsState.ts` no longer checks for a focused chat composer. `.bundlesize.json` drops the retired chunks; `useUnifiedResources` grew to about 10 kB gzipped because Rollup moved the shared filter toolbar modules into it once the `Chat` chunk stopped sharing them. + +### Hosted workload-chart observations keep three distinct outcomes + +The optional hosted job reports an invalidated measurement as inconclusive: +exit 78 is neutral, not a passed 90 ms budget. Valid capacity still requires a +single executed passing test at that budget. A contended FAIL can be neutral +only when its sole test diagnostic is the budget violation; skips, missing or +duplicate reports, wrong budgets, cleanup and child errors remain failures. +The private disposable-worker route continues to reject every nonzero exit, +including 78. Raw return codes, logs, digests and competing-capacity evidence +remain intact; invalid timings do not demonstrate absence of regression. +The registered API CLI and installtest workflow controls execute synthetic +children/telemetry and the actual evidence/writer shells, not a latency +workload. See `workload-chart-reporting-completion-2026-10-21.md` in `../records/` +for the source proposal, adverse governance result and available hosted sample. diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 4c18b15efa..577269a91c 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -7198,3 +7198,14 @@ The `internal/api` change of 2026-10-23 that finalizes Patrol investigation runs when they close and publishes the action plans the inbox expiry sweep expires opens no storage, backup or recovery path. Recovery evidence, restore rules and storage capabilities are unchanged. + +### Capacity-invalid chart evidence is not storage recovery + +The hosted workload-chart proof may call a complete, contended measurement +inconclusive without clearing its raw budget failure. It must not turn a +cleanup/child error, absent report or skipped test into neutral evidence. +Neither neutrality nor synthetic CLI classification establishes a valid +History latency result, intact stored data, guest liveness or installed +recovery. `TestContractChartSLOProofCLIClassifiesExecution` preserves child +exit, output and executable identity across those distinctions; the 90 ms +valid-capacity requirement and independent native recovery remain unchanged. diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index e85797cd45..c7b60cdd9a 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -15,6 +15,7 @@ import ( "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" "reflect" "regexp" @@ -79,6 +80,87 @@ import ( tmock "github.com/stretchr/testify/mock" ) +// Exercise the proof's real CLI, child execution and on-disk receipts without +// running a latency workload. Only capacity telemetry and sampling delays are +// synthetic; a malformed/failed child must never become a neutral budget result. +func TestContractChartSLOProofCLIClassifiesExecution(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "python3", "-c", ` +import hashlib, json, os, shlex, sys, tempfile +from pathlib import Path +sys.path.insert(0, str(Path('testdata').resolve())) +import chart_latency_proof as proof + +sleep = proof.time.sleep +proof.time.sleep = lambda seconds: sleep(0.005) +proof.wait_idle = lambda prefix: None +proof.snapshot = lambda: {} +proof.process_counters = lambda: {} +proof.ticks = lambda pid: 0 +proof.process_io = lambda pid: {} +os.environ['GITHUB_TOKEN'] = 'synthetic-must-not-reach-child' +report = ' slo_bench_test.go:637: charts/workloads p50=75ms p95=107ms p99=112ms SLO=90ms\n' +violation = ' slo_bench_test.go:637: SLO VIOLATION: p95=107ms exceeds target 90ms\n' +passed = report.replace('p95=107ms p99=112ms', 'p95=82ms p99=86ms') + '--- PASS: TestSLO_WorkloadCharts (0.03s)\nPASS\n' +failed = report + violation + '--- FAIL: TestSLO_WorkloadCharts (0.03s)\nFAIL\n' +cases = [ + ('idle-pass', False, passed, 0, 0), + ('idle-budget-failure', False, failed, 1, 1), + ('contended-pass', True, passed, 0, 78), + ('contended-budget-failure', True, failed, 1, 78), + ('contended-cleanup-error', True, failed.replace(violation, violation + + ' testing.go:1369: TempDir RemoveAll cleanup: directory not empty\n'), 1, 1), + ('contended-skip', True, report + '--- SKIP: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), + ('contended-process-error', True, failed, 2, 1), + ('contended-no-report', True, '--- PASS: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), + ('contended-wrong-budget', True, passed.replace('90ms', '550ms'), 0, 1), + ('contended-duplicate', True, passed + passed, 0, 1), +] +with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, contended, text, child_exit, expected in cases: + proof.interval = lambda before, after: dict(total_ticks=100, + busy_pct=25 if contended else 0, guest_pct=0, steal_pct=0, swap_pages=0) + binary = root / (name + '.sh') + binary.write_text('#!/bin/sh\n' + 'test -z "${GITHUB_TOKEN:-}" || exit 7\n' + 'test "$1" = "-test.run=^TestSLO_WorkloadCharts$" || exit 8\n' + 'sleep 0.03\nprintf %s ' + shlex.quote(text) + '\nexit ' + str(child_exit) + '\n') + binary.chmod(0o700) + output = root / name + summary = root / (name + '.summary') + os.environ['GITHUB_STEP_SUMMARY'] = str(summary) + sys.argv = ['chart_latency_proof.py', '--candidate', str(binary), '--output', str(output)] + observed = 0 + try: + proof.main() + except SystemExit as error: + observed = error.code + except RuntimeError: + observed = 1 + assert observed == expected, (name, observed, expected) + receipt = json.loads((output / 'candidate.result.json').read_text()) + assert receipt['returncode'] == child_exit, name + assert receipt['binary_sha256'] == hashlib.sha256(binary.read_bytes()).hexdigest(), name + assert receipt['environment']['PULSE_STRICT_LOCAL_API_SLOS'] == 'true', name + assert 'GITHUB_TOKEN' not in receipt['environment'], name + assert (output / 'candidate.log').read_text() == text, name + assert bool(receipt['capacity_errors']) == contended, name + assert receipt['command'][1:] == ['-test.run=^TestSLO_WorkloadCharts$', '-test.v', + '-test.count=1', '-test.cpu=2', '-test.timeout=5m'], name + assert summary.exists() == (expected == 78), name + if summary.exists(): + assert 'inconclusive' in summary.read_text() and '(not a verdict)' in summary.read_text(), name + print('CLASSIFIED', name, observed) +`) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("chart proof CLI fixture failed: %v\n%s", err, output) + } else { + t.Logf("synthetic CLI classification, not measured latency:\n%s", output) + } +} + func TestAgentConfigFetchAuditCapacityContract(t *testing.T) { capture := &auditCaptureLogger{} previousLogger, previousManager := audit.GetLogger(), GetTenantAuditManager() diff --git a/internal/api/testdata/chart_latency_proof.py b/internal/api/testdata/chart_latency_proof.py index 9b12bac4c9..188b3855cb 100644 --- a/internal/api/testdata/chart_latency_proof.py +++ b/internal/api/testdata/chart_latency_proof.py @@ -7,8 +7,13 @@ import os from pathlib import Path import re import subprocess +import sys import time +# A capacity-invalid proof is neither a pass nor a failure. The workflow maps +# this exit status to a neutral check conclusion. +INCONCLUSIVE_EXIT = 78 + def snapshot(): cpu = list(map(int, Path('/proc/stat').read_text().splitlines()[0].split()[1:])) @@ -121,15 +126,51 @@ def digest(path): return hashlib.file_digest(source, 'sha256').hexdigest() +def slo_report(text): + reports = re.findall(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=(\S+)', text) + if len(reports) != 1 or reports[0] != '90ms': + raise RuntimeError(f'Expected one explicit SLO report at the selected budget, got {reports}') + return re.search(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=\S+', text).group(0) + + def validate_output(text, returncode): if returncode != 0: raise RuntimeError(f'SLO process exited {returncode}') passes = re.findall(r'^--- PASS: TestSLO_WorkloadCharts \(', text, re.MULTILINE) if len(passes) != 1 or re.search(r'^--- SKIP: TestSLO_WorkloadCharts ', text, re.MULTILINE): raise RuntimeError('SLO did not pass, including a skipped or missing test') - reports = re.findall(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=(\S+)', text) - if len(reports) != 1 or reports[0] != '90ms': - raise RuntimeError(f'Expected one explicit SLO report at the selected budget, got {reports}') + slo_report(text) + + +def validate_contended_output(text, returncode): + # Contention only voids the latency verdict. The test must still have + # executed once and reported, so a crash, skip or broken fixture fails. + report = slo_report(text) + outcome = re.findall(r'^--- (PASS|FAIL|SKIP): TestSLO_WorkloadCharts \(', text, re.MULTILINE) + diagnostics = re.findall(r'^\s+\S+\.go:\d+: (.*)$', text, re.MULTILINE) + violations = [line for line in diagnostics if line.startswith('SLO VIOLATION: p95=')] + others = [line for line in diagnostics + if line not in violations and not line.startswith('charts/workloads p50=')] + budget_only = (outcome == ['FAIL'] and returncode == 1 + and len(violations) == 1 and not others) + if not (outcome == ['PASS'] and returncode == 0) and not budget_only: + raise RuntimeError(f'SLO process exited {returncode} with outcome {outcome}') + return report + + +def report_inconclusive(report, capacity_errors, summary_path): + names = sorted({row['name'] for error in capacity_errors + for row in error['background_processes'] if row['cpu_pct'] > 2}) + peak = max(error['background_busy_pct'] for error in capacity_errors) + lines = ['Workload chart 90ms SLO: inconclusive', + f'{report} (not a verdict)', + f'{len(capacity_errors)} measurement intervals had competing capacity ' + f'(peak {peak:.1f}% background CPU; processes: {", ".join(names) or "none named"}).', + 'The hosted runner was not idle, so this result neither passes nor fails the budget.'] + print('\n'.join(lines), flush=True) + if summary_path: + with open(summary_path, 'a') as summary: + summary.write('\n\n'.join(lines) + '\n') def measure(binary, prefix): @@ -174,17 +215,21 @@ def measure(binary, prefix): after = snapshot() receipt.write(json.dumps(interval(before, after)) + '\n') before = after + verdict = 'inconclusive' if capacity_errors else 'valid' result = dict(command=command, binary_sha256=identity, environment=env, - pid=process.pid, returncode=returncode, capacity_errors=capacity_errors) + pid=process.pid, returncode=returncode, verdict=verdict, + capacity_errors=capacity_errors) prefix.with_suffix('.result.json').write_text(json.dumps(result, indent=2) + '\n') text = prefix.with_suffix('.log').read_text() print(text, flush=True) if digest(binary) != identity: raise RuntimeError('Measured executable changed') if capacity_errors: - raise RuntimeError('Background CPU or guest/steal activity invalidated the idle measurement ' - f'(SLO process exit {returncode}; see raw log)') + report = validate_contended_output(text, returncode) + report_inconclusive(report, capacity_errors, os.environ.get('GITHUB_STEP_SUMMARY')) + return verdict validate_output(text, returncode) + return verdict def main(): @@ -195,7 +240,8 @@ def main(): args.output.mkdir(parents=True, exist_ok=False) # Absolute qualification executes the candidate directly. The separate # paired benchmark job owns relative comparisons and baseline context. - measure(args.candidate.resolve(), args.output / 'candidate') + if measure(args.candidate.resolve(), args.output / 'candidate') == 'inconclusive': + sys.exit(INCONCLUSIVE_EXIT) if __name__ == '__main__': diff --git a/internal/api/testdata/test_chart_latency_proof.py b/internal/api/testdata/test_chart_latency_proof.py index 5998c0cec5..04a1cb9235 100644 --- a/internal/api/testdata/test_chart_latency_proof.py +++ b/internal/api/testdata/test_chart_latency_proof.py @@ -1,4 +1,6 @@ +import tempfile import unittest +from pathlib import Path import chart_latency_proof as proof @@ -34,6 +36,43 @@ class ChartLatencyProofTests(unittest.TestCase): with self.assertRaises(RuntimeError): proof.validate_output(text, status) + def test_contention_voids_only_the_latency_verdict(self): + report = ' slo_bench_test.go:637: charts/workloads p50=75ms p95=107ms p99=112ms SLO=90ms\n' + violation = ' slo_bench_test.go:637: SLO VIOLATION: p95=107ms exceeds target 90ms\n' + passed = report + '--- PASS: TestSLO_WorkloadCharts (1.0s)\n' + failed = report + violation + '--- FAIL: TestSLO_WorkloadCharts (1.0s)\nFAIL\n' + for text, status in [(passed, 0), (failed, 1)]: + with self.subTest(text=text): + self.assertEqual(proof.validate_contended_output(text, status), + report.split(': ', 1)[1].strip()) + cleanup = ' testing.go:1369: TempDir RemoveAll cleanup: directory not empty\n' + for text, status in [(report + '--- SKIP: TestSLO_WorkloadCharts (1.0s)\n', 0), + (failed.replace(violation, ''), 1), (failed, 2), (passed, 1), + (failed.replace(violation, violation + cleanup), 1), + (failed.replace(violation, violation + violation), 1), + ('--- FAIL: TestSLO_WorkloadCharts (1.0s)\n', 1), + (passed.replace('90ms', '550ms'), 0), (passed + passed, 0)]: + with self.subTest(text=text, status=status): + with self.assertRaises(RuntimeError): + proof.validate_contended_output(text, status) + + def test_inconclusive_summary_names_competing_processes(self): + errors = [dict(background_busy_pct=25.2, background_processes=[ + dict(pid=1, name='provjobd312033', cpu_pct=25.2), + dict(pid=2, name='python3', cpu_pct=0.2)]), + dict(background_busy_pct=14.4, background_processes=[ + dict(pid=1, name='provjobd312033', cpu_pct=14.2)])] + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / 'summary.md' + proof.report_inconclusive('charts/workloads p50=1ms p95=2ms p99=3ms SLO=90ms', + errors, summary) + text = summary.read_text() + self.assertIn('inconclusive', text) + self.assertIn('2 measurement intervals', text) + self.assertIn('peak 25.2%', text) + self.assertIn('provjobd312033', text) + self.assertNotIn('python3', text) + def test_measurement_io_stays_in_latency_and_background_cpu_invalidates(self): row = dict(background_busy_pct=0, guest_pct=0, steal_pct=0, swap_pages=0, iowait_pct=20) diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 1646cd2463..d5cea47d59 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -22,6 +22,118 @@ import ( "gopkg.in/yaml.v3" ) +// Run the maintained evidence/writer shells with a fake proof process and a +// capturing, credential-free provider. This protects the actual exit mapping +// and token separation without creating a check or executing a timed workload. +func TestChartSLOWorkflowPreservesInconclusiveAndFailure(t *testing.T) { + content, err := os.ReadFile(repoFile(".github", "workflows", "build-and-test.yml")) + if err != nil { + t.Fatal(err) + } + type step struct { + ID, Run, Uses string + Env map[string]string + } + var workflow struct { + Permissions map[string]string + Jobs map[string]struct { + Permissions map[string]string + Needs any + If string + Steps []step + } + } + if err := yaml.Unmarshal(content, &workflow); err != nil { + t.Fatal(err) + } + evidence := workflow.Jobs["chart-latency-slo"] + writer := workflow.Jobs["chart-latency-verdict"] + if len(workflow.Permissions) != 1 || workflow.Permissions["contents"] != "read" || len(evidence.Permissions) != 0 { + t.Fatal("PR-built executables must keep the read-only workflow token") + } + if len(writer.Permissions) != 1 || writer.Permissions["checks"] != "write" || writer.Needs != "chart-latency-slo" || + !strings.Contains(writer.If, "always()") || !strings.Contains(writer.If, "'failure'") || len(writer.Steps) != 1 || writer.Steps[0].Uses != "" { + t.Fatal("verdict writer must remain a separate no-checkout job, including failed evidence") + } + publish := writer.Steps[0] + if publish.Env["CONCLUSION"] != "${{ needs.chart-latency-slo.outputs.conclusion || 'failure' }}" || + publish.Env["HEAD_SHA"] != "${{ github.event.pull_request.head.sha }}" { + t.Fatal("missing proof output must fail at the exact PR head") + } + proofShell := "" + for _, candidate := range evidence.Steps { + if candidate.ID == "proof" { + proofShell = candidate.Run + } + } + if proofShell == "" || publish.Run == "" || !strings.Contains(proofShell, "chart_latency_proof.py") { + t.Fatal("missing actual chart proof/writer shell") + } + for _, tc := range []struct { + name, conclusion string + processExit int + stepPass bool + }{ + {"valid", "success", 0, true}, + {"contended", "neutral", 78, true}, + {"budget failure", "failure", 1, false}, + {"broken proof", "failure", 2, false}, + } { + t.Run(tc.name, func(t *testing.T) { + directory := t.TempDir() + bin := filepath.Join(directory, "bin") + samples := filepath.Join(directory, "chart-latency", "samples") + for _, path := range []string{bin, samples} { + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + } + for name, script := range map[string]string{ + "python3": "#!/bin/sh\nexit \"$CHART_PROOF_STATUS\"\n", + "gh": "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$CAPTURE_ARGS\"\n", + } { + if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0700); err != nil { + t.Fatal(err) + } + } + const report = "charts/workloads p50=40ms p95=82ms p99=86ms SLO=90ms" + if err := os.WriteFile(filepath.Join(samples, "candidate.log"), []byte(report+"\n"), 0600); err != nil { + t.Fatal(err) + } + outputs := filepath.Join(directory, "outputs") + summary := filepath.Join(directory, "summary") + capture := filepath.Join(directory, "provider-args") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", proofShell) + command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), + "RUNNER_TEMP="+directory, "GITHUB_OUTPUT="+outputs, "GITHUB_STEP_SUMMARY="+summary, + "CHART_PROOF_STATUS="+strconv.Itoa(tc.processExit)) + if output, err := command.CombinedOutput(); (err == nil) != tc.stepPass { + t.Fatalf("evidence shell status: %v\n%s", err, output) + } + got, err := os.ReadFile(outputs) + if err != nil || string(got) != "conclusion="+tc.conclusion+"\nreport="+report+"\n" { + t.Fatalf("wrong downstream evidence: %q, %v", got, err) + } + command = exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", publish.Run) + const head = "1234567890123456789012345678901234567890" + command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), + "CONCLUSION="+tc.conclusion, "REPORT="+report, "CAPTURE_ARGS="+capture, "HEAD_SHA="+head, + "GITHUB_REPOSITORY=rcourtman/Pulse", "DETAILS_URL=https://github.com/rcourtman/Pulse/actions/runs/1", "GH_TOKEN=") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("writer shell: %v\n%s", err, output) + } + args, err := os.ReadFile(capture) + if err != nil || !strings.Contains(string(args), "api\nrepos/rcourtman/Pulse/check-runs\n") || + !strings.Contains(string(args), "head_sha="+head+"\n") || + !strings.Contains(string(args), "conclusion="+tc.conclusion+"\n") { + t.Fatalf("wrong captured check identity/verdict: %q, %v", args, err) + } + }) + } +} + func TestBuildReleaseUsesV6InstallScripts(t *testing.T) { content, err := os.ReadFile(repoFile("scripts", "build-release.sh")) if err != nil {