diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 755d6eaeee..ed2fa7e0a7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -811,9 +811,9 @@ jobs: # This additional evidence job keeps the ordinary hosted envelope and # required source/release gates intact. A fresh VM must first demonstrate # idle capacity; the candidate then enforces the existing local 90 ms SLO. - # Competing capacity during the measurement (on hosted runners, the - # runner's own provjobd daemon in every observed run) voids the latency - # verdict: the proof exits 78 and the verdict check is neutral, not red. + # Bounded single-CPU hosted provisioning overhead stays in the measured + # 90 ms outcome. Other competing CPU, guest/steal or swap activity voids + # the verdict: exit 78 is neutral, never a passed budget. - name: Verify chart latency on observed idle capacity id: proof run: | diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 51204cfd20..72d0ae8ce8 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -10278,3 +10278,8 @@ The actual CLI/receipt control is `TestContractChartSLOProofCLIClassifiesExecution`; the separate read-only evidence and check-writer boundary is `TestChartSLOWorkflowPreservesInconclusiveAndFailure`. +Bounded hosted provisioning-agent CPU is now attributed separately, retaining +raw observations and enforcing the same 90 ms budget with that overhead present. +An over-budget run in this normal hosted envelope fails rather than disappearing +into neutral. Other/unattributed CPU and guest/steal/swap still invalidate the +timing. Neither verdict supplies any of the independent lifecycle proofs above. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 739388fdc4..c3540f1f41 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -12636,6 +12636,14 @@ report, wrong budget, cleanup failure or other child failure remains an error. A valid-capacity budget failure still fails. No API payload, route, History sample, incident state or operational-recovery result is changed by this CI classification. `TestContractChartSLOProofCLIClassifiesExecution` drives the -real CLI and disposable child processes through all ten outcomes, verifies +real CLI and disposable child processes through idle, hosted-agent and contended outcomes, verifies unaltered logs/result/digest/strict arguments and excludes the synthetic token from the child environment. Its capacity rows are synthetic, not latency proof. +On actual Actions execution, one exactly named `provjobd` process may +account for at most one logical CPU plus one percentage point of sampling skew. +Only that bounded contribution is separated from competing CPU; all other +observed and unattributed CPU retains the 10% limit. Multiple or over-bound +agents, non-Actions execution, guest/steal and swap get no exception. Receipts +retain raw CPU rows plus runner-agent/competing attribution and CPU count. +The hosted overhead remains in the absolute latency: under-budget passes and +over-budget failures both receive real verdicts, without normalization. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 9445acc4b1..159bdfc2d7 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -3806,6 +3806,12 @@ shells with a synthetic proof exit and a credential-free capturing provider, covering success, neutral, budget failure and a broken proof. It checks the read-only evidence/separate-writer admission and missing-output failure path. No real check is created, workflow dispatched or latency workload executed. +The hosted envelope includes only the proof's bounded single-CPU provisioning +agent attribution, never arbitrary background work. Both its passing and +over-90-ms child results must reach the unchanged success/failure shell mapping; +only genuinely competing capacity can map a budget-only result to neutral. +The workflow-shell control preserves the actual over-budget report through +both neutral and failure paths. This changes no release or install gate. ### Registered Windows native-exit completion proof (8 October 2026) diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 45c155a84b..3c113a3dc2 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1883,6 +1883,14 @@ dashboard-specific overview, trend, or summary transport. its latency, with I/O wait and process I/O counters retained for diagnosis. Competing CPU, guest/steal or swap activity invalidates the measurement, and process CPU counters identify the largest observed background users. + On Actions, exactly one `provjobd` process bounded to one logical + CPU plus one percentage point of sampling skew is normal hosted overhead. + Keep it inside the absolute latency, while retaining its raw CPU and a + separate attribution. Apply the unchanged 10% limit to all remaining + observed/unattributed CPU; multiple/over-bound agents or missing hosted/CPU + context get no exception. Guest/steal and swap limits remain unconditional. + Real budget failures on this envelope must fail, not become neutral; + historical classifications are evidence, not retroactive qualification. Only a passing explicit 90 ms report on valid capacity establishes the local SLO. The manual `pulse-enterprise:.github/workflows/verify-workload-chart-latency.yml` reuses that same public proof on the existing disposable `pulse-ci-vm-large` diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index c066d21d66..8c2017e2ab 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -7249,3 +7249,8 @@ History latency result, intact stored data, guest liveness or installed recovery. `TestContractChartSLOProofCLIClassifiesExecution` preserves child exit, output and executable identity across those distinctions; the 90 ms valid-capacity requirement and independent native recovery remain unchanged. +One bounded hosted provisioning agent is part of the absolute measured envelope, +not a blanket contention exemption: other/unattributed CPU, guest/steal and swap +remain invalidating. Raw CPU and agent attribution are both retained. This +permits a real failure as well as a pass at 90 ms; it clears no earlier adverse +timing, storage failure or excluded investigation and changes no store code. diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index acaebe5486..3995ef1bc1 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -100,29 +100,42 @@ proof.snapshot = lambda: {} proof.process_counters = lambda: {} proof.ticks = lambda pid: 0 proof.process_io = lambda pid: {} +proof.os.cpu_count = lambda: 4 +os.environ['GITHUB_ACTIONS'] = 'true' os.environ['GITHUB_TOKEN'] = 'synthetic-must-not-reach-child' report = ' slo_bench_test.go:637: charts/workloads p50=75ms p95=107ms p99=112ms SLO=90ms\n' violation = ' slo_bench_test.go:637: SLO VIOLATION: p95=107ms exceeds target 90ms\n' passed = report.replace('p95=107ms p99=112ms', 'p95=82ms p99=86ms') + '--- PASS: TestSLO_WorkloadCharts (0.03s)\nPASS\n' failed = report + violation + '--- FAIL: TestSLO_WorkloadCharts (0.03s)\nFAIL\n' cases = [ - ('idle-pass', False, passed, 0, 0), - ('idle-budget-failure', False, failed, 1, 1), - ('contended-pass', True, passed, 0, 78), - ('contended-budget-failure', True, failed, 1, 78), - ('contended-cleanup-error', True, failed.replace(violation, violation + + ('idle-pass', 'idle', passed, 0, 0), + ('idle-budget-failure', 'idle', failed, 1, 1), + ('runner-agent-pass', 'agent', passed, 0, 0), + ('runner-agent-budget-failure', 'agent', failed, 1, 1), + ('runner-agent-with-other-cpu', 'mixed', passed, 0, 78), + ('runner-agent-with-steal', 'steal', passed, 0, 78), + ('contended-pass', 'other', passed, 0, 78), + ('contended-budget-failure', 'other', failed, 1, 78), + ('contended-cleanup-error', 'other', failed.replace(violation, violation + ' testing.go:1369: TempDir RemoveAll cleanup: directory not empty\n'), 1, 1), - ('contended-skip', True, report + '--- SKIP: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), - ('contended-process-error', True, failed, 2, 1), - ('contended-no-report', True, '--- PASS: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), - ('contended-wrong-budget', True, passed.replace('90ms', '550ms'), 0, 1), - ('contended-duplicate', True, passed + passed, 0, 1), + ('contended-skip', 'other', report + '--- SKIP: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), + ('contended-process-error', 'other', failed, 2, 1), + ('contended-no-report', 'other', '--- PASS: TestSLO_WorkloadCharts (0.03s)\n', 0, 1), + ('contended-wrong-budget', 'other', passed.replace('90ms', '550ms'), 0, 1), + ('contended-duplicate', 'other', passed + passed, 0, 1), ] with tempfile.TemporaryDirectory() as directory: root = Path(directory) - for name, contended, text, child_exit, expected in cases: + for name, capacity, text, child_exit, expected in cases: + contended = capacity in ('other', 'mixed', 'steal') + agent = dict(pid=1, name='provjobd1892260', cpu_pct=25) + other = dict(pid=2, name='node', cpu_pct=11) + processes = {'idle': [], 'agent': [agent], 'other': [other], + 'mixed': [agent, other], 'steal': [agent]}[capacity] + proof.background_processes = lambda *args: processes proof.interval = lambda before, after: dict(total_ticks=100, - busy_pct=25 if contended else 0, guest_pct=0, steal_pct=0, swap_pages=0) + busy_pct=sum(row['cpu_pct'] for row in processes), guest_pct=0, + steal_pct=1 if capacity == 'steal' else 0, swap_pages=0) binary = root / (name + '.sh') binary.write_text('#!/bin/sh\n' 'test -z "${GITHUB_TOKEN:-}" || exit 7\n' @@ -145,9 +158,15 @@ with tempfile.TemporaryDirectory() as directory: assert receipt['returncode'] == child_exit, name assert receipt['binary_sha256'] == hashlib.sha256(binary.read_bytes()).hexdigest(), name assert receipt['environment']['PULSE_STRICT_LOCAL_API_SLOS'] == 'true', name + assert receipt['capacity_context'] == dict(hosted=True, cpu_count=4), name assert 'GITHUB_TOKEN' not in receipt['environment'], name assert (output / 'candidate.log').read_text() == text, name assert bool(receipt['capacity_errors']) == contended, name + rows = [json.loads(line) for line in (output / 'candidate.cpu.jsonl').read_text().splitlines()] + assert rows and all('runner_agent_cpu_pct' in row and 'competing_busy_pct' in row for row in rows), name + if capacity == 'agent': + assert all(row['background_busy_pct'] == 25 and row['runner_agent_cpu_pct'] == 25 + and row['competing_busy_pct'] == 0 for row in rows), name assert receipt['command'][1:] == ['-test.run=^TestSLO_WorkloadCharts$', '-test.v', '-test.count=1', '-test.cpu=2', '-test.timeout=5m'], name assert summary.exists() == (expected == 78), name diff --git a/internal/api/testdata/chart_latency_proof.py b/internal/api/testdata/chart_latency_proof.py index 188b3855cb..bcf51959dc 100644 --- a/internal/api/testdata/chart_latency_proof.py +++ b/internal/api/testdata/chart_latency_proof.py @@ -113,11 +113,36 @@ def process_io(pid): return None +def background_capacity(row, *, hosted=False, cpu_count=None): + # The two retained hosted samples (40077319330 and 40075384922) have + # opposite 90 ms outcomes, but the same single-CPU provjobd activity. + # Treat that bounded runner overhead as part of the hosted envelope, not + # a reason to discard either outcome. Never subtract it from the latency. + processes = row.get('background_processes', []) + agents = [process for process in processes + if re.fullmatch(r'provjobd[0-9]+', process['name'])] + agent = None + if hosted and cpu_count and cpu_count > 0 and len(agents) == 1: + # /proc/stat and per-process counters are sampled separately. Allow + # one percentage point of accounting skew above one logical CPU; + # multiple agents or activity beyond that bound get no exemption. + if 0 <= agents[0]['cpu_pct'] <= 100 / cpu_count + 1: + agent = agents[0] + agent_cpu = agent['cpu_pct'] if agent else 0 + # Keep unaccounted CPU (including exited processes and the top-five tail) + # and every other observed process. Attribution must not hide contention + # when the independent aggregate/process counters disagree. + competing = max(0, row['background_busy_pct'] - agent_cpu, + sum(process['cpu_pct'] for process in processes if process is not agent)) + return dict(runner_agent_cpu_pct=agent_cpu, + runner_agent_pid=agent['pid'] if agent else None, competing_busy_pct=competing) + + def competing_capacity(row): # The measured SQLite process performs I/O. Keep I/O wait in the raw # receipt and latency, rather than mistaking its own reads for contention. # Admission still requires quiet I/O before the process starts. - return (row['background_busy_pct'] > 10 or row['guest_pct'] > 2 + return (row.get('competing_busy_pct', row['background_busy_pct']) > 10 or row['guest_pct'] > 2 or row['steal_pct'] > 0.1 or row['swap_pages'] != 0) @@ -160,12 +185,14 @@ def validate_contended_output(text, returncode): def report_inconclusive(report, capacity_errors, summary_path): names = sorted({row['name'] for error in capacity_errors - for row in error['background_processes'] if row['cpu_pct'] > 2}) - peak = max(error['background_busy_pct'] for error in capacity_errors) + for row in error['background_processes'] + if row['cpu_pct'] > 2 and row['pid'] != error.get('runner_agent_pid')}) + peak = max(error.get('competing_busy_pct', error['background_busy_pct']) + for error in capacity_errors) lines = ['Workload chart 90ms SLO: inconclusive', f'{report} (not a verdict)', f'{len(capacity_errors)} measurement intervals had competing capacity ' - f'(peak {peak:.1f}% background CPU; processes: {", ".join(names) or "none named"}).', + f'(peak {peak:.1f}% competing CPU; processes: {", ".join(names) or "none named"}).', 'The hosted runner was not idle, so this result neither passes nor fails the budget.'] print('\n'.join(lines), flush=True) if summary_path: @@ -181,6 +208,7 @@ def measure(binary, prefix): 'PATH', 'HOME', 'TMPDIR', 'TZ', 'GOMAXPROCS', 'CI', 'GITHUB_ACTIONS', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT') if key in os.environ} env['PULSE_STRICT_LOCAL_API_SLOS'] = 'true' + capacity_context = dict(hosted=env.get('GITHUB_ACTIONS') == 'true', cpu_count=os.cpu_count()) command = [str(binary), '-test.run=^TestSLO_WorkloadCharts$', '-test.v', '-test.count=1', '-test.cpu=2', '-test.timeout=5m'] capacity_errors = [] @@ -201,6 +229,7 @@ def measure(binary, prefix): if own_before is not None and own_after is not None and row['total_ticks'] > 0: row['measurement_cpu_pct'] = 100 * (own_after - own_before) / row['total_ticks'] row['background_busy_pct'] = max(0, row['busy_pct'] - row['measurement_cpu_pct']) + row.update(background_capacity(row, **capacity_context)) if competing_capacity(row): capacity_errors.append(row) receipt.write(json.dumps(row) + '\n') @@ -217,6 +246,7 @@ def measure(binary, prefix): before = after verdict = 'inconclusive' if capacity_errors else 'valid' result = dict(command=command, binary_sha256=identity, environment=env, + capacity_context=capacity_context, pid=process.pid, returncode=returncode, verdict=verdict, capacity_errors=capacity_errors) prefix.with_suffix('.result.json').write_text(json.dumps(result, indent=2) + '\n') diff --git a/internal/api/testdata/test_chart_latency_proof.py b/internal/api/testdata/test_chart_latency_proof.py index 04a1cb9235..6762e0285e 100644 --- a/internal/api/testdata/test_chart_latency_proof.py +++ b/internal/api/testdata/test_chart_latency_proof.py @@ -82,6 +82,72 @@ class ChartLatencyProofTests(unittest.TestCase): with self.subTest(field=field): self.assertTrue(proof.competing_capacity(dict(row, **{field: value}))) + def test_bounded_hosted_agent_is_attributed_without_changing_raw_evidence(self): + row = dict(background_busy_pct=26.799007444168744, guest_pct=0, steal_pct=0, + swap_pages=0, background_processes=[ + dict(pid=2106, name='provjobd1892260', cpu_pct=25.31017369727047), + dict(pid=45, name='kworker/u16:1-flush-8:0', cpu_pct=1.2406947890818858), + dict(pid=3197, name='python3', cpu_pct=0.49627791563275436)]) + original = repr(row) + capacity = proof.background_capacity(row, hosted=True, cpu_count=4) + self.assertEqual(capacity['runner_agent_cpu_pct'], row['background_processes'][0]['cpu_pct']) + self.assertAlmostEqual(capacity['competing_busy_pct'], 1.7369727047146402) + self.assertFalse(proof.competing_capacity(dict(row, **capacity))) + self.assertEqual(repr(row), original) + + def test_agent_exception_requires_hosted_context_exact_name_and_one_cpu_bound(self): + for hosted, cpus, name, cpu in [ + (False, 4, 'provjobd1892260', 25), + (True, None, 'provjobd1892260', 25), + (True, 0, 'provjobd1892260', 25), + (True, 8, 'provjobd1892260', 25), + (True, 4, 'provjobd1892260', 26.01), + (True, 4, 'provjobd', 25), + (True, 4, 'provjobd1892260-extra', 25), + (True, 4, 'provjobd-1', 25), + (True, 4, 'Runner.Worker', 25), + (True, 4, 'node', 25)]: + with self.subTest(hosted=hosted, cpus=cpus, name=name, cpu=cpu): + row = dict(background_busy_pct=cpu, background_processes=[ + dict(pid=1, name=name, cpu_pct=cpu)]) + capacity = proof.background_capacity(row, hosted=hosted, cpu_count=cpus) + self.assertEqual(capacity['runner_agent_cpu_pct'], 0) + self.assertEqual(capacity['competing_busy_pct'], cpu) + + def test_other_unattributed_or_multiple_agent_cpu_stays_competing(self): + agent = dict(pid=1, name='provjobd1892260', cpu_pct=25) + other = dict(pid=2, name='node', cpu_pct=11) + for background, processes in [ + (36, [agent, other]), (5, [agent, other]), (36, [agent]), + (25, []), (25, [agent, dict(pid=2, name='provjobd2772537', cpu_pct=0.25)])]: + with self.subTest(background=background, processes=processes): + row = dict(background_busy_pct=background, guest_pct=0, steal_pct=0, + swap_pages=0, background_processes=processes) + capacity = proof.background_capacity(row, hosted=True, cpu_count=4) + self.assertTrue(proof.competing_capacity(dict(row, **capacity))) + + def test_runner_agent_cannot_clear_guest_steal_or_swap(self): + row = dict(background_busy_pct=25, guest_pct=0, steal_pct=0, swap_pages=0, + background_processes=[dict(pid=1, name='provjobd1892260', cpu_pct=25)]) + row.update(proof.background_capacity(row, hosted=True, cpu_count=4)) + self.assertFalse(proof.competing_capacity(row)) + for field, value in [('guest_pct', 2.01), ('steal_pct', 0.11), ('swap_pages', 1)]: + with self.subTest(field=field): + self.assertTrue(proof.competing_capacity(dict(row, **{field: value}))) + + def test_inconclusive_summary_separates_agent_from_actual_contention(self): + row = dict(background_busy_pct=36, background_processes=[ + dict(pid=1, name='provjobd1892260', cpu_pct=25), + dict(pid=2, name='node', cpu_pct=11)]) + row.update(proof.background_capacity(row, hosted=True, cpu_count=4)) + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / 'summary.md' + proof.report_inconclusive('charts/workloads p50=1ms p95=2ms p99=3ms SLO=90ms', + [row], summary) + text = summary.read_text() + self.assertIn('peak 11.0% competing CPU; processes: node', text) + self.assertNotIn('provjobd', text) + def test_process_receipt_excludes_measurement_and_handles_new_or_reused_pids(self): before = {10: dict(name='test', ticks=20, start='1'), 20: dict(name='other', ticks=30, start='2'), diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index d5cea47d59..ae30fff668 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -71,13 +71,14 @@ func TestChartSLOWorkflowPreservesInconclusiveAndFailure(t *testing.T) { } for _, tc := range []struct { name, conclusion string + p95 int processExit int stepPass bool }{ - {"valid", "success", 0, true}, - {"contended", "neutral", 78, true}, - {"budget failure", "failure", 1, false}, - {"broken proof", "failure", 2, false}, + {"valid hosted envelope", "success", 82, 0, true}, + {"contended", "neutral", 107, 78, true}, + {"hosted envelope budget failure", "failure", 107, 1, false}, + {"broken proof", "failure", 82, 2, false}, } { t.Run(tc.name, func(t *testing.T) { directory := t.TempDir() @@ -96,7 +97,7 @@ func TestChartSLOWorkflowPreservesInconclusiveAndFailure(t *testing.T) { t.Fatal(err) } } - const report = "charts/workloads p50=40ms p95=82ms p99=86ms SLO=90ms" + report := "charts/workloads p50=40ms p95=" + strconv.Itoa(tc.p95) + "ms p99=112ms SLO=90ms" if err := os.WriteFile(filepath.Join(samples, "candidate.log"), []byte(report+"\n"), 0600); err != nil { t.Fatal(err) }