mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-25 21:20:00 +00:00
Complete credited chart evidence reporting on current main
Isolate the retained PR3095 completion (7f4ec192and93d87118) without importing unrelated history. Keep the exact proposed Python classifier and maintained chart shells, real CLI/workflow controls and dependent contracts. Inconclusive capacity never establishes the 90 ms budget; child, cleanup and malformed-proof failures remain errors. Source-PR: https://github.com/rcourtman/Pulse/pull/3095 Source-Commit:b31fabd778Co-authored-by: rcourtman <8825017+rcourtman@users.noreply.github.com> Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
@@ -759,11 +759,14 @@ jobs:
|
||||
run: go build ./cmd/pulse
|
||||
|
||||
chart-latency-slo:
|
||||
name: Workload chart 90ms SLO
|
||||
name: Workload chart SLO evidence
|
||||
needs: changes
|
||||
if: github.event_name == 'pull_request' && needs.changes.outputs.go_compile == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
outputs:
|
||||
conclusion: ${{ steps.proof.outputs.conclusion }}
|
||||
report: ${{ steps.proof.outputs.report }}
|
||||
env:
|
||||
GOMAXPROCS: "2"
|
||||
steps:
|
||||
@@ -808,11 +811,34 @@ jobs:
|
||||
# This additional evidence job keeps the ordinary hosted envelope and
|
||||
# required source/release gates intact. A fresh VM must first demonstrate
|
||||
# idle capacity; the candidate then enforces the existing local 90 ms SLO.
|
||||
# Competing capacity during the measurement (on hosted runners, the
|
||||
# runner's own provjobd daemon in every observed run) voids the latency
|
||||
# verdict: the proof exits 78 and the verdict check is neutral, not red.
|
||||
- name: Verify chart latency on observed idle capacity
|
||||
id: proof
|
||||
run: |
|
||||
set +e
|
||||
python3 internal/api/testdata/chart_latency_proof.py \
|
||||
--candidate "$RUNNER_TEMP/chart-latency/candidate-api.test" \
|
||||
--output "$RUNNER_TEMP/chart-latency/samples"
|
||||
status=$?
|
||||
set -e
|
||||
case "$status" in
|
||||
0) conclusion=success ;;
|
||||
78) conclusion=neutral ;;
|
||||
*) conclusion=failure ;;
|
||||
esac
|
||||
report="$(grep -o 'charts/workloads p50=[^ ]* p95=[^ ]* p99=[^ ]* SLO=[^ ]*' \
|
||||
"$RUNNER_TEMP/chart-latency/samples/candidate.log" 2>/dev/null | head -n 1 || true)"
|
||||
{
|
||||
echo "conclusion=$conclusion"
|
||||
echo "report=$report"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
# The proof writes its own summary for an inconclusive run.
|
||||
[ "$conclusion" = neutral ] ||
|
||||
printf 'Workload chart 90ms SLO: %s\n\n%s\n' "$conclusion" \
|
||||
"${report:-No SLO report was produced.}" >> "$GITHUB_STEP_SUMMARY"
|
||||
[ "$status" -eq 0 ] || [ "$status" -eq 78 ]
|
||||
|
||||
- name: Upload chart latency evidence
|
||||
if: always()
|
||||
@@ -825,6 +851,40 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
# A separate job holds the check-write token, so the evidence job that runs
|
||||
# PR-built executables stays read-only. Dependabot's read-only token cannot
|
||||
# create check runs; the evidence job summary still carries the verdict.
|
||||
# An evidence job that failed before the proof ran publishes a failure.
|
||||
chart-latency-verdict:
|
||||
name: Publish chart SLO verdict
|
||||
needs: chart-latency-slo
|
||||
if: always() && (needs.chart-latency-slo.result == 'success' || needs.chart-latency-slo.result == 'failure')
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
checks: write
|
||||
steps:
|
||||
- name: Publish SLO verdict check
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
CONCLUSION: ${{ needs.chart-latency-slo.outputs.conclusion || 'failure' }}
|
||||
REPORT: ${{ needs.chart-latency-slo.outputs.report }}
|
||||
DETAILS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
case "$CONCLUSION" in
|
||||
success) title='p95 within 90 ms on an idle runner' ;;
|
||||
neutral) title='Inconclusive: the runner was not idle during the measurement' ;;
|
||||
*) title='Failed on an idle runner, or the proof could not run' ;;
|
||||
esac
|
||||
gh api "repos/$GITHUB_REPOSITORY/check-runs" \
|
||||
-f name='Workload chart 90ms SLO' -f head_sha="$HEAD_SHA" \
|
||||
-f status=completed -f conclusion="$CONCLUSION" -f details_url="$DETAILS_URL" \
|
||||
-f "output[title]=$title" \
|
||||
-f "output[summary]=${REPORT:-No SLO report was produced.} See the evidence job summary and chart-latency artifact." \
|
||||
> /dev/null ||
|
||||
echo "::warning title=Chart SLO verdict not published::This token cannot create check runs; the evidence job carries the verdict."
|
||||
|
||||
benchmarks:
|
||||
name: Benchmarks
|
||||
needs: changes
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
# Completing hosted workload-chart reporting, 21 October 2026
|
||||
|
||||
Original proposal: [Pulse PR 3095](https://github.com/rcourtman/Pulse/pull/3095),
|
||||
Richard Courtman's shared `b31fabd778f64f4bc35d1f33438256a2b24ff443`.
|
||||
The two Python proof files are preserved byte-for-byte; only its chart evidence
|
||||
and verdict jobs are composed into the assigned maintained workflow. Other
|
||||
workflow jobs and the current frontend/toolchain remain untouched. The shared
|
||||
commit is not amended, and its unqualified historical timing conclusions are
|
||||
not adopted as a regression verdict.
|
||||
|
||||
[Governance job 119307595195](https://github.com/rcourtman/Pulse/actions/runs/39624922365/job/119307595195)
|
||||
rejects missing dependent contracts and API/deployment verification. This
|
||||
completion supplies those contracts and executed CLI/workflow regression
|
||||
controls rather than waiving the guard or changing the latency threshold.
|
||||
A broad composition was abandoned because it would pull unrelated frontend
|
||||
and toolchain changes into this bounded repair. The original proposal remains
|
||||
shared and unchanged; the credited canonical equivalent receives its own
|
||||
independent review, containing CI and protected publication.
|
||||
|
||||
The classifier distinguishes a valid 90 ms pass/fail from an invalidated
|
||||
measurement. A neutral hosted check is not latency qualification, native
|
||||
recovery, released availability or evidence that a regression is absent.
|
||||
Synthetic child/telemetry and fake-provider controls establish classification,
|
||||
raw-evidence preservation and permission separation only. No performance test,
|
||||
database investigation, workflow retry or release operation is part of this
|
||||
completion. The current hosted artifact, exact source proof, adverse results
|
||||
and remaining delivery state are retained in Delivery's linked outcome.
|
||||
@@ -10210,3 +10210,15 @@ runs when they close, failed runs included, and publishes the action plans the
|
||||
inbox expiry sweep expires, enrols, commands, updates or removes no agent and
|
||||
touches no agent identity or credential. Investigations still reach hosts only
|
||||
through the read-only evidence tools and the governed action lifecycle.
|
||||
|
||||
### Hosted chart evidence cannot qualify an agent lifecycle
|
||||
|
||||
The optional hosted workload-chart capacity proof reports an executed but
|
||||
contended measurement as inconclusive, never as a healthy reading or a passed
|
||||
90 ms budget. This CI-only classification does not satisfy agent installation,
|
||||
restart, confinement, credential persistence or signed recovery acceptance.
|
||||
Skips, missing reports and non-budget child failures still fail the proof.
|
||||
The actual CLI/receipt control is
|
||||
`TestContractChartSLOProofCLIClassifiesExecution`; the separate read-only
|
||||
evidence and check-writer boundary is
|
||||
`TestChartSLOWorkflowPreservesInconclusiveAndFailure`.
|
||||
|
||||
@@ -12563,3 +12563,17 @@ changes. Proofs: `internal/actionlifecycle/service_test.go`
|
||||
`TestRefreshedActionIDNamesTheReplacedAction`),
|
||||
`internal/api/patrol_action_expiry_sweep_test.go` and
|
||||
`internal/api/ai_handlers_investigation_finalize_test.go`.
|
||||
|
||||
### Workload-chart proof distinguishes an invalid measurement from recovery
|
||||
|
||||
`internal/api/testdata/chart_latency_proof.py` keeps the strict 90 ms budget,
|
||||
exact one-test execution, executable digest and child result in its receipt.
|
||||
Only a single complete PASS or budget-only FAIL with competing capacity can
|
||||
exit 78 and describe the timing as inconclusive. A skipped/missing/duplicate
|
||||
report, wrong budget, cleanup failure or other child failure remains an error.
|
||||
A valid-capacity budget failure still fails. No API payload, route, History
|
||||
sample, incident state or operational-recovery result is changed by this CI
|
||||
classification. `TestContractChartSLOProofCLIClassifiesExecution` drives the
|
||||
real CLI and disposable child processes through all ten outcomes, verifies
|
||||
unaltered logs/result/digest/strict arguments and excludes the synthetic token
|
||||
from the child environment. Its capacity rows are synthetic, not latency proof.
|
||||
|
||||
@@ -3764,6 +3764,22 @@ artifact-selection behaviour.
|
||||
|
||||
## Current State
|
||||
|
||||
### Hosted chart verdict writer has no proposed executable
|
||||
|
||||
Build and Test's optional workload-chart evidence job retains a read-only
|
||||
repository token while running the PR-built executable. Its proof maps only
|
||||
exit 0 to success and 78 to neutral; any other exit fails. The separate check
|
||||
writer has only `checks: write`, no checkout or proposed executable, uses the
|
||||
exact PR head and defaults missing proof output to failure. The raw timing and
|
||||
capacity artifact and evidence summary remain necessary: a neutral check
|
||||
establishes neither the 90 ms budget nor release or installed acceptance.
|
||||
Dependabot/fork token limitations remain explicit, not alternate credentials.
|
||||
`TestChartSLOWorkflowPreservesInconclusiveAndFailure` executes the actual YAML
|
||||
shells with a synthetic proof exit and a credential-free capturing provider,
|
||||
covering success, neutral, budget failure and a broken proof. It checks the
|
||||
read-only evidence/separate-writer admission and missing-output failure path.
|
||||
No real check is created, workflow dispatched or latency workload executed.
|
||||
|
||||
### Registered Windows native-exit completion proof (8 October 2026)
|
||||
|
||||
The workflow's seven immediate native exit checks and the lifecycle harness's
|
||||
|
||||
@@ -4412,3 +4412,18 @@ handler in `useWorkloadsControlsState.ts` no longer checks for a focused chat
|
||||
composer. `.bundlesize.json` drops the retired chunks; `useUnifiedResources`
|
||||
grew to about 10 kB gzipped because Rollup moved the shared filter toolbar
|
||||
modules into it once the `Chat` chunk stopped sharing them.
|
||||
|
||||
### Hosted workload-chart observations keep three distinct outcomes
|
||||
|
||||
The optional hosted job reports an invalidated measurement as inconclusive:
|
||||
exit 78 is neutral, not a passed 90 ms budget. Valid capacity still requires a
|
||||
single executed passing test at that budget. A contended FAIL can be neutral
|
||||
only when its sole test diagnostic is the budget violation; skips, missing or
|
||||
duplicate reports, wrong budgets, cleanup and child errors remain failures.
|
||||
The private disposable-worker route continues to reject every nonzero exit,
|
||||
including 78. Raw return codes, logs, digests and competing-capacity evidence
|
||||
remain intact; invalid timings do not demonstrate absence of regression.
|
||||
The registered API CLI and installtest workflow controls execute synthetic
|
||||
children/telemetry and the actual evidence/writer shells, not a latency
|
||||
workload. See `workload-chart-reporting-completion-2026-10-21.md` in `../records/`
|
||||
for the source proposal, adverse governance result and available hosted sample.
|
||||
|
||||
@@ -7198,3 +7198,14 @@ The `internal/api` change of 2026-10-23 that finalizes Patrol investigation
|
||||
runs when they close and publishes the action plans the inbox expiry sweep
|
||||
expires opens no storage, backup or recovery path. Recovery evidence, restore
|
||||
rules and storage capabilities are unchanged.
|
||||
|
||||
### Capacity-invalid chart evidence is not storage recovery
|
||||
|
||||
The hosted workload-chart proof may call a complete, contended measurement
|
||||
inconclusive without clearing its raw budget failure. It must not turn a
|
||||
cleanup/child error, absent report or skipped test into neutral evidence.
|
||||
Neither neutrality nor synthetic CLI classification establishes a valid
|
||||
History latency result, intact stored data, guest liveness or installed
|
||||
recovery. `TestContractChartSLOProofCLIClassifiesExecution` preserves child
|
||||
exit, output and executable identity across those distinctions; the 90 ms
|
||||
valid-capacity requirement and independent native recovery remain unchanged.
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
@@ -79,6 +80,87 @@ import (
|
||||
tmock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// Exercise the proof's real CLI, child execution and on-disk receipts without
|
||||
// running a latency workload. Only capacity telemetry and sampling delays are
|
||||
// synthetic; a malformed/failed child must never become a neutral budget result.
|
||||
func TestContractChartSLOProofCLIClassifiesExecution(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
command := exec.CommandContext(ctx, "python3", "-c", `
|
||||
import hashlib, json, os, shlex, sys, tempfile
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path('testdata').resolve()))
|
||||
import chart_latency_proof as proof
|
||||
|
||||
sleep = proof.time.sleep
|
||||
proof.time.sleep = lambda seconds: sleep(0.005)
|
||||
proof.wait_idle = lambda prefix: None
|
||||
proof.snapshot = lambda: {}
|
||||
proof.process_counters = lambda: {}
|
||||
proof.ticks = lambda pid: 0
|
||||
proof.process_io = lambda pid: {}
|
||||
os.environ['GITHUB_TOKEN'] = 'synthetic-must-not-reach-child'
|
||||
report = ' slo_bench_test.go:637: charts/workloads p50=75ms p95=107ms p99=112ms SLO=90ms\n'
|
||||
violation = ' slo_bench_test.go:637: SLO VIOLATION: p95=107ms exceeds target 90ms\n'
|
||||
passed = report.replace('p95=107ms p99=112ms', 'p95=82ms p99=86ms') + '--- PASS: TestSLO_WorkloadCharts (0.03s)\nPASS\n'
|
||||
failed = report + violation + '--- FAIL: TestSLO_WorkloadCharts (0.03s)\nFAIL\n'
|
||||
cases = [
|
||||
('idle-pass', False, passed, 0, 0),
|
||||
('idle-budget-failure', False, failed, 1, 1),
|
||||
('contended-pass', True, passed, 0, 78),
|
||||
('contended-budget-failure', True, failed, 1, 78),
|
||||
('contended-cleanup-error', True, failed.replace(violation, violation +
|
||||
' testing.go:1369: TempDir RemoveAll cleanup: directory not empty\n'), 1, 1),
|
||||
('contended-skip', True, report + '--- SKIP: TestSLO_WorkloadCharts (0.03s)\n', 0, 1),
|
||||
('contended-process-error', True, failed, 2, 1),
|
||||
('contended-no-report', True, '--- PASS: TestSLO_WorkloadCharts (0.03s)\n', 0, 1),
|
||||
('contended-wrong-budget', True, passed.replace('90ms', '550ms'), 0, 1),
|
||||
('contended-duplicate', True, passed + passed, 0, 1),
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name, contended, text, child_exit, expected in cases:
|
||||
proof.interval = lambda before, after: dict(total_ticks=100,
|
||||
busy_pct=25 if contended else 0, guest_pct=0, steal_pct=0, swap_pages=0)
|
||||
binary = root / (name + '.sh')
|
||||
binary.write_text('#!/bin/sh\n'
|
||||
'test -z "${GITHUB_TOKEN:-}" || exit 7\n'
|
||||
'test "$1" = "-test.run=^TestSLO_WorkloadCharts$" || exit 8\n'
|
||||
'sleep 0.03\nprintf %s ' + shlex.quote(text) + '\nexit ' + str(child_exit) + '\n')
|
||||
binary.chmod(0o700)
|
||||
output = root / name
|
||||
summary = root / (name + '.summary')
|
||||
os.environ['GITHUB_STEP_SUMMARY'] = str(summary)
|
||||
sys.argv = ['chart_latency_proof.py', '--candidate', str(binary), '--output', str(output)]
|
||||
observed = 0
|
||||
try:
|
||||
proof.main()
|
||||
except SystemExit as error:
|
||||
observed = error.code
|
||||
except RuntimeError:
|
||||
observed = 1
|
||||
assert observed == expected, (name, observed, expected)
|
||||
receipt = json.loads((output / 'candidate.result.json').read_text())
|
||||
assert receipt['returncode'] == child_exit, name
|
||||
assert receipt['binary_sha256'] == hashlib.sha256(binary.read_bytes()).hexdigest(), name
|
||||
assert receipt['environment']['PULSE_STRICT_LOCAL_API_SLOS'] == 'true', name
|
||||
assert 'GITHUB_TOKEN' not in receipt['environment'], name
|
||||
assert (output / 'candidate.log').read_text() == text, name
|
||||
assert bool(receipt['capacity_errors']) == contended, name
|
||||
assert receipt['command'][1:] == ['-test.run=^TestSLO_WorkloadCharts$', '-test.v',
|
||||
'-test.count=1', '-test.cpu=2', '-test.timeout=5m'], name
|
||||
assert summary.exists() == (expected == 78), name
|
||||
if summary.exists():
|
||||
assert 'inconclusive' in summary.read_text() and '(not a verdict)' in summary.read_text(), name
|
||||
print('CLASSIFIED', name, observed)
|
||||
`)
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
t.Fatalf("chart proof CLI fixture failed: %v\n%s", err, output)
|
||||
} else {
|
||||
t.Logf("synthetic CLI classification, not measured latency:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentConfigFetchAuditCapacityContract(t *testing.T) {
|
||||
capture := &auditCaptureLogger{}
|
||||
previousLogger, previousManager := audit.GetLogger(), GetTenantAuditManager()
|
||||
|
||||
+53
-7
@@ -7,8 +7,13 @@ import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
# A capacity-invalid proof is neither a pass nor a failure. The workflow maps
|
||||
# this exit status to a neutral check conclusion.
|
||||
INCONCLUSIVE_EXIT = 78
|
||||
|
||||
|
||||
def snapshot():
|
||||
cpu = list(map(int, Path('/proc/stat').read_text().splitlines()[0].split()[1:]))
|
||||
@@ -121,15 +126,51 @@ def digest(path):
|
||||
return hashlib.file_digest(source, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def slo_report(text):
|
||||
reports = re.findall(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=(\S+)', text)
|
||||
if len(reports) != 1 or reports[0] != '90ms':
|
||||
raise RuntimeError(f'Expected one explicit SLO report at the selected budget, got {reports}')
|
||||
return re.search(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=\S+', text).group(0)
|
||||
|
||||
|
||||
def validate_output(text, returncode):
|
||||
if returncode != 0:
|
||||
raise RuntimeError(f'SLO process exited {returncode}')
|
||||
passes = re.findall(r'^--- PASS: TestSLO_WorkloadCharts \(', text, re.MULTILINE)
|
||||
if len(passes) != 1 or re.search(r'^--- SKIP: TestSLO_WorkloadCharts ', text, re.MULTILINE):
|
||||
raise RuntimeError('SLO did not pass, including a skipped or missing test')
|
||||
reports = re.findall(r'charts/workloads p50=\S+ p95=\S+ p99=\S+ SLO=(\S+)', text)
|
||||
if len(reports) != 1 or reports[0] != '90ms':
|
||||
raise RuntimeError(f'Expected one explicit SLO report at the selected budget, got {reports}')
|
||||
slo_report(text)
|
||||
|
||||
|
||||
def validate_contended_output(text, returncode):
|
||||
# Contention only voids the latency verdict. The test must still have
|
||||
# executed once and reported, so a crash, skip or broken fixture fails.
|
||||
report = slo_report(text)
|
||||
outcome = re.findall(r'^--- (PASS|FAIL|SKIP): TestSLO_WorkloadCharts \(', text, re.MULTILINE)
|
||||
diagnostics = re.findall(r'^\s+\S+\.go:\d+: (.*)$', text, re.MULTILINE)
|
||||
violations = [line for line in diagnostics if line.startswith('SLO VIOLATION: p95=')]
|
||||
others = [line for line in diagnostics
|
||||
if line not in violations and not line.startswith('charts/workloads p50=')]
|
||||
budget_only = (outcome == ['FAIL'] and returncode == 1
|
||||
and len(violations) == 1 and not others)
|
||||
if not (outcome == ['PASS'] and returncode == 0) and not budget_only:
|
||||
raise RuntimeError(f'SLO process exited {returncode} with outcome {outcome}')
|
||||
return report
|
||||
|
||||
|
||||
def report_inconclusive(report, capacity_errors, summary_path):
|
||||
names = sorted({row['name'] for error in capacity_errors
|
||||
for row in error['background_processes'] if row['cpu_pct'] > 2})
|
||||
peak = max(error['background_busy_pct'] for error in capacity_errors)
|
||||
lines = ['Workload chart 90ms SLO: inconclusive',
|
||||
f'{report} (not a verdict)',
|
||||
f'{len(capacity_errors)} measurement intervals had competing capacity '
|
||||
f'(peak {peak:.1f}% background CPU; processes: {", ".join(names) or "none named"}).',
|
||||
'The hosted runner was not idle, so this result neither passes nor fails the budget.']
|
||||
print('\n'.join(lines), flush=True)
|
||||
if summary_path:
|
||||
with open(summary_path, 'a') as summary:
|
||||
summary.write('\n\n'.join(lines) + '\n')
|
||||
|
||||
|
||||
def measure(binary, prefix):
|
||||
@@ -174,17 +215,21 @@ def measure(binary, prefix):
|
||||
after = snapshot()
|
||||
receipt.write(json.dumps(interval(before, after)) + '\n')
|
||||
before = after
|
||||
verdict = 'inconclusive' if capacity_errors else 'valid'
|
||||
result = dict(command=command, binary_sha256=identity, environment=env,
|
||||
pid=process.pid, returncode=returncode, capacity_errors=capacity_errors)
|
||||
pid=process.pid, returncode=returncode, verdict=verdict,
|
||||
capacity_errors=capacity_errors)
|
||||
prefix.with_suffix('.result.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
text = prefix.with_suffix('.log').read_text()
|
||||
print(text, flush=True)
|
||||
if digest(binary) != identity:
|
||||
raise RuntimeError('Measured executable changed')
|
||||
if capacity_errors:
|
||||
raise RuntimeError('Background CPU or guest/steal activity invalidated the idle measurement '
|
||||
f'(SLO process exit {returncode}; see raw log)')
|
||||
report = validate_contended_output(text, returncode)
|
||||
report_inconclusive(report, capacity_errors, os.environ.get('GITHUB_STEP_SUMMARY'))
|
||||
return verdict
|
||||
validate_output(text, returncode)
|
||||
return verdict
|
||||
|
||||
|
||||
def main():
|
||||
@@ -195,7 +240,8 @@ def main():
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
# Absolute qualification executes the candidate directly. The separate
|
||||
# paired benchmark job owns relative comparisons and baseline context.
|
||||
measure(args.candidate.resolve(), args.output / 'candidate')
|
||||
if measure(args.candidate.resolve(), args.output / 'candidate') == 'inconclusive':
|
||||
sys.exit(INCONCLUSIVE_EXIT)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import chart_latency_proof as proof
|
||||
|
||||
@@ -34,6 +36,43 @@ class ChartLatencyProofTests(unittest.TestCase):
|
||||
with self.assertRaises(RuntimeError):
|
||||
proof.validate_output(text, status)
|
||||
|
||||
def test_contention_voids_only_the_latency_verdict(self):
|
||||
report = ' slo_bench_test.go:637: charts/workloads p50=75ms p95=107ms p99=112ms SLO=90ms\n'
|
||||
violation = ' slo_bench_test.go:637: SLO VIOLATION: p95=107ms exceeds target 90ms\n'
|
||||
passed = report + '--- PASS: TestSLO_WorkloadCharts (1.0s)\n'
|
||||
failed = report + violation + '--- FAIL: TestSLO_WorkloadCharts (1.0s)\nFAIL\n'
|
||||
for text, status in [(passed, 0), (failed, 1)]:
|
||||
with self.subTest(text=text):
|
||||
self.assertEqual(proof.validate_contended_output(text, status),
|
||||
report.split(': ', 1)[1].strip())
|
||||
cleanup = ' testing.go:1369: TempDir RemoveAll cleanup: directory not empty\n'
|
||||
for text, status in [(report + '--- SKIP: TestSLO_WorkloadCharts (1.0s)\n', 0),
|
||||
(failed.replace(violation, ''), 1), (failed, 2), (passed, 1),
|
||||
(failed.replace(violation, violation + cleanup), 1),
|
||||
(failed.replace(violation, violation + violation), 1),
|
||||
('--- FAIL: TestSLO_WorkloadCharts (1.0s)\n', 1),
|
||||
(passed.replace('90ms', '550ms'), 0), (passed + passed, 0)]:
|
||||
with self.subTest(text=text, status=status):
|
||||
with self.assertRaises(RuntimeError):
|
||||
proof.validate_contended_output(text, status)
|
||||
|
||||
def test_inconclusive_summary_names_competing_processes(self):
|
||||
errors = [dict(background_busy_pct=25.2, background_processes=[
|
||||
dict(pid=1, name='provjobd312033', cpu_pct=25.2),
|
||||
dict(pid=2, name='python3', cpu_pct=0.2)]),
|
||||
dict(background_busy_pct=14.4, background_processes=[
|
||||
dict(pid=1, name='provjobd312033', cpu_pct=14.2)])]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
summary = Path(directory) / 'summary.md'
|
||||
proof.report_inconclusive('charts/workloads p50=1ms p95=2ms p99=3ms SLO=90ms',
|
||||
errors, summary)
|
||||
text = summary.read_text()
|
||||
self.assertIn('inconclusive', text)
|
||||
self.assertIn('2 measurement intervals', text)
|
||||
self.assertIn('peak 25.2%', text)
|
||||
self.assertIn('provjobd312033', text)
|
||||
self.assertNotIn('python3', text)
|
||||
|
||||
def test_measurement_io_stays_in_latency_and_background_cpu_invalidates(self):
|
||||
row = dict(background_busy_pct=0, guest_pct=0, steal_pct=0, swap_pages=0,
|
||||
iowait_pct=20)
|
||||
|
||||
@@ -22,6 +22,118 @@ import (
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Run the maintained evidence/writer shells with a fake proof process and a
|
||||
// capturing, credential-free provider. This protects the actual exit mapping
|
||||
// and token separation without creating a check or executing a timed workload.
|
||||
func TestChartSLOWorkflowPreservesInconclusiveAndFailure(t *testing.T) {
|
||||
content, err := os.ReadFile(repoFile(".github", "workflows", "build-and-test.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
type step struct {
|
||||
ID, Run, Uses string
|
||||
Env map[string]string
|
||||
}
|
||||
var workflow struct {
|
||||
Permissions map[string]string
|
||||
Jobs map[string]struct {
|
||||
Permissions map[string]string
|
||||
Needs any
|
||||
If string
|
||||
Steps []step
|
||||
}
|
||||
}
|
||||
if err := yaml.Unmarshal(content, &workflow); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence := workflow.Jobs["chart-latency-slo"]
|
||||
writer := workflow.Jobs["chart-latency-verdict"]
|
||||
if len(workflow.Permissions) != 1 || workflow.Permissions["contents"] != "read" || len(evidence.Permissions) != 0 {
|
||||
t.Fatal("PR-built executables must keep the read-only workflow token")
|
||||
}
|
||||
if len(writer.Permissions) != 1 || writer.Permissions["checks"] != "write" || writer.Needs != "chart-latency-slo" ||
|
||||
!strings.Contains(writer.If, "always()") || !strings.Contains(writer.If, "'failure'") || len(writer.Steps) != 1 || writer.Steps[0].Uses != "" {
|
||||
t.Fatal("verdict writer must remain a separate no-checkout job, including failed evidence")
|
||||
}
|
||||
publish := writer.Steps[0]
|
||||
if publish.Env["CONCLUSION"] != "${{ needs.chart-latency-slo.outputs.conclusion || 'failure' }}" ||
|
||||
publish.Env["HEAD_SHA"] != "${{ github.event.pull_request.head.sha }}" {
|
||||
t.Fatal("missing proof output must fail at the exact PR head")
|
||||
}
|
||||
proofShell := ""
|
||||
for _, candidate := range evidence.Steps {
|
||||
if candidate.ID == "proof" {
|
||||
proofShell = candidate.Run
|
||||
}
|
||||
}
|
||||
if proofShell == "" || publish.Run == "" || !strings.Contains(proofShell, "chart_latency_proof.py") {
|
||||
t.Fatal("missing actual chart proof/writer shell")
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name, conclusion string
|
||||
processExit int
|
||||
stepPass bool
|
||||
}{
|
||||
{"valid", "success", 0, true},
|
||||
{"contended", "neutral", 78, true},
|
||||
{"budget failure", "failure", 1, false},
|
||||
{"broken proof", "failure", 2, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
bin := filepath.Join(directory, "bin")
|
||||
samples := filepath.Join(directory, "chart-latency", "samples")
|
||||
for _, path := range []string{bin, samples} {
|
||||
if err := os.MkdirAll(path, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for name, script := range map[string]string{
|
||||
"python3": "#!/bin/sh\nexit \"$CHART_PROOF_STATUS\"\n",
|
||||
"gh": "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$CAPTURE_ARGS\"\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(bin, name), []byte(script), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const report = "charts/workloads p50=40ms p95=82ms p99=86ms SLO=90ms"
|
||||
if err := os.WriteFile(filepath.Join(samples, "candidate.log"), []byte(report+"\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outputs := filepath.Join(directory, "outputs")
|
||||
summary := filepath.Join(directory, "summary")
|
||||
capture := filepath.Join(directory, "provider-args")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
command := exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", proofShell)
|
||||
command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"),
|
||||
"RUNNER_TEMP="+directory, "GITHUB_OUTPUT="+outputs, "GITHUB_STEP_SUMMARY="+summary,
|
||||
"CHART_PROOF_STATUS="+strconv.Itoa(tc.processExit))
|
||||
if output, err := command.CombinedOutput(); (err == nil) != tc.stepPass {
|
||||
t.Fatalf("evidence shell status: %v\n%s", err, output)
|
||||
}
|
||||
got, err := os.ReadFile(outputs)
|
||||
if err != nil || string(got) != "conclusion="+tc.conclusion+"\nreport="+report+"\n" {
|
||||
t.Fatalf("wrong downstream evidence: %q, %v", got, err)
|
||||
}
|
||||
command = exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", publish.Run)
|
||||
const head = "1234567890123456789012345678901234567890"
|
||||
command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"),
|
||||
"CONCLUSION="+tc.conclusion, "REPORT="+report, "CAPTURE_ARGS="+capture, "HEAD_SHA="+head,
|
||||
"GITHUB_REPOSITORY=rcourtman/Pulse", "DETAILS_URL=https://github.com/rcourtman/Pulse/actions/runs/1", "GH_TOKEN=")
|
||||
if output, err := command.CombinedOutput(); err != nil {
|
||||
t.Fatalf("writer shell: %v\n%s", err, output)
|
||||
}
|
||||
args, err := os.ReadFile(capture)
|
||||
if err != nil || !strings.Contains(string(args), "api\nrepos/rcourtman/Pulse/check-runs\n") ||
|
||||
!strings.Contains(string(args), "head_sha="+head+"\n") ||
|
||||
!strings.Contains(string(args), "conclusion="+tc.conclusion+"\n") {
|
||||
t.Fatalf("wrong captured check identity/verdict: %q, %v", args, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReleaseUsesV6InstallScripts(t *testing.T) {
|
||||
content, err := os.ReadFile(repoFile("scripts", "build-release.sh"))
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user