test(perf): strengthen HotPath evidence contracts (#5581)

* test(perf): mark nonformal ABBA artifacts

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(io-metrics): lock disabled EC stage accounting

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-01 23:07:17 +08:00
committed by GitHub
parent c8016cbcdb
commit a08fb56607
3 changed files with 192 additions and 0 deletions
+30
View File
@@ -423,6 +423,33 @@ isolation_mode() {
fi
}
evidence_mode() {
if [[ "$DRY_RUN" == "true" ]]; then
echo dry-run
elif [[ "$EXTERNAL" != "true" ]]; then
echo local-formal
elif [[ -n "$DEPLOY_HOOK" ]]; then
echo external-deploy-hook-attested
else
echo external-unmanaged-nonformal
fi
}
formal_evidence() {
case "$(evidence_mode)" in
local-formal|external-deploy-hook-attested) echo true ;;
*) echo false ;;
esac
}
performance_conclusion() {
case "$(evidence_mode)" in
dry-run) echo not_measured_dry_run ;;
external-unmanaged-nonformal) echo not_formal_unmanaged_external ;;
*) echo not_claimed_gate_and_artifacts_required ;;
esac
}
write_manifest() {
local baseline_sha candidate_sha workloads matrix
baseline_sha="$(sha256_file "$BASELINE_BIN" 2>/dev/null || echo unknown)"
@@ -447,6 +474,9 @@ workloads=$workloads
drive_sync_matrix=$matrix
external=$EXTERNAL
external_isolation=$(isolation_mode)
evidence_mode=$(evidence_mode)
formal_evidence=$(formal_evidence)
performance_conclusion=$(performance_conclusion)
dataset_namespace=$DATASET_NAMESPACE
local_run_data_root=$RUN_DATA_ROOT
bucket_isolation=per-leg
+78
View File
@@ -54,6 +54,9 @@ rg -qx 'rounds=3' "$OUT_DIR/manifest.env"
rg -qx 'baseline_revision=baseline-test' "$OUT_DIR/manifest.env"
rg -qx 'candidate_revision=candidate-test' "$OUT_DIR/manifest.env"
rg -qx 'external_isolation=local-per-cell-disks' "$OUT_DIR/manifest.env"
rg -qx 'evidence_mode=dry-run' "$OUT_DIR/manifest.env"
rg -qx 'formal_evidence=false' "$OUT_DIR/manifest.env"
rg -qx 'performance_conclusion=not_measured_dry_run' "$OUT_DIR/manifest.env"
rg -qx 'bucket_isolation=per-leg' "$OUT_DIR/manifest.env"
rg -qx 'dataset_setup=get-and-mixed-via-warp-put' "$OUT_DIR/manifest.env"
[[ "$(rg -c -- '--extra-args --noclear' "$TRACE_FILE")" == "64" ]]
@@ -84,6 +87,43 @@ if "$RUNNER" \
exit 1
fi
"$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--baseline-revision baseline-test \
--candidate-revision candidate-test \
--endpoint 127.0.0.1:9000 \
--allow-unmanaged-external \
--warp-bin /usr/bin/true \
--rounds 3 \
--out-dir "${TMP_DIR}/unmanaged-external-dry-run" \
--dry-run >"${TMP_DIR}/unmanaged-external-trace.log" 2>&1
rg -qx 'external=true' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
rg -qx 'external_isolation=unmanaged' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
rg -qx 'evidence_mode=dry-run' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
rg -qx 'formal_evidence=false' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
rg -qx 'performance_conclusion=not_measured_dry_run' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
if "$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--baseline-revision baseline-test \
--candidate-revision candidate-test \
--endpoint 127.0.0.1:9 \
--allow-unmanaged-external \
--warp-bin /usr/bin/true \
--rounds 3 \
--health-timeout 1 \
--out-dir "${TMP_DIR}/unmanaged-external-nonformal" >/dev/null 2>&1; then
echo "expected unmanaged external mode to fail readiness in this test" >&2
exit 1
fi
rg -qx 'external=true' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
rg -qx 'external_isolation=unmanaged' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
rg -qx 'evidence_mode=external-unmanaged-nonformal' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
rg -qx 'formal_evidence=false' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
rg -qx 'performance_conclusion=not_formal_unmanaged_external' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
if "$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
@@ -109,5 +149,43 @@ if "$RUNNER" \
echo "expected a formal external deploy hook to write evidence" >&2
exit 1
fi
rg -qx 'external=true' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
rg -qx 'external_isolation=deploy-hook-attested' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
rg -qx 'evidence_mode=external-deploy-hook-attested' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
rg -qx 'formal_evidence=true' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
rg -qx 'performance_conclusion=not_claimed_gate_and_artifacts_required' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
BAD_EVIDENCE_HOOK="${TMP_DIR}/bad-evidence-hook.sh"
cat >"$BAD_EVIDENCE_HOOK" <<'HOOK'
#!/usr/bin/env bash
set -euo pipefail
cat >"$HOTPATH_ABBA_DEPLOY_EVIDENCE_FILE" <<EOF
leg=$HOTPATH_ABBA_LEG
phase=$HOTPATH_ABBA_PHASE
bucket=wrong-bucket
binary_sha256=$HOTPATH_ABBA_BINARY_SHA256
dataset_reset=true
EOF
HOOK
chmod +x "$BAD_EVIDENCE_HOOK"
if "$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--baseline-revision baseline-test \
--candidate-revision candidate-test \
--endpoint 127.0.0.1:9000 \
--deploy-hook "$BAD_EVIDENCE_HOOK" \
--warp-bin /usr/bin/true \
--rounds 3 \
--out-dir "${TMP_DIR}/bad-deploy-evidence" >/dev/null 2>&1; then
echo "expected a formal external deploy hook with wrong evidence to fail" >&2
exit 1
fi
rg -qx 'external=true' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
rg -qx 'external_isolation=deploy-hook-attested' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
rg -qx 'evidence_mode=external-deploy-hook-attested' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
rg -qx 'formal_evidence=true' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
rg -qx 'performance_conclusion=not_claimed_gate_and_artifacts_required' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
echo "hotpath warp ABBA tests passed"