test(perf): strengthen HotPath evidence contracts (#5581)

* test(perf): mark nonformal ABBA artifacts Co-Authored-By: heihutu <heihutu@gmail.com> * test(io-metrics): lock disabled EC stage accounting Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-01 19:12:14 +00:00 · 2026-08-01 23:07:17 +08:00
parent c8016cbcdb
commit a08fb56607
3 changed files with 192 additions and 0 deletions
@@ -3045,6 +3045,90 @@ mod tests {
        set_put_stage_metrics_enabled(false);
    }

+    #[test]
+    fn test_ec_encode_payload_stage_guards_are_noop_when_disabled() {
+        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+        set_put_stage_metrics_enabled(false);
+
+        let producer_current_before = current_ec_encode_producer_bytes();
+        let producer_peak_before = current_ec_encode_producer_bytes_peak();
+        let writer_current_before = current_ec_encode_writer_bytes();
+        let writer_peak_before = current_ec_encode_writer_bytes_peak();
+
+        let producer = track_ec_encode_producer_bytes(1024);
+        let writer = track_ec_encode_writer_bytes(2048);
+        assert_eq!(current_ec_encode_producer_bytes(), producer_current_before);
+        assert_eq!(current_ec_encode_producer_bytes_peak(), producer_peak_before);
+        assert_eq!(current_ec_encode_writer_bytes(), writer_current_before);
+        assert_eq!(current_ec_encode_writer_bytes_peak(), writer_peak_before);
+
+        drop((producer, writer));
+        assert_eq!(current_ec_encode_producer_bytes(), producer_current_before);
+        assert_eq!(current_ec_encode_producer_bytes_peak(), producer_peak_before);
+        assert_eq!(current_ec_encode_writer_bytes(), writer_current_before);
+        assert_eq!(current_ec_encode_writer_bytes_peak(), writer_peak_before);
+    }
+
+    fn assert_concurrent_ec_encode_stage_guards_aggregate_and_settle(
+        track: fn(usize) -> EcEncodePayloadStageGuard,
+        current: fn() -> u64,
+        peak: fn() -> u64,
+    ) {
+        const WORKERS: usize = 4;
+        const STAGE_BYTES: usize = 1536;
+
+        let current_before = current();
+        let peak_before = peak();
+        let entered = Arc::new(Barrier::new(WORKERS + 1));
+        let release = Arc::new(Barrier::new(WORKERS + 1));
+        let mut workers = Vec::with_capacity(WORKERS);
+
+        for _ in 0..WORKERS {
+            let entered = Arc::clone(&entered);
+            let release = Arc::clone(&release);
+            workers.push(std::thread::spawn(move || {
+                let stage = track(STAGE_BYTES);
+                entered.wait();
+                release.wait();
+                drop(stage);
+            }));
+        }
+
+        entered.wait();
+        let expected_delta = u64::try_from(WORKERS).expect("worker count should fit in u64")
+            * u64::try_from(STAGE_BYTES).expect("stage bytes should fit in u64");
+        assert_eq!(current(), current_before + expected_delta);
+        assert!(
+            peak() >= peak_before.max(current_before + expected_delta),
+            "stage peak must expose process-wide concurrent ownership"
+        );
+
+        release.wait();
+        for worker in workers {
+            worker.join().expect("stage worker should not panic");
+        }
+        assert_eq!(current(), current_before);
+    }
+
+    #[test]
+    fn test_ec_encode_payload_stage_guards_track_concurrent_ownership() {
+        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
+        set_put_stage_metrics_enabled(true);
+
+        assert_concurrent_ec_encode_stage_guards_aggregate_and_settle(
+            track_ec_encode_producer_bytes,
+            current_ec_encode_producer_bytes,
+            current_ec_encode_producer_bytes_peak,
+        );
+        assert_concurrent_ec_encode_stage_guards_aggregate_and_settle(
+            track_ec_encode_writer_bytes,
+            current_ec_encode_writer_bytes,
+            current_ec_encode_writer_bytes_peak,
+        );
+
+        set_put_stage_metrics_enabled(false);
+    }
+
    #[test]
    fn test_ec_encode_producer_peak_exports_the_high_water_mark() {
        let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
@@ -423,6 +423,33 @@ isolation_mode() {
  fi
 }

+evidence_mode() {
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo dry-run
+  elif [[ "$EXTERNAL" != "true" ]]; then
+    echo local-formal
+  elif [[ -n "$DEPLOY_HOOK" ]]; then
+    echo external-deploy-hook-attested
+  else
+    echo external-unmanaged-nonformal
+  fi
+}
+
+formal_evidence() {
+  case "$(evidence_mode)" in
+    local-formal|external-deploy-hook-attested) echo true ;;
+    *) echo false ;;
+  esac
+}
+
+performance_conclusion() {
+  case "$(evidence_mode)" in
+    dry-run) echo not_measured_dry_run ;;
+    external-unmanaged-nonformal) echo not_formal_unmanaged_external ;;
+    *) echo not_claimed_gate_and_artifacts_required ;;
+  esac
+}
+
 write_manifest() {
  local baseline_sha candidate_sha workloads matrix
  baseline_sha="$(sha256_file "$BASELINE_BIN" 2>/dev/null || echo unknown)"
@@ -447,6 +474,9 @@ workloads=$workloads
 drive_sync_matrix=$matrix
 external=$EXTERNAL
 external_isolation=$(isolation_mode)
+evidence_mode=$(evidence_mode)
+formal_evidence=$(formal_evidence)
+performance_conclusion=$(performance_conclusion)
 dataset_namespace=$DATASET_NAMESPACE
 local_run_data_root=$RUN_DATA_ROOT
 bucket_isolation=per-leg
@@ -54,6 +54,9 @@ rg -qx 'rounds=3' "$OUT_DIR/manifest.env"
 rg -qx 'baseline_revision=baseline-test' "$OUT_DIR/manifest.env"
 rg -qx 'candidate_revision=candidate-test' "$OUT_DIR/manifest.env"
 rg -qx 'external_isolation=local-per-cell-disks' "$OUT_DIR/manifest.env"
+rg -qx 'evidence_mode=dry-run' "$OUT_DIR/manifest.env"
+rg -qx 'formal_evidence=false' "$OUT_DIR/manifest.env"
+rg -qx 'performance_conclusion=not_measured_dry_run' "$OUT_DIR/manifest.env"
 rg -qx 'bucket_isolation=per-leg' "$OUT_DIR/manifest.env"
 rg -qx 'dataset_setup=get-and-mixed-via-warp-put' "$OUT_DIR/manifest.env"
 [[ "$(rg -c -- '--extra-args --noclear' "$TRACE_FILE")" == "64" ]]
@@ -84,6 +87,43 @@ if "$RUNNER" \
  exit 1
 fi

+"$RUNNER" \
+  --baseline-bin /usr/bin/true \
+  --candidate-bin /usr/bin/true \
+  --baseline-revision baseline-test \
+  --candidate-revision candidate-test \
+  --endpoint 127.0.0.1:9000 \
+  --allow-unmanaged-external \
+  --warp-bin /usr/bin/true \
+  --rounds 3 \
+  --out-dir "${TMP_DIR}/unmanaged-external-dry-run" \
+  --dry-run >"${TMP_DIR}/unmanaged-external-trace.log" 2>&1
+rg -qx 'external=true' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
+rg -qx 'external_isolation=unmanaged' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
+rg -qx 'evidence_mode=dry-run' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
+rg -qx 'formal_evidence=false' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
+rg -qx 'performance_conclusion=not_measured_dry_run' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env"
+
+if "$RUNNER" \
+  --baseline-bin /usr/bin/true \
+  --candidate-bin /usr/bin/true \
+  --baseline-revision baseline-test \
+  --candidate-revision candidate-test \
+  --endpoint 127.0.0.1:9 \
+  --allow-unmanaged-external \
+  --warp-bin /usr/bin/true \
+  --rounds 3 \
+  --health-timeout 1 \
+  --out-dir "${TMP_DIR}/unmanaged-external-nonformal" >/dev/null 2>&1; then
+  echo "expected unmanaged external mode to fail readiness in this test" >&2
+  exit 1
+fi
+rg -qx 'external=true' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
+rg -qx 'external_isolation=unmanaged' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
+rg -qx 'evidence_mode=external-unmanaged-nonformal' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
+rg -qx 'formal_evidence=false' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
+rg -qx 'performance_conclusion=not_formal_unmanaged_external' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env"
+
 if "$RUNNER" \
  --baseline-bin /usr/bin/true \
  --candidate-bin /usr/bin/true \
@@ -109,5 +149,43 @@ if "$RUNNER" \
  echo "expected a formal external deploy hook to write evidence" >&2
  exit 1
 fi
+rg -qx 'external=true' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
+rg -qx 'external_isolation=deploy-hook-attested' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
+rg -qx 'evidence_mode=external-deploy-hook-attested' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
+rg -qx 'formal_evidence=true' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
+rg -qx 'performance_conclusion=not_claimed_gate_and_artifacts_required' "${TMP_DIR}/missing-deploy-evidence/manifest.env"
+
+BAD_EVIDENCE_HOOK="${TMP_DIR}/bad-evidence-hook.sh"
+cat >"$BAD_EVIDENCE_HOOK" <<'HOOK'
+#!/usr/bin/env bash
+set -euo pipefail
+cat >"$HOTPATH_ABBA_DEPLOY_EVIDENCE_FILE" <<EOF
+leg=$HOTPATH_ABBA_LEG
+phase=$HOTPATH_ABBA_PHASE
+bucket=wrong-bucket
+binary_sha256=$HOTPATH_ABBA_BINARY_SHA256
+dataset_reset=true
+EOF
+HOOK
+chmod +x "$BAD_EVIDENCE_HOOK"
+
+if "$RUNNER" \
+  --baseline-bin /usr/bin/true \
+  --candidate-bin /usr/bin/true \
+  --baseline-revision baseline-test \
+  --candidate-revision candidate-test \
+  --endpoint 127.0.0.1:9000 \
+  --deploy-hook "$BAD_EVIDENCE_HOOK" \
+  --warp-bin /usr/bin/true \
+  --rounds 3 \
+  --out-dir "${TMP_DIR}/bad-deploy-evidence" >/dev/null 2>&1; then
+  echo "expected a formal external deploy hook with wrong evidence to fail" >&2
+  exit 1
+fi
+rg -qx 'external=true' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
+rg -qx 'external_isolation=deploy-hook-attested' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
+rg -qx 'evidence_mode=external-deploy-hook-attested' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
+rg -qx 'formal_evidence=true' "${TMP_DIR}/bad-deploy-evidence/manifest.env"
+rg -qx 'performance_conclusion=not_claimed_gate_and_artifacts_required' "${TMP_DIR}/bad-deploy-evidence/manifest.env"

 echo "hotpath warp ABBA tests passed"