diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 68b4be072..994777b49 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -3045,6 +3045,90 @@ mod tests { set_put_stage_metrics_enabled(false); } + #[test] + fn test_ec_encode_payload_stage_guards_are_noop_when_disabled() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + set_put_stage_metrics_enabled(false); + + let producer_current_before = current_ec_encode_producer_bytes(); + let producer_peak_before = current_ec_encode_producer_bytes_peak(); + let writer_current_before = current_ec_encode_writer_bytes(); + let writer_peak_before = current_ec_encode_writer_bytes_peak(); + + let producer = track_ec_encode_producer_bytes(1024); + let writer = track_ec_encode_writer_bytes(2048); + assert_eq!(current_ec_encode_producer_bytes(), producer_current_before); + assert_eq!(current_ec_encode_producer_bytes_peak(), producer_peak_before); + assert_eq!(current_ec_encode_writer_bytes(), writer_current_before); + assert_eq!(current_ec_encode_writer_bytes_peak(), writer_peak_before); + + drop((producer, writer)); + assert_eq!(current_ec_encode_producer_bytes(), producer_current_before); + assert_eq!(current_ec_encode_producer_bytes_peak(), producer_peak_before); + assert_eq!(current_ec_encode_writer_bytes(), writer_current_before); + assert_eq!(current_ec_encode_writer_bytes_peak(), writer_peak_before); + } + + fn assert_concurrent_ec_encode_stage_guards_aggregate_and_settle( + track: fn(usize) -> EcEncodePayloadStageGuard, + current: fn() -> u64, + peak: fn() -> u64, + ) { + const WORKERS: usize = 4; + const STAGE_BYTES: usize = 1536; + + let current_before = current(); + let peak_before = peak(); + let entered = Arc::new(Barrier::new(WORKERS + 1)); + let release = Arc::new(Barrier::new(WORKERS + 1)); + let mut workers = Vec::with_capacity(WORKERS); + + for _ in 0..WORKERS { + let entered = Arc::clone(&entered); + let release = Arc::clone(&release); + workers.push(std::thread::spawn(move || { + let stage = track(STAGE_BYTES); + entered.wait(); + release.wait(); + drop(stage); + })); + } + + entered.wait(); + let expected_delta = u64::try_from(WORKERS).expect("worker count should fit in u64") + * u64::try_from(STAGE_BYTES).expect("stage bytes should fit in u64"); + assert_eq!(current(), current_before + expected_delta); + assert!( + peak() >= peak_before.max(current_before + expected_delta), + "stage peak must expose process-wide concurrent ownership" + ); + + release.wait(); + for worker in workers { + worker.join().expect("stage worker should not panic"); + } + assert_eq!(current(), current_before); + } + + #[test] + fn test_ec_encode_payload_stage_guards_track_concurrent_ownership() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + set_put_stage_metrics_enabled(true); + + assert_concurrent_ec_encode_stage_guards_aggregate_and_settle( + track_ec_encode_producer_bytes, + current_ec_encode_producer_bytes, + current_ec_encode_producer_bytes_peak, + ); + assert_concurrent_ec_encode_stage_guards_aggregate_and_settle( + track_ec_encode_writer_bytes, + current_ec_encode_writer_bytes, + current_ec_encode_writer_bytes_peak, + ); + + set_put_stage_metrics_enabled(false); + } + #[test] fn test_ec_encode_producer_peak_exports_the_high_water_mark() { let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/scripts/run_hotpath_warp_abba.sh b/scripts/run_hotpath_warp_abba.sh index 480161b72..e7049c076 100755 --- a/scripts/run_hotpath_warp_abba.sh +++ b/scripts/run_hotpath_warp_abba.sh @@ -423,6 +423,33 @@ isolation_mode() { fi } +evidence_mode() { + if [[ "$DRY_RUN" == "true" ]]; then + echo dry-run + elif [[ "$EXTERNAL" != "true" ]]; then + echo local-formal + elif [[ -n "$DEPLOY_HOOK" ]]; then + echo external-deploy-hook-attested + else + echo external-unmanaged-nonformal + fi +} + +formal_evidence() { + case "$(evidence_mode)" in + local-formal|external-deploy-hook-attested) echo true ;; + *) echo false ;; + esac +} + +performance_conclusion() { + case "$(evidence_mode)" in + dry-run) echo not_measured_dry_run ;; + external-unmanaged-nonformal) echo not_formal_unmanaged_external ;; + *) echo not_claimed_gate_and_artifacts_required ;; + esac +} + write_manifest() { local baseline_sha candidate_sha workloads matrix baseline_sha="$(sha256_file "$BASELINE_BIN" 2>/dev/null || echo unknown)" @@ -447,6 +474,9 @@ workloads=$workloads drive_sync_matrix=$matrix external=$EXTERNAL external_isolation=$(isolation_mode) +evidence_mode=$(evidence_mode) +formal_evidence=$(formal_evidence) +performance_conclusion=$(performance_conclusion) dataset_namespace=$DATASET_NAMESPACE local_run_data_root=$RUN_DATA_ROOT bucket_isolation=per-leg diff --git a/scripts/test_hotpath_warp_abba.sh b/scripts/test_hotpath_warp_abba.sh index 5a24f97de..719f33241 100755 --- a/scripts/test_hotpath_warp_abba.sh +++ b/scripts/test_hotpath_warp_abba.sh @@ -54,6 +54,9 @@ rg -qx 'rounds=3' "$OUT_DIR/manifest.env" rg -qx 'baseline_revision=baseline-test' "$OUT_DIR/manifest.env" rg -qx 'candidate_revision=candidate-test' "$OUT_DIR/manifest.env" rg -qx 'external_isolation=local-per-cell-disks' "$OUT_DIR/manifest.env" +rg -qx 'evidence_mode=dry-run' "$OUT_DIR/manifest.env" +rg -qx 'formal_evidence=false' "$OUT_DIR/manifest.env" +rg -qx 'performance_conclusion=not_measured_dry_run' "$OUT_DIR/manifest.env" rg -qx 'bucket_isolation=per-leg' "$OUT_DIR/manifest.env" rg -qx 'dataset_setup=get-and-mixed-via-warp-put' "$OUT_DIR/manifest.env" [[ "$(rg -c -- '--extra-args --noclear' "$TRACE_FILE")" == "64" ]] @@ -84,6 +87,43 @@ if "$RUNNER" \ exit 1 fi +"$RUNNER" \ + --baseline-bin /usr/bin/true \ + --candidate-bin /usr/bin/true \ + --baseline-revision baseline-test \ + --candidate-revision candidate-test \ + --endpoint 127.0.0.1:9000 \ + --allow-unmanaged-external \ + --warp-bin /usr/bin/true \ + --rounds 3 \ + --out-dir "${TMP_DIR}/unmanaged-external-dry-run" \ + --dry-run >"${TMP_DIR}/unmanaged-external-trace.log" 2>&1 +rg -qx 'external=true' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env" +rg -qx 'external_isolation=unmanaged' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env" +rg -qx 'evidence_mode=dry-run' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env" +rg -qx 'formal_evidence=false' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env" +rg -qx 'performance_conclusion=not_measured_dry_run' "${TMP_DIR}/unmanaged-external-dry-run/manifest.env" + +if "$RUNNER" \ + --baseline-bin /usr/bin/true \ + --candidate-bin /usr/bin/true \ + --baseline-revision baseline-test \ + --candidate-revision candidate-test \ + --endpoint 127.0.0.1:9 \ + --allow-unmanaged-external \ + --warp-bin /usr/bin/true \ + --rounds 3 \ + --health-timeout 1 \ + --out-dir "${TMP_DIR}/unmanaged-external-nonformal" >/dev/null 2>&1; then + echo "expected unmanaged external mode to fail readiness in this test" >&2 + exit 1 +fi +rg -qx 'external=true' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env" +rg -qx 'external_isolation=unmanaged' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env" +rg -qx 'evidence_mode=external-unmanaged-nonformal' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env" +rg -qx 'formal_evidence=false' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env" +rg -qx 'performance_conclusion=not_formal_unmanaged_external' "${TMP_DIR}/unmanaged-external-nonformal/manifest.env" + if "$RUNNER" \ --baseline-bin /usr/bin/true \ --candidate-bin /usr/bin/true \ @@ -109,5 +149,43 @@ if "$RUNNER" \ echo "expected a formal external deploy hook to write evidence" >&2 exit 1 fi +rg -qx 'external=true' "${TMP_DIR}/missing-deploy-evidence/manifest.env" +rg -qx 'external_isolation=deploy-hook-attested' "${TMP_DIR}/missing-deploy-evidence/manifest.env" +rg -qx 'evidence_mode=external-deploy-hook-attested' "${TMP_DIR}/missing-deploy-evidence/manifest.env" +rg -qx 'formal_evidence=true' "${TMP_DIR}/missing-deploy-evidence/manifest.env" +rg -qx 'performance_conclusion=not_claimed_gate_and_artifacts_required' "${TMP_DIR}/missing-deploy-evidence/manifest.env" + +BAD_EVIDENCE_HOOK="${TMP_DIR}/bad-evidence-hook.sh" +cat >"$BAD_EVIDENCE_HOOK" <<'HOOK' +#!/usr/bin/env bash +set -euo pipefail +cat >"$HOTPATH_ABBA_DEPLOY_EVIDENCE_FILE" </dev/null 2>&1; then + echo "expected a formal external deploy hook with wrong evidence to fail" >&2 + exit 1 +fi +rg -qx 'external=true' "${TMP_DIR}/bad-deploy-evidence/manifest.env" +rg -qx 'external_isolation=deploy-hook-attested' "${TMP_DIR}/bad-deploy-evidence/manifest.env" +rg -qx 'evidence_mode=external-deploy-hook-attested' "${TMP_DIR}/bad-deploy-evidence/manifest.env" +rg -qx 'formal_evidence=true' "${TMP_DIR}/bad-deploy-evidence/manifest.env" +rg -qx 'performance_conclusion=not_claimed_gate_and_artifacts_required' "${TMP_DIR}/bad-deploy-evidence/manifest.env" echo "hotpath warp ABBA tests passed"