fix(ci): make Warp ABBA evidence bounded and complete (#6417)

.github/workflows/performance-ab.yml

@@ -121,14 +121,14 @@ jobs:
           candidate_sha="$(git rev-parse HEAD)"
           if [[ "${{ github.event_name }}" == "schedule" ]]; then
             baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
-            if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
-              echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
-              exit 1
-            fi
           else
             baseline_sha="$(git rev-parse origin/main)"
           fi
           git cat-file -e "${baseline_sha}^{commit}"
+          if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
+            echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
+            exit 1
+          fi
           echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
           echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
           echo "baseline commit: $baseline_sha"
@@ -342,6 +342,10 @@ jobs:
         if: always()
         run: |
           status="${{ steps.ab.outputs.status }}"
+          if [[ -z "$status" ]]; then
+            echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
+            exit 1
+          fi
           if [[ "$status" != "0" ]]; then
             echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
             exit "$status"

scripts/hotpath_warp_ab_gate.sh

@@ -30,12 +30,16 @@ REQUIRE_TAIL_ERROR="false"
 MARKDOWN_OUT=""
 EXEMPTION_REASON="deliberate correctness tradeoff"
 declare -a COMPARE_CSVS=()
+declare -a COMPARE_LABELS=()
 
 usage() {
   cat <<'USAGE'
 Usage: hotpath_warp_ab_gate.sh --compare-csv <file> [--compare-csv <file> ...] [options]
 
   --compare-csv <file>   baseline_compare.csv to evaluate (repeatable).
+  --labeled-compare-csv <label> <file>
+                         Evaluate a CSV and identify its configuration in the
+                         result table (repeatable).
   --fail-pct <n>         Regression budget that fails the gate (default 10).
   --warn-pct <n>         Regression budget that warns (default 5).
   --allow-regression     Downgrade every FAIL to an exempted WARN (deliberate
@@ -50,7 +54,12 @@ USAGE
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
-    --compare-csv) COMPARE_CSVS+=("$2"); shift 2 ;;
+    --compare-csv) COMPARE_CSVS+=("$2"); COMPARE_LABELS+=(""); shift 2 ;;
+    --labeled-compare-csv)
+      COMPARE_LABELS+=("$2")
+      COMPARE_CSVS+=("$3")
+      shift 3
+      ;;
     --fail-pct) FAIL_PCT="$2"; shift 2 ;;
     --warn-pct) WARN_PCT="$2"; shift 2 ;;
     --allow-regression) ALLOW_REGRESSION="true"; shift ;;
@@ -85,10 +94,19 @@ if [[ "$REQUIRE_TAIL_ERROR" == "true" ]]; then
   done
 fi
 
-# One awk pass over all CSVs. Emits TSV rows "verdict\tworkload\tmetric\tdelta"
+has_labels="false"
+for label in "${COMPARE_LABELS[@]}"; do
+  [[ -n "$label" ]] && has_labels="true"
+done
+label_separator=$'\034'
+compare_labels="$(IFS="$label_separator"; echo "${COMPARE_LABELS[*]}")"
+
+# One awk pass over all CSVs. Emits TSV rows
+# "verdict\tconfiguration\tworkload\tmetric\tdelta"
 # on stdout and a final "OVERALL\t<verdict>" line. Verdict is PASS/WARN/FAIL.
 gate_output="$(
-  awk -v fail_pct="$FAIL_PCT" -v warn_pct="$WARN_PCT" -v strict="$REQUIRE_TAIL_ERROR" '
+  awk -v fail_pct="$FAIL_PCT" -v warn_pct="$WARN_PCT" -v strict="$REQUIRE_TAIL_ERROR" \
+      -v label_blob="$compare_labels" -v label_separator="$label_separator" '
     function classify(delta, higher_is_better,   regress) {
       if (delta == "" || delta == "N/A") return "SKIP"
       # Signed regression magnitude: positive means "worse than baseline".
@@ -97,14 +115,14 @@ gate_output="$(
       if (regress > warn_pct) return "WARN"
       return "PASS"
     }
-    function emit(v, workload, metric, delta,   rank) {
+    function emit(v, configuration, workload, metric, delta,   rank) {
       if (v == "SKIP") return
-      print v "\t" workload "\t" metric "\t" delta "%"
+      print v "\t" configuration "\t" workload "\t" metric "\t" delta "%"
       rank = (v == "FAIL") ? 3 : (v == "WARN") ? 2 : 1
       if (rank > worst) worst = rank
     }
-    function contract_failure(workload, metric) {
-      print "FAIL\t" workload "\t" metric "\tinvalid evidence"
+    function contract_failure(configuration, workload, metric) {
+      print "FAIL\t" configuration "\t" workload "\t" metric "\tinvalid evidence"
       if (3 > worst) worst = 3
     }
     function decimal(value) {
@@ -122,18 +140,26 @@ gate_output="$(
     function abs(value) {
       return value < 0 ? -value : value
     }
-    BEGIN { worst = 1 }
-    FNR == 1 { next }                      # skip each file header
+    BEGIN {
+      worst = 1
+      split(label_blob, file_labels, label_separator)
+    }
+    FNR == 1 {
+      file_index++
+      configuration = file_labels[file_index]
+      if (configuration == "") configuration = "-"
+      next
+    }
     {
       n = split($0, f, ",")
       if (n < 12) {
-        if (strict == "true") contract_failure(FILENAME, "compare-schema")
+        if (strict == "true") contract_failure(configuration, FILENAME, "compare-schema")
         next
       }
       workload = f[1] "/" f[2] "@" f[3]    # size/tool@concurrency
-      emit(classify(f[6],  1), workload, "reqps",      f[6])
-      emit(classify(f[9],  0), workload, "latency",    f[9])
-      emit(classify(f[12], 1), workload, "throughput", f[12])
+      emit(classify(f[6],  1), configuration, workload, "reqps",      f[6])
+      emit(classify(f[9],  0), configuration, workload, "latency",    f[9])
+      emit(classify(f[12], 1), configuration, workload, "throughput", f[12])
       if (strict == "true") {
         valid = n == 25
         for (i = 4; i <= 18; i++) {
@@ -147,12 +173,12 @@ gate_output="$(
         baseline_error_rate = f[22] / (f[20] + f[22]) * 100
         if (abs(f[23] - new_error_rate) > 0.005 || abs(f[24] - baseline_error_rate) > 0.005 || abs(f[25] - (new_error_rate - baseline_error_rate)) > 0.005) valid = 0
         if (!valid) {
-          contract_failure(workload, "tail-error-evidence")
+          contract_failure(configuration, workload, "tail-error-evidence")
           next
         }
-        emit(classify(f[15], 0), workload, "p90-latency", f[15])
-        emit(classify(f[18], 0), workload, "p99-latency", f[18])
-        emit(classify(f[25], 0), workload, "error-rate", f[25])
+        emit(classify(f[15], 0), configuration, workload, "p90-latency", f[15])
+        emit(classify(f[18], 0), configuration, workload, "p99-latency", f[18])
+        emit(classify(f[25], 0), configuration, workload, "error-rate", f[25])
       }
     }
     END {
@@ -182,14 +208,27 @@ render() {
     echo "> Gate FAIL exempted via \`--allow-regression\`: ${EXEMPTION_REASON}."
   fi
   echo
-  echo "| Workload | Metric | Δ vs baseline | Verdict |"
-  echo "| --- | --- | --- | --- |"
-  if [[ -z "$rows" ]]; then
-    echo "| _(no rows)_ | | | |"
+  if [[ "$has_labels" == "true" ]]; then
+    echo "| Configuration | Workload | Metric | Δ vs baseline | Verdict |"
+    echo "| --- | --- | --- | --- | --- |"
   else
-    printf '%s\n' "$rows" | while IFS=$'\t' read -r v workload metric delta; do
+    echo "| Workload | Metric | Δ vs baseline | Verdict |"
+    echo "| --- | --- | --- | --- |"
+  fi
+  if [[ -z "$rows" ]]; then
+    if [[ "$has_labels" == "true" ]]; then
+      echo "| _(no rows)_ | | | | |"
+    else
+      echo "| _(no rows)_ | | | |"
+    fi
+  else
+    printf '%s\n' "$rows" | while IFS=$'\t' read -r v configuration workload metric delta; do
       [[ -z "$v" ]] && continue
-      echo "| $workload | $metric | $delta | $(emoji "$v") $v |"
+      if [[ "$has_labels" == "true" ]]; then
+        echo "| $configuration | $workload | $metric | $delta | $(emoji "$v") $v |"
+      else
+        echo "| $workload | $metric | $delta | $(emoji "$v") $v |"
+      fi
     done
   fi
 }

scripts/run_hotpath_warp_abba.sh

@@ -34,8 +34,8 @@ CONCURRENCY=8
 DURATION="60s"
 ROUNDS=3
 COOLDOWN_SECS=20
-DATASET_SETUP_DURATION="10s"
 HEALTH_TIMEOUT_SECS=180
+DATASET_OBJECTS_PER_WORKER=8
 FAIL_PCT=10
 WARN_PCT=5
 ALLOW_REGRESSION=false
@@ -100,9 +100,6 @@ Benchmark:
   --duration <dur>          warp duration per cell (default 60s).
   --rounds <n>              rounds per cell; must be >= 3 (default 3).
   --cooldown <n>            cooldown seconds between rounds/sizes (default 20).
-  --dataset-setup-duration <dur>
-                            isolated Warp PUT warm-up for get/mixed legs
-                            (default 10s; not included in the measurement).
   --concurrency <n>         warp concurrency (default 8).
   --warp-bin <path>         warp binary (default warp).
 
@@ -173,7 +170,6 @@ while [[ $# -gt 0 ]]; do
     --duration) DURATION="$2"; shift 2 ;;
     --rounds) ROUNDS="$2"; shift 2 ;;
     --cooldown) COOLDOWN_SECS="$2"; shift 2 ;;
-    --dataset-setup-duration) DATASET_SETUP_DURATION="$2"; shift 2 ;;
     --health-timeout) HEALTH_TIMEOUT_SECS="$2"; shift 2 ;;
     --fail-pct) FAIL_PCT="$2"; shift 2 ;;
     --warn-pct) WARN_PCT="$2"; shift 2 ;;
@@ -366,29 +362,18 @@ measure() {
     --duration "$DURATION" --rounds "$ROUNDS" --cooldown-secs "$COOLDOWN_SECS"
     --out-dir "$cell"
   )
-  [[ "$mode" == "put" ]] || args+=(--extra-args "--noclear")
+  if [[ "$mode" != "put" ]]; then
+    # Warp defaults to 2,500 setup objects per round. At 10 MiB that writes
+    # 25 GiB before every 12-second measurement, so the matrix cannot finish
+    # inside the workflow budget. Eight objects per worker keeps preparation
+    # bounded while retaining a multi-object working set for relative A/B.
+    args+=(--extra-args "--objects $((CONCURRENCY * DATASET_OBJECTS_PER_WORKER)) --noclear")
+  fi
   [[ -n "$baseline_csv" ]] && args+=(--baseline-csv "$baseline_csv")
   run "$ENHANCED_BENCH" "${args[@]}" >&2
   echo "$cell"
 }
 
-prepare_dataset() {
-  local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" bucket="$6"
-  [[ "$mode" != "put" ]] || return 0
-
-  local setup_cell="$OUT_DIR/$workload/$sync_label/$leg/dataset-setup"
-  local args=(
-    --tool warp --warp-bin "$WARP_BIN" --warp-mode put
-    --endpoint "$ADDRESS" --access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY"
-    --region "$REGION" --bucket "$bucket" --sizes "$size" --concurrency "$CONCURRENCY"
-    --duration "$DATASET_SETUP_DURATION" --rounds 1 --cooldown-secs 0
-    --extra-args "--noclear"
-    --out-dir "$setup_cell"
-  )
-  log "preparing isolated dataset: $sync_label/$workload/$leg bucket=$bucket"
-  run "$ENHANCED_BENCH" "${args[@]}" >&2
-}
-
 write_schedule_header() {
   echo "sync_label,drive_sync,workload,mode,size,leg,phase,binary,out_dir,bucket,dataset_setup" >"$OUT_DIR/abba_schedule.csv"
 }
@@ -399,7 +384,7 @@ append_schedule() {
   phase="$(phase_for_leg "$leg")"
   bin="$(binary_for_leg "$leg")"
   local dataset_setup="none"
-  [[ "$mode" == "put" ]] || dataset_setup="warp-put"
+  [[ "$mode" == "put" ]] || dataset_setup="warp-native-bounded"
   echo "$sync_label,$drive_sync,$workload,$mode,$size,$leg,$phase,$bin,$OUT_DIR/$workload/$sync_label/$leg,$bucket,$dataset_setup" >>"$OUT_DIR/abba_schedule.csv"
 }
 
@@ -481,7 +466,8 @@ dataset_namespace=$DATASET_NAMESPACE
 local_run_data_root=$RUN_DATA_ROOT
 bucket_isolation=per-leg
 bucket_prefix=rustfs-abba-$DATASET_NAMESPACE
-dataset_setup=get-and-mixed-via-warp-put
+dataset_setup=get-and-mixed-via-bounded-warp-native
+dataset_objects=$((CONCURRENCY * DATASET_OBJECTS_PER_WORKER))
 endpoint=$ADDRESS
 warp_version=$("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown)
 EOF
@@ -489,6 +475,8 @@ EOF
 
 declare -a CANDIDATE_COMPARE_CSVS=()
 declare -a DRIFT_COMPARE_CSVS=()
+declare -a CANDIDATE_COMPARE_LABELS=()
+declare -a DRIFT_COMPARE_LABELS=()
 
 write_manifest
 write_schedule_header
@@ -502,7 +490,6 @@ for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do
       log "=== $sync_label $workload leg $leg ($(phase_for_leg "$leg")) ==="
       bucket="$(bucket_for_leg "$sync_label" "$workload" "$leg")"
       bring_up "$leg" "$drive_sync" "$workload" "$mode" "$size" "$sync_label" "$bucket"
-      prepare_dataset "$leg" "$workload" "$mode" "$size" "$sync_label" "$bucket"
       append_schedule "$sync_label" "$drive_sync" "$workload" "$mode" "$size" "$leg" "$bucket"
 
       baseline_csv=""
@@ -512,8 +499,14 @@ for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do
 
       cell="$(measure "$leg" "$workload" "$mode" "$size" "$sync_label" "$bucket" "$baseline_csv")"
       case "$leg" in
-        B1|B2) CANDIDATE_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;;
-        A2) DRIFT_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;;
+        B1|B2)
+          CANDIDATE_COMPARE_CSVS+=("$cell/baseline_compare.csv")
+          CANDIDATE_COMPARE_LABELS+=("$sync_label/$workload/$leg-vs-A1")
+          ;;
+        A2)
+          DRIFT_COMPARE_CSVS+=("$cell/baseline_compare.csv")
+          DRIFT_COMPARE_LABELS+=("$sync_label/$workload/A2-vs-A1")
+          ;;
       esac
       tear_down
     done
@@ -521,14 +514,14 @@ for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do
 done
 
 gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --require-tail-error --markdown "$OUT_DIR/candidate_gate.md")
-for csv in "${CANDIDATE_COMPARE_CSVS[@]}"; do
-  gate_args+=(--compare-csv "$csv")
+for i in "${!CANDIDATE_COMPARE_CSVS[@]}"; do
+  gate_args+=(--labeled-compare-csv "${CANDIDATE_COMPARE_LABELS[$i]}" "${CANDIDATE_COMPARE_CSVS[$i]}")
 done
 [[ "$ALLOW_REGRESSION" == "true" ]] && gate_args+=(--allow-regression --exemption-reason "$EXEMPTION_REASON")
 
 drift_gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --require-tail-error --markdown "$OUT_DIR/baseline_drift_gate.md")
-for csv in "${DRIFT_COMPARE_CSVS[@]}"; do
-  drift_gate_args+=(--compare-csv "$csv")
+for i in "${!DRIFT_COMPARE_CSVS[@]}"; do
+  drift_gate_args+=(--labeled-compare-csv "${DRIFT_COMPARE_LABELS[$i]}" "${DRIFT_COMPARE_CSVS[$i]}")
 done
 
 if [[ "$DRY_RUN" == "true" ]]; then

scripts/run_object_batch_bench_enhanced.sh

@@ -459,7 +459,7 @@ write_manifest_entry() {
 
 write_run_manifest() {
   local manifest_file="$OUT_DIR/run_manifest.env"
-  local started_at_utc git_commit git_branch git_dirty rustc_version uname_s service_metrics_csv
+  local started_at_utc git_commit git_branch git_dirty rustc_version uname_s service_metrics_csv report_operation
   started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
   git_commit="$(git_value unknown rev-parse --short HEAD)"
   git_branch="$(git_value unknown branch --show-current)"
@@ -467,6 +467,10 @@ write_run_manifest() {
   rustc_version="$(rustc --version 2>/dev/null || echo unknown)"
   uname_s="$(uname -a 2>/dev/null || echo unknown)"
   service_metrics_csv="${SERVICE_METRICS_CSV:-}"
+  report_operation="N/A"
+  if [[ "$TOOL" == "warp" ]]; then
+    report_operation="$(warp_report_operation)"
+  fi
 
   cat >"$manifest_file" <<EOF
 started_at_utc=${started_at_utc}
@@ -494,6 +498,7 @@ retry_sleep_secs=${RETRY_SLEEP_SECS}
 cooldown_secs=${COOLDOWN_SECS}
 warp_bin=${WARP_BIN}
 warp_mode=${WARP_MODE}
+warp_report_operation=${report_operation}
 duration=${DURATION}
 s3bench_bin=${S3BENCH_BIN}
 samples=${SAMPLES}
@@ -662,24 +667,23 @@ to_ms() {
   awk -v n="$number" -v f="$factor" 'BEGIN { printf "%.6f\n", n * f }'
 }
 
-extract_first() {
-  local regex="$1"
-  local file="$2"
-  rg -o "$regex" "$file" | head -n1 || true
+warp_report_operation() {
+  case "$WARP_MODE" in
+    get) echo GET ;;
+    put|mixed) echo PUT ;;
+  esac
 }
 
-extract_report_line() {
-  local regex="$1"
+extract_warp_report() {
+  local operation="$1"
   local file="$2"
-  awk -v regex="$regex" '
-    /^Report:/ {
-      in_report = 1
+  awk -v operation="$operation" '
+    /^(Report|Operation):/ {
+      target = "^(Report|Operation):[[:space:]]*" operation "([[:space:].-]|$)"
+      in_report = ($0 ~ target || $0 ~ /^Report:[[:space:]]*$/)
       next
     }
-    in_report && $0 ~ regex {
-      print
-      exit
-    }
+    in_report { print }
   ' "$file"
 }
 
@@ -703,30 +707,35 @@ normalize_duration_metric() {
 extract_metrics() {
   local log_file="$1"
 
-  local average_line reqs_line throughput reqps latency req_p90 req_p99 reqps_num
-  average_line="$(extract_report_line '^[[:space:]]*[*][[:space:]]+Average:' "$log_file")"
-  reqs_line="$(extract_report_line '^[[:space:]]*[*][[:space:]]+Reqs:' "$log_file")"
+  local report average_line throughput_line request_line throughput reqps latency req_p90 req_p99 reqps_num
+  if [[ "$TOOL" == "warp" ]]; then
+    report="$(extract_warp_report "$(warp_report_operation)" "$log_file")"
+  else
+    report="$(<"$log_file")"
+  fi
+  average_line="$(printf '%s\n' "$report" | awk '/^[[:space:]]*[*][[:space:]]+Average:/ { print; exit }')"
+  throughput_line="$(printf '%s\n' "$report" | awk '/^[[:space:]]*[*][[:space:]]+Throughput:/ { print; exit }')"
+  request_line="$(printf '%s\n' "$report" | awk '/^[[:space:]]*[*][[:space:]]+(Reqs:[[:space:]]+)?Avg:/ { print; exit }')"
 
   if [[ -n "$average_line" ]]; then
     throughput="$(echo "$average_line" | sed -E 's/^.*Average:[[:space:]]*//; s/,[[:space:]]*.*$//')"
     reqps="$(echo "$average_line" | sed -E 's/^.*Average:[[:space:]]*[^,]+,[[:space:]]*//; s/[[:space:]]*$//')"
+  elif [[ -n "$throughput_line" ]]; then
+    throughput="$(printf '%s\n' "$throughput_line" | rg -o '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' | head -n1 || true)"
+    reqps="$(printf '%s\n' "$throughput_line" | rg -o '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' | head -n1 || true)"
   else
-    throughput="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")"
-    reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")"
+    throughput="$(printf '%s\n' "$report" | rg -o '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' | head -n1 || true)"
+    reqps="$(printf '%s\n' "$report" | rg -o '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' | head -n1 || true)"
   fi
 
-  if [[ -n "$reqs_line" ]]; then
-    latency="$(echo "$reqs_line" | rg -o 'Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^Avg:[[:space:]]+//')"
-    req_p90="$(echo "$reqs_line" | rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^90%:[[:space:]]+//')"
-    req_p99="$(echo "$reqs_line" | rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^99%:[[:space:]]+//')"
+  if [[ -n "$request_line" ]]; then
+    latency="$(echo "$request_line" | rg -o 'Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^Avg:[[:space:]]+//')"
+    req_p90="$(echo "$request_line" | rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^90%:[[:space:]]+//')"
+    req_p99="$(echo "$request_line" | rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | sed -E 's/^99%:[[:space:]]+//')"
   else
-    latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')"
-    req_p90="$(rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^90%:[[:space:]]+//')"
-    req_p99="$(rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^99%:[[:space:]]+//')"
-  fi
-
-  if [[ -z "$latency" ]]; then
-    latency="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(ms|us|µs|s)' "$log_file")"
+    latency="$(printf '%s\n' "$report" | rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//' || true)"
+    req_p90="$(printf '%s\n' "$report" | rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | head -n1 | sed -E 's/^90%:[[:space:]]+//' || true)"
+    req_p99="$(printf '%s\n' "$report" | rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' | head -n1 | sed -E 's/^99%:[[:space:]]+//' || true)"
   fi
 
   throughput="$(trim "${throughput:-N/A}")"
@@ -1128,6 +1137,8 @@ run_one_attempt() {
       "--concurrent" "$CONCURRENCY"
       "--duration" "$DURATION"
       "--region" "$REGION"
+      "--no-color"
+      "--analyze.v"
     )
     if [[ "$INSECURE" == "true" ]]; then
       cmd+=("--insecure")
@@ -1212,6 +1223,12 @@ run_one_attempt() {
     req_p99_ms="$(to_ms "$req_p99_human")"
   fi
 
+  if [[ "$DRY_RUN" != "true" && "$TOOL" == "warp" && "$status" == "ok" ]] \
+    && rg -q '^[[:space:]]*(Total[[:space:]]+)?Errors:[[:space:]]*[1-9][0-9]*[.]?([[:space:]]|$)' "$log_file"; then
+    status="failed"
+    exit_code=1
+  fi
+
   if [[ "$DRY_RUN" != "true" && "$status" == "ok" ]]; then
     if [[ "$throughput_bps" == "N/A" && "$reqps" == "N/A" ]]; then
       status="failed"
@@ -1318,10 +1335,21 @@ compare_baseline() {
 
       dr="N/A"; dl="N/A"; dt="N/A"; dp90="N/A"; dp99="N/A"; ne="N/A"; be="N/A"; de="N/A"
       if (br!="N/A" && n_req!="N/A" && br+0!=0) dr=sprintf("%.2f", ((n_req-br)/br)*100)
-      if (bl!="N/A" && n_lat!="N/A" && bl+0!=0) dl=sprintf("%.2f", ((n_lat-bl)/bl)*100)
+      if (bl!="N/A" && n_lat!="N/A") {
+        if (bl+0!=0) dl=sprintf("%.2f", ((n_lat-bl)/bl)*100)
+        else if (n_lat+0==0) dl="0.00"
+      }
       if (bt!="N/A" && n_thr!="N/A" && bt+0!=0) dt=sprintf("%.2f", ((n_thr-bt)/bt)*100)
-      if (bp90!="N/A" && n_p90!="N/A" && bp90+0!=0) dp90=sprintf("%.2f", ((n_p90-bp90)/bp90)*100)
-      if (bp99!="N/A" && n_p99!="N/A" && bp99+0!=0) dp99=sprintf("%.2f", ((n_p99-bp99)/bp99)*100)
+      # Warp v1 rounds sub-millisecond latency to 0s. Two zero readings are
+      # the same below-resolution bucket; a nonzero candidate remains invalid.
+      if (bp90!="N/A" && n_p90!="N/A") {
+        if (bp90+0!=0) dp90=sprintf("%.2f", ((n_p90-bp90)/bp90)*100)
+        else if (n_p90+0==0) dp90="0.00"
+      }
+      if (bp99!="N/A" && n_p99!="N/A") {
+        if (bp99+0!=0) dp99=sprintf("%.2f", ((n_p99-bp99)/bp99)*100)
+        else if (n_p99+0==0) dp99="0.00"
+      }
       if (n_ok!="N/A" && n_fail!="N/A" && n_ok+n_fail>0) ne=sprintf("%.2f", (n_fail/(n_ok+n_fail))*100)
       if (bok!="N/A" && bfail!="N/A" && bok+bfail>0) be=sprintf("%.2f", (bfail/(bok+bfail))*100)
       if (ne!="N/A" && be!="N/A") de=sprintf("%.2f", ne-be)

scripts/security/check_performance_ab_workflow.sh

@@ -31,7 +31,9 @@ require_present_pattern 'listWorkflowRuns' "the scheduled baseline must come fro
 require_present_pattern 'status:[[:space:]]*"success"' "the scheduled baseline must be a successful run"
 require_present_pattern 'SCHEDULED_BASELINE_SHA' "the resolved scheduled baseline must reach the comparison"
 require_present_pattern "SCHEDULED_BASELINE_SHA:-\\\$candidate_sha" "the first scheduled run must seed from its verified candidate"
-require_present_pattern 'git merge-base --is-ancestor' "the scheduled baseline must stay on candidate history"
+require_present_pattern 'git merge-base --is-ancestor' "every baseline must stay on candidate history"
+require_present_pattern 'update the selected ref before comparing' "stale manual refs must fail before the benchmark"
+require_present_pattern 'setup failed before the rig ran' "an earlier setup failure must not become an empty exit status"
 require_present_pattern 'Cache successful candidate baseline' "a successful candidate must become the next cached baseline"
 if ! sed -n '/^  warp-ab:/,/^  alert-on-failure:/p' "$workflow" | grep -Eq '^    timeout-minutes:[[:space:]]*180([[:space:]]|$)'; then
   echo "invalid performance A/B workflow contract: the cold-cache path must fit both builds, the A/B run, and evidence publication" >&2

scripts/test_hotpath_warp_ab_gate.sh

@@ -16,6 +16,14 @@ printf '%s\n' "$header" >"$TMP_DIR/equal-budget.csv"
 printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,9,10,1,0,10.00,0.00,10.00' >>"$TMP_DIR/equal-budget.csv"
 "$GATE" --require-tail-error --fail-pct 10 --warn-pct 5 --compare-csv "$TMP_DIR/equal-budget.csv" >/dev/null
 
+"$GATE" --require-tail-error \
+  --labeled-compare-csv sync-on/put-4kib/B1-vs-A1 "$TMP_DIR/equal-budget.csv" \
+  --labeled-compare-csv sync-off/put-4kib/B1-vs-A1 "$TMP_DIR/equal-budget.csv" \
+  --markdown "$TMP_DIR/labeled.md" >/dev/null
+rg -qF '| Configuration | Workload | Metric |' "$TMP_DIR/labeled.md"
+rg -qF '| sync-on/put-4kib/B1-vs-A1 | 1MiB/warp@1 |' "$TMP_DIR/labeled.md"
+rg -qF '| sync-off/put-4kib/B1-vs-A1 | 1MiB/warp@1 |' "$TMP_DIR/labeled.md"
+
 printf '%s\n' "$header" >"$TMP_DIR/rounded-error.csv"
 printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,2,5,1,1,33.33,16.67,16.67' >>"$TMP_DIR/rounded-error.csv"
 if "$GATE" --require-tail-error --fail-pct 20 --warn-pct 10 --compare-csv "$TMP_DIR/rounded-error.csv" >/dev/null; then

scripts/test_hotpath_warp_abba.sh

@@ -58,9 +58,21 @@ rg -qx 'evidence_mode=dry-run' "$OUT_DIR/manifest.env"
 rg -qx 'formal_evidence=false' "$OUT_DIR/manifest.env"
 rg -qx 'performance_conclusion=not_measured_dry_run' "$OUT_DIR/manifest.env"
 rg -qx 'bucket_isolation=per-leg' "$OUT_DIR/manifest.env"
-rg -qx 'dataset_setup=get-and-mixed-via-warp-put' "$OUT_DIR/manifest.env"
-[[ "$(rg -c -- '--extra-args --noclear' "$TRACE_FILE")" == "64" ]]
-! rg -q -- 'rustfs-bench' "$TRACE_FILE"
+rg -qx 'dataset_setup=get-and-mixed-via-bounded-warp-native' "$OUT_DIR/manifest.env"
+rg -qx 'dataset_objects=64' "$OUT_DIR/manifest.env"
+[[ "$(rg -c -- '--extra-args --objects\\ 64\\ --noclear' "$TRACE_FILE")" == "32" ]]
+if rg -q -- 'dataset-setup' "$TRACE_FILE"; then
+  echo "unexpected redundant dataset setup command" >&2
+  exit 1
+fi
+if rg -q -- 'rustfs-bench' "$TRACE_FILE"; then
+  echo "unexpected rustfs-bench command" >&2
+  exit 1
+fi
+rg -qF -- '--labeled-compare-csv sync-on/put-4kib/B1-vs-A1' "$TRACE_FILE"
+rg -qF -- '--labeled-compare-csv sync-off/put-4kib/B1-vs-A1' "$TRACE_FILE"
+rg -qF -- '--labeled-compare-csv sync-on/put-4kib/A2-vs-A1' "$TRACE_FILE"
+rg -qF -- '--labeled-compare-csv sync-off/put-4kib/A2-vs-A1' "$TRACE_FILE"
 
 if "$RUNNER" \
   --baseline-bin /usr/bin/true \

scripts/test_object_batch_bench_enhanced.sh

@@ -37,6 +37,7 @@ trap cleanup EXIT
 rg -qx 'server_image_ref=rustfs/rustfs:bench' "${OUT_DIR}/run_manifest.env"
 rg -qx 'server_image_digest=sha256:0123456789abcdef' "${OUT_DIR}/run_manifest.env"
 rg -qx 'server_revision=9f61bad94' "${OUT_DIR}/run_manifest.env"
+rg -qx 'warp_report_operation=PUT' "${OUT_DIR}/run_manifest.env"
 rg -qx 'run_label_topology=4x2' "${OUT_DIR}/run_manifest.env"
 rg -qx 'run_label_workload=get' "${OUT_DIR}/run_manifest.env"
 rg -q '^node1,rustfs-bench-1,not_run_dry_run,N/A,N/A,N/A,N/A$' "${OUT_DIR}/node_inventory.csv"
@@ -74,13 +75,40 @@ FAKE_WARP="${TMP_DIR}/fake-warp"
 cat >"$FAKE_WARP" <<'EOF'
 #!/usr/bin/env bash
 set -euo pipefail
+[[ " $* " == *" --analyze.v "* ]]
+[[ " $* " == *" --no-color "* ]]
+if [[ "${FAKE_WARP_ZERO_LATENCY:-0}" == "1" ]]; then
+  cat <<'LOG'
+Operation: PUT. Concurrency: 8. Ran: 7s
+Requests considered: 1000:
+ * Average: 160.00 MiB/s, 40960.00 obj/s
+ * Avg: 0s, 50%: 0s, 90%: 0s, 99%: 0s, Fastest: 0s, Slowest: 1ms, StdDev: 0s
+LOG
+  exit 0
+fi
 cat <<'LOG'
  -       PUT Average: 161 Obj/s, 5.0MiB/s; Current 161 Obj/s, 5.0MiB/s.
-Report: GET. Concurrency: 64. Ran: 7s
+Operation: DELETE - total: 100, 10.0%, Concurrency: 8, Ran 7s
+ * Throughput: 11.00 obj/s
+Requests considered: 100:
+ * Avg: 100ms, 50%: 50ms, 90%: 150ms, 99%: 200ms, Fastest: 1ms, Slowest: 300ms, StdDev: 20ms
+Operation: GET. Concurrency: 64. Ran: 7s
+Requests considered: 1000:
  * Average: 653.90 MiB/s, 20925.58 obj/s
- * Reqs: Avg: 3.5ms, 50%: 2.0ms, 90%: 3.6ms, 99%: 24.1ms, Fastest: 0.2ms, Slowest: 607.7ms, StdDev: 20.6ms
-Throughput, split into 7 x 1s:
+ * Avg: 3.5ms, 50%: 2.0ms, 90%: 3.6ms, 99%: 24.1ms, Fastest: 0.2ms, Slowest: 607.7ms, StdDev: 20.6ms
+Operation: PUT - total: 300, 15.0%, Size: 32767 bytes. Concurrency: 8, Ran 7s
+ * Throughput: 63.53 MiB/s, 254.13 obj/s
+Requests considered: 300:
+ * Avg: 18ms, 50%: 3ms, 90%: 58ms, 99%: 216ms, Fastest: 2ms, Slowest: 608ms, StdDev: 45ms
+Operation: STAT - total: 600, 30.0%, Concurrency: 8, Ran 7s
+ * Throughput: 508.42 obj/s
+Requests considered: 600:
+ * Avg: 0s, 50%: 0s, 90%: 1ms, 99%: 1ms, Fastest: 0s, Slowest: 150ms, StdDev: 3ms
 LOG
+case "${FAKE_WARP_ERRORS:-}" in
+  spaced) echo 'Total Errors: 1.' ;;
+  compact) echo 'Total Errors:1.' ;;
+esac
 EOF
 chmod +x "$FAKE_WARP"
 
@@ -101,7 +129,43 @@ chmod +x "$FAKE_WARP"
   --server-revision 9f61bad94 \
   --require-server-provenance >/dev/null 2>&1
 
-rg -q '^32767B,warp,1,1,128,ok,0,[^,]+,[^,]+,653.90 MiB/s,685663846.400000,20925.58,3.5 ms,3.500000,[^,]+,3.6 ms,3.600000,24.1 ms,24.100000$' "${TMP_DIR}/fake-warp-run/round_results.csv"
+rg -q '^32767B,warp,1,1,128,ok,0,[^,]+,[^,]+,63.53 MiB/s,66616033.280000,254.13,18 ms,18.000000,[^,]+,58 ms,58.000000,216 ms,216.000000$' "${TMP_DIR}/fake-warp-run/round_results.csv"
+
+cat >"${TMP_DIR}/warp-no-details.log" <<'EOF'
+warp: Starting benchmark in 3s...
+Operation: PUT. Concurrency: 8
+ * Average: 2.76 MiB/s, 707.03 obj/s
+EOF
+"$RUNNER" --extract-metrics-from-log "${TMP_DIR}/warp-no-details.log" >"${TMP_DIR}/warp-no-details.csv"
+rg -qx '2.76 MiB/s,2894069.760000,707.03,N/A,N/A,N/A,N/A,N/A,N/A' "${TMP_DIR}/warp-no-details.csv"
+
+cat >"${TMP_DIR}/warp-legacy-report.log" <<'EOF'
+Report:
+ * Average: 10.00 MiB/s, 40.00 obj/s
+ * Reqs: Avg: 2ms, 50%: 1ms, 90%: 3ms, 99%: 4ms, Fastest: 1ms, Slowest: 5ms, StdDev: 1ms
+EOF
+"$RUNNER" --warp-mode put --extract-metrics-from-log "${TMP_DIR}/warp-legacy-report.log" >"${TMP_DIR}/warp-legacy-report.csv"
+rg -qx '10.00 MiB/s,10485760.000000,40.00,2 ms,2.000000,3 ms,3.000000,4 ms,4.000000' "${TMP_DIR}/warp-legacy-report.csv"
+
+for error_format in spaced compact; do
+  if FAKE_WARP_ERRORS="$error_format" "$RUNNER" \
+    --tool warp \
+    --endpoint http://127.0.0.1:9000 \
+    --access-key test-access \
+    --secret-key test-secret \
+    --sizes 32767B \
+    --rounds 1 \
+    --retry-per-round 1 \
+    --retry-sleep-secs 1 \
+    --cooldown-secs 0 \
+    --duration 1s \
+    --out-dir "${TMP_DIR}/fake-warp-errors-${error_format}" \
+    --warp-bin "$FAKE_WARP" >/dev/null 2>&1; then
+    echo "expected Warp ${error_format} request errors to fail the benchmark" >&2
+    exit 1
+  fi
+  rg -q ',failed,1,' "${TMP_DIR}/fake-warp-errors-${error_format}/round_results.csv"
+done
 
 "$RUNNER" \
   --tool warp \
@@ -132,4 +196,28 @@ awk -F',' '
   END { exit found ? 0 : 1 }
 ' "${TMP_DIR}/fake-warp-candidate/baseline_compare.csv"
 
+for leg in baseline candidate; do
+  zero_args=(
+    --tool warp
+    --endpoint http://127.0.0.1:9000
+    --access-key test-access
+    --secret-key test-secret
+    --sizes 4KiB
+    --rounds 1
+    --retry-per-round 1
+    --cooldown-secs 0
+    --duration 1s
+    --out-dir "${TMP_DIR}/fake-warp-zero-${leg}"
+    --warp-bin "$FAKE_WARP"
+  )
+  if [[ "$leg" == "candidate" ]]; then
+    zero_args+=(--baseline-csv "${TMP_DIR}/fake-warp-zero-baseline/median_summary.csv")
+  fi
+  FAKE_WARP_ZERO_LATENCY=1 "$RUNNER" "${zero_args[@]}" >/dev/null 2>&1
+done
+
+"${SCRIPT_DIR}/hotpath_warp_ab_gate.sh" \
+  --compare-csv "${TMP_DIR}/fake-warp-zero-candidate/baseline_compare.csv" \
+  --require-tail-error >/dev/null
+
 echo "object batch benchmark enhanced tests passed"