feat(bench): harden cooled get ab harness (#3960)

scripts/run_get_codec_streaming_smoke.sh

@@ -18,14 +18,16 @@ SECRET_KEY="rustfsadmin"
 BUCKET="rustfs-get-codec-smoke"
 REGION="us-east-1"
 SIZES="1MiB,4MiB,10MiB"
-CONCURRENCY=32
+CONCURRENCY=16
-DURATION="15s"
+DURATION="30s"
 ROUNDS=3
 RETRY_PER_ROUND=1
-ROUND_COOLDOWN_SECS=0
+ROUND_COOLDOWN_SECS=20
 MODE="both"
 CODEC_ENGINES="legacy"
 CODEC_MAX_INFLIGHT=1
+METADATA_EARLY_STOP="off"
+SHARD_LOCALITY_PREFERENCE="off"
 OUTPUT_HANDOFF_ATTRIBUTION=false
 OUT_DIR=""
 RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs"
@@ -38,6 +40,8 @@ COMPAT_OBJECT_KEY="__rustfs_get_v2_pr24_compat/object.bin"
 COMPAT_OBJECT_SIZE=65536
 DRY_RUN=false
 SKIP_BUILD=false
+PROFILE_FAILURES=0
+declare -a ORIGINAL_ARGS=()
 
 SERVER_PID=""
 
@@ -53,16 +57,19 @@ Purpose:
 Core options:
   --mode <legacy|codec|both>     Which profile(s) to run (default: both)
   --codec-engine <csv>           Codec engine(s) for codec profiles (default: legacy)
+  --metadata-early-stop <on|off> Enable metadata early-stop observe/opt-in env (default: off)
+  --shard-locality-preference <on|off>
+                                 Enable shard locality preference env (default: off)
   --codec-max-inflight <n>       RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT (default: 1)
   --handoff-attribution          Enable output handoff attribution metrics
   --address <host:port>          RustFS listen address (default: 127.0.0.1:19030)
   --bucket <name>                Benchmark bucket (default: rustfs-get-codec-smoke)
   --sizes <csv>                  Object sizes (default: 1MiB,4MiB,10MiB)
-  --concurrency <n>              warp concurrency (default: 32)
+  --concurrency <n>              warp concurrency (default: 16)
-  --duration <duration>          warp duration per round (default: 15s)
+  --duration <duration>          warp duration per round (default: 30s)
   --rounds <n>                   rounds per size (default: 3)
   --retry-per-round <n>          failed-attempt retries per round (default: 1)
-  --round-cooldown-secs <n>      cooldown seconds after each completed round (default: 0)
+  --round-cooldown-secs <n>      cooldown seconds after each completed round (default: 20)
   --out-dir <path>               output directory (default: target/bench/get-codec-streaming-<timestamp>)
 
 Binary/options:
@@ -81,6 +88,9 @@ Credentials:
   --region <value>               S3 region (default: us-east-1)
 
 Output:
+  <out-dir>/environment.txt
+  <out-dir>/manifest.env
+  <out-dir>/baseline_compare.csv
   <out-dir>/legacy/warp/median_summary.csv
   <out-dir>/codec-legacy/warp/median_summary.csv
   <out-dir>/codec-rustfs/warp/median_summary.csv
@@ -99,6 +109,7 @@ Output:
   <out-dir>/<profile>/compat/response_headers.json
   <out-dir>/<profile>/compat/body_sha256.txt
   <out-dir>/<profile>/rustfs.log
+  <out-dir>/<profile>/warp/logs/*.log
 
 Example:
   scripts/run_get_codec_streaming_smoke.sh \
@@ -142,6 +153,8 @@ parse_args() {
     case "$1" in
       --mode) MODE="$2"; shift 2 ;;
       --codec-engine) CODEC_ENGINES="$2"; shift 2 ;;
+      --metadata-early-stop) METADATA_EARLY_STOP="$2"; shift 2 ;;
+      --shard-locality-preference) SHARD_LOCALITY_PREFERENCE="$2"; shift 2 ;;
       --codec-max-inflight) CODEC_MAX_INFLIGHT="$2"; shift 2 ;;
       --handoff-attribution) OUTPUT_HANDOFF_ATTRIBUTION=true; shift ;;
       --address) ADDRESS="$2"; shift 2 ;;
@@ -179,6 +192,16 @@ validate_args() {
     *) die "--mode must be legacy, codec, or both" ;;
   esac
 
+  case "$METADATA_EARLY_STOP" in
+    on|off) ;;
+    *) die "--metadata-early-stop must be on or off" ;;
+  esac
+
+  case "$SHARD_LOCALITY_PREFERENCE" in
+    on|off) ;;
+    *) die "--shard-locality-preference must be on or off" ;;
+  esac
+
   local raw engine
   IFS=',' read -r -a engines <<< "$CODEC_ENGINES"
   [[ "${#engines[@]}" -gt 0 ]] || die "--codec-engine must not be empty"
@@ -222,6 +245,111 @@ setup_output() {
   mkdir -p "$OUT_DIR"
 }
 
+bool_from_on_off() {
+  case "$1" in
+    on) echo "true" ;;
+    off) echo "false" ;;
+    *) die "expected on/off value, got: $1" ;;
+  esac
+}
+
+command_line_string() {
+  printf '%q ' "${BASH_SOURCE[0]}" "${ORIGINAL_ARGS[@]}"
+}
+
+detect_cpu_summary() {
+  if command -v sysctl >/dev/null 2>&1; then
+    local brand logical
+    brand="$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true)"
+    logical="$(sysctl -n hw.logicalcpu 2>/dev/null || true)"
+    if [[ -n "$brand" && -n "$logical" ]]; then
+      printf '%s (%s logical cores)\n' "$brand" "$logical"
+      return
+    fi
+  fi
+
+  if command -v lscpu >/dev/null 2>&1; then
+    lscpu 2>/dev/null | awk -F: '
+      /^Model name:/ { model=$2 }
+      /^CPU\(s\):/ { cpus=$2 }
+      END {
+        gsub(/^[ \t]+|[ \t]+$/, "", model)
+        gsub(/^[ \t]+|[ \t]+$/, "", cpus)
+        if (model != "" && cpus != "") {
+          printf "%s (%s logical cores)\n", model, cpus
+        } else if (model != "") {
+          printf "%s\n", model
+        }
+      }
+    '
+    return
+  fi
+
+  echo "unknown"
+}
+
+write_root_environment() {
+  local branch git_head git_dirty_count rustc_version cargo_version command_line
+  branch="$(git -C "$PROJECT_ROOT" rev-parse --abbrev-ref HEAD)"
+  git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)"
+  git_dirty_count="$(git -C "$PROJECT_ROOT" status --porcelain | awk 'END { print NR + 0 }')"
+  rustc_version="$(rustc --version 2>/dev/null || echo unavailable)"
+  cargo_version="$(cargo --version 2>/dev/null || echo unavailable)"
+  command_line="$(command_line_string)"
+
+  cat >"${OUT_DIR}/environment.txt" <<EOF
+generated_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+branch=${branch}
+git_head=${git_head}
+dirty_count=${git_dirty_count}
+rustc_version=${rustc_version}
+cargo_version=${cargo_version}
+os=$(uname -srmo 2>/dev/null || uname -a)
+cpu=$(detect_cpu_summary)
+command_line=${command_line% }
+EOF
+}
+
+write_root_manifest() {
+  local profiles_csv="$1"
+  local branch git_head git_dirty_count command_line
+  branch="$(git -C "$PROJECT_ROOT" rev-parse --abbrev-ref HEAD)"
+  git_head="$(git -C "$PROJECT_ROOT" rev-parse HEAD)"
+  git_dirty_count="$(git -C "$PROJECT_ROOT" status --porcelain | awk 'END { print NR + 0 }')"
+  command_line="$(command_line_string)"
+
+  cat >"${OUT_DIR}/manifest.env" <<EOF
+branch=${branch}
+git_head=${git_head}
+git_dirty_count=${git_dirty_count}
+profiles=${profiles_csv}
+mode=${MODE}
+codec_engines=${CODEC_ENGINES}
+metadata_early_stop=${METADATA_EARLY_STOP}
+shard_locality_preference=${SHARD_LOCALITY_PREFERENCE}
+codec_max_inflight=${CODEC_MAX_INFLIGHT}
+output_handoff_attribution=${OUTPUT_HANDOFF_ATTRIBUTION}
+address=${ADDRESS}
+bucket=${BUCKET}
+region=${REGION}
+sizes=${SIZES}
+concurrency=${CONCURRENCY}
+duration=${DURATION}
+rounds=${ROUNDS}
+retry_per_round=${RETRY_PER_ROUND}
+round_cooldown_secs=${ROUND_COOLDOWN_SECS}
+skip_build=${SKIP_BUILD}
+dry_run=${DRY_RUN}
+rustfs_bin=${RUSTFS_BIN}
+warp_bin=${WARP_BIN}
+python_bin=${PYTHON_BIN}
+codec_min_size=${CODEC_MIN_SIZE}
+compat_object_key=${COMPAT_OBJECT_KEY}
+compat_object_size=${COMPAT_OBJECT_SIZE}
+command_line=${command_line% }
+EOF
+}
+
 build_rustfs_if_needed() {
   if [[ "$DRY_RUN" == "true" || "$SKIP_BUILD" == "true" ]]; then
     return
@@ -308,6 +436,8 @@ compat_object_size=${COMPAT_OBJECT_SIZE}
 rustfs_volumes=${volumes}
 RUSTFS_GET_CODEC_STREAMING_ENABLE=${codec_enabled}
 RUSTFS_GET_CODEC_STREAMING_ENGINE=${codec_engine}
+RUSTFS_GET_METADATA_EARLY_STOP_ENABLE=$(bool_from_on_off "$METADATA_EARLY_STOP")
+RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE=$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE")
 RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT=${CODEC_MAX_INFLIGHT}
 RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=${OUTPUT_HANDOFF_ATTRIBUTION}
 RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE}
@@ -387,6 +517,8 @@ start_server() {
     export RUSTFS_CONSOLE_ENABLE=false
     export RUSTFS_GET_CODEC_STREAMING_ENABLE="$codec_enabled"
     export RUSTFS_GET_CODEC_STREAMING_ENGINE="$codec_engine"
+    export RUSTFS_GET_METADATA_EARLY_STOP_ENABLE="$(bool_from_on_off "$METADATA_EARLY_STOP")"
+    export RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE="$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE")"
     export RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT="$CODEC_MAX_INFLIGHT"
     export RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE="$OUTPUT_HANDOFF_ATTRIBUTION"
     export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE"
@@ -773,14 +905,15 @@ write_root_metrics_summary() {
 
   [[ "${#profile_dirs[@]}" -gt 0 ]] || return
 
-  "$PYTHON_BIN" - "${OUT_DIR}/metrics_summary.csv" "${OUT_DIR}/engine_compare.csv" "${profile_dirs[@]}" <<'PY'
+  "$PYTHON_BIN" - "${OUT_DIR}/metrics_summary.csv" "${OUT_DIR}/engine_compare.csv" "${OUT_DIR}/baseline_compare.csv" "${profile_dirs[@]}" <<'PY'
 import csv
 import pathlib
 import sys
 
 metrics_csv = pathlib.Path(sys.argv[1])
 engine_compare_csv = pathlib.Path(sys.argv[2])
-profile_dirs = [pathlib.Path(value) for value in sys.argv[3:]]
+baseline_compare_csv = pathlib.Path(sys.argv[3])
+profile_dirs = [pathlib.Path(value) for value in sys.argv[4:]]
 
 
 def load_manifest(path):
@@ -834,6 +967,12 @@ for profile_dir in profile_dirs:
                 "read_path": manifest.get("metrics_path", profile),
                 "codec_streaming_enabled": manifest.get("RUSTFS_GET_CODEC_STREAMING_ENABLE", ""),
                 "codec_engine": manifest.get("RUSTFS_GET_CODEC_STREAMING_ENGINE", ""),
+                "metadata_early_stop": manifest.get("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", ""),
+                "shard_locality_preference": manifest.get("RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE", ""),
+                "codec_max_inflight": manifest.get("RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT", ""),
+                "output_handoff_attribution": manifest.get("RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE", ""),
+                "round_cooldown_secs": manifest.get("round_cooldown_secs", ""),
+                "duration": manifest.get("duration", ""),
                 "size": row["size"],
                 "tool": row["tool"],
                 "concurrency": row["concurrency"],
@@ -869,6 +1008,12 @@ fieldnames = [
     "read_path",
     "codec_streaming_enabled",
     "codec_engine",
+    "metadata_early_stop",
+    "shard_locality_preference",
+    "codec_max_inflight",
+    "output_handoff_attribution",
+    "round_cooldown_secs",
+    "duration",
     "size",
     "tool",
     "concurrency",
@@ -888,6 +1033,77 @@ with open(metrics_csv, "w", encoding="utf-8", newline="") as handle:
     writer.writeheader()
     writer.writerows(rows)
 
+baseline_fields = [
+    "profile",
+    "read_path",
+    "codec_streaming_enabled",
+    "codec_engine",
+    "metadata_early_stop",
+    "shard_locality_preference",
+    "codec_max_inflight",
+    "output_handoff_attribution",
+    "round_cooldown_secs",
+    "duration",
+    "size",
+    "tool",
+    "concurrency",
+    "successful_rounds",
+    "failed_rounds",
+    "baseline_profile",
+    "new_median_reqps",
+    "baseline_median_reqps",
+    "delta_reqps_pct_vs_legacy",
+    "new_median_latency_ms",
+    "baseline_median_latency_ms",
+    "delta_latency_pct_vs_legacy",
+    "new_median_throughput_bps",
+    "baseline_median_throughput_bps",
+    "delta_throughput_pct_vs_legacy",
+    "performance_note",
+]
+baseline_rows = []
+for row in rows:
+    if row["profile"] == "legacy" or row["baseline_profile"] != "legacy":
+        continue
+    baseline = legacy_by_size.get(row["size"])
+    if baseline is None:
+        continue
+    baseline_rows.append(
+        {
+            "profile": row["profile"],
+            "read_path": row["read_path"],
+            "codec_streaming_enabled": row["codec_streaming_enabled"],
+            "codec_engine": row["codec_engine"],
+            "metadata_early_stop": row["metadata_early_stop"],
+            "shard_locality_preference": row["shard_locality_preference"],
+            "codec_max_inflight": row["codec_max_inflight"],
+            "output_handoff_attribution": row["output_handoff_attribution"],
+            "round_cooldown_secs": row["round_cooldown_secs"],
+            "duration": row["duration"],
+            "size": row["size"],
+            "tool": row["tool"],
+            "concurrency": row["concurrency"],
+            "successful_rounds": row["successful_rounds"],
+            "failed_rounds": row["failed_rounds"],
+            "baseline_profile": "legacy",
+            "new_median_reqps": row["median_reqps"],
+            "baseline_median_reqps": baseline.get("median_reqps", ""),
+            "delta_reqps_pct_vs_legacy": row["delta_reqps_pct_vs_legacy"],
+            "new_median_latency_ms": row["median_latency_ms"],
+            "baseline_median_latency_ms": baseline.get("median_latency_ms", ""),
+            "delta_latency_pct_vs_legacy": row["delta_latency_pct_vs_legacy"],
+            "new_median_throughput_bps": row["median_throughput_bps"],
+            "baseline_median_throughput_bps": baseline.get("median_throughput_bps", ""),
+            "delta_throughput_pct_vs_legacy": row["delta_throughput_pct_vs_legacy"],
+            "performance_note": row["performance_note"],
+        }
+    )
+
+with open(baseline_compare_csv, "w", encoding="utf-8", newline="") as handle:
+    writer = csv.DictWriter(handle, fieldnames=baseline_fields)
+    writer.writeheader()
+    writer.writerows(baseline_rows)
+
 codec_legacy_by_size = {row["size"]: row for row in by_profile.get("codec-legacy", [])}
 codec_rustfs_by_size = {row["size"]: row for row in by_profile.get("codec-rustfs", [])}
 if not legacy_by_size:
@@ -964,10 +1180,15 @@ PY
 run_profile() {
   local profile="$1"
   local baseline_csv="${2:-}"
+  local bench_rc=0
 
   stop_server
   start_server "$profile"
-  run_bench "$profile" "$baseline_csv"
+  if run_bench "$profile" "$baseline_csv"; then
+    :
+  else
+    bench_rc=$?
+  fi
   write_metrics_summary "$profile"
   run_compat_probe "$profile"
   stop_server
@@ -978,19 +1199,18 @@ run_profile() {
   if [[ -f "${OUT_DIR}/${profile}/warp/baseline_compare.csv" ]]; then
     log "Baseline compare: ${OUT_DIR}/${profile}/warp/baseline_compare.csv"
   fi
+
+  return "$bench_rc"
 }
 
 main() {
+  ORIGINAL_ARGS=("$@")
   parse_args "$@"
   validate_args
   setup_output
-  build_rustfs_if_needed
-
-  trap stop_server EXIT INT TERM
-
-  log "Output dir: $OUT_DIR"
   local codec_profiles=()
   local profiles=()
+  local profiles_csv=""
   local raw engine profile
   IFS=',' read -r -a engines <<< "$CODEC_ENGINES"
   for raw in "${engines[@]}"; do
@@ -1011,14 +1231,29 @@ main() {
       profiles=("legacy" "${codec_profiles[@]}")
       ;;
   esac
+  profiles_csv="$(IFS=,; echo "${profiles[*]}")"
+
+  write_root_environment
+  write_root_manifest "$profiles_csv"
+  build_rustfs_if_needed
+
+  trap stop_server EXIT INT TERM
+
+  log "Output dir: $OUT_DIR"
 
   local legacy_baseline_csv=""
   for profile in "${profiles[@]}"; do
     if [[ "$profile" == "legacy" ]]; then
-      run_profile "$profile"
+      if ! run_profile "$profile"; then
-      legacy_baseline_csv="${OUT_DIR}/legacy/warp/median_summary.csv"
+        PROFILE_FAILURES=$((PROFILE_FAILURES + 1))
+      fi
+      if [[ -f "${OUT_DIR}/legacy/warp/median_summary.csv" ]]; then
+        legacy_baseline_csv="${OUT_DIR}/legacy/warp/median_summary.csv"
+      fi
     else
-      run_profile "$profile" "$legacy_baseline_csv"
+      if ! run_profile "$profile" "$legacy_baseline_csv"; then
+        PROFILE_FAILURES=$((PROFILE_FAILURES + 1))
+      fi
     fi
     copy_profile_compat_artifacts "$profile" "${profile//-/_}"
   done
@@ -1035,6 +1270,12 @@ main() {
   if [[ -f "${OUT_DIR}/metrics_summary.csv" ]]; then
     log "Metrics summary: ${OUT_DIR}/metrics_summary.csv"
   fi
+  if [[ -f "${OUT_DIR}/baseline_compare.csv" ]]; then
+    log "Baseline compare: ${OUT_DIR}/baseline_compare.csv"
+  fi
+  if [[ "$DRY_RUN" != "true" && "$PROFILE_FAILURES" -gt 0 ]]; then
+    die "one or more benchmark profiles reported failed rounds; see per-profile median/baseline outputs for details"
+  fi
   log "GET codec streaming smoke finished."
 }
 

scripts/run_object_batch_bench_enhanced.sh

@@ -37,6 +37,7 @@ RETRY_SLEEP_SECS=2
 COOLDOWN_SECS=0
 BASELINE_CSV=""
 EXTRA_ARGS=()
+FAILED_FINAL_ROUNDS=0
 
 usage() {
   cat <<'USAGE'
@@ -219,7 +220,7 @@ setup_output() {
   MEDIAN_CSV="$OUT_DIR/median_summary.csv"
   COMPARE_CSV="$OUT_DIR/baseline_compare.csv"
 
-  echo "size,tool,round,attempt,concurrency,status,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file" > "$ROUND_CSV"
+  echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file" > "$ROUND_CSV"
   echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms" > "$MEDIAN_CSV"
 }
 
@@ -342,6 +343,9 @@ run_one_attempt() {
   local attempt="$3"
   local log_file="$OUT_DIR/logs/${TOOL}_${size}_r${round}_a${attempt}.log"
   local status="ok"
+  local exit_code=0
+  local started_at_utc finished_at_utc
+  started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 
   if [[ "$TOOL" == "warp" ]]; then
     local warp_host
@@ -369,7 +373,10 @@ run_one_attempt() {
       printf '\n'
       echo "dry run" > "$log_file"
     else
-      if ! "${cmd[@]}" 2>&1 | tee "$log_file" >&2; then
+      if "${cmd[@]}" 2>&1 | tee "$log_file" >&2; then
+        :
+      else
+        exit_code=$?
         status="failed"
       fi
     fi
@@ -397,27 +404,40 @@ run_one_attempt() {
       printf '\n'
       echo "dry run" > "$log_file"
     else
-      if ! "${cmd[@]}" 2>&1 | tee "$log_file" >&2; then
+      if "${cmd[@]}" 2>&1 | tee "$log_file" >&2; then
+        :
+      else
+        exit_code=$?
         status="failed"
       fi
     fi
   fi
+  finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 
   local metrics throughput_human reqps latency_human throughput_bps latency_ms
-  metrics="$(extract_metrics "$log_file")"
+  if [[ "$DRY_RUN" == "true" ]]; then
-  throughput_human="$(echo "$metrics" | cut -d',' -f1)"
+    throughput_human="N/A"
-  reqps="$(echo "$metrics" | cut -d',' -f2)"
+    reqps="N/A"
-  latency_human="$(echo "$metrics" | cut -d',' -f3)"
+    latency_human="N/A"
-  throughput_bps="$(to_bps "$throughput_human")"
+    throughput_bps="N/A"
-  latency_ms="$(to_ms "$latency_human")"
+    latency_ms="N/A"
+  else
+    metrics="$(extract_metrics "$log_file")"
+    throughput_human="$(echo "$metrics" | cut -d',' -f1)"
+    reqps="$(echo "$metrics" | cut -d',' -f2)"
+    latency_human="$(echo "$metrics" | cut -d',' -f3)"
+    throughput_bps="$(to_bps "$throughput_human")"
+    latency_ms="$(to_ms "$latency_human")"
+  fi
 
   if [[ "$DRY_RUN" != "true" && "$status" == "ok" ]]; then
     if [[ "$throughput_bps" == "N/A" && "$reqps" == "N/A" ]]; then
       status="failed"
+      exit_code=1
     fi
   fi
 
-  echo "$size,$TOOL,$round,$attempt,$CONCURRENCY,$status,$throughput_human,$throughput_bps,$reqps,$latency_human,$latency_ms,$log_file" >> "$ROUND_CSV"
+  echo "$size,$TOOL,$round,$attempt,$CONCURRENCY,$status,$exit_code,$started_at_utc,$finished_at_utc,$throughput_human,$throughput_bps,$reqps,$latency_human,$latency_ms,$log_file" >> "$ROUND_CSV"
   echo "$status"
 }
 
@@ -437,12 +457,13 @@ run_size() {
       fi
       if (( attempt < RETRY_PER_ROUND+1 )); then
         echo "Round failed, retry in ${RETRY_SLEEP_SECS}s..."
-        sleep "$RETRY_SLEEP_SECS"
+        cooldown_sleep "$RETRY_SLEEP_SECS"
       fi
     done
 
     if [[ "$success" == "no" ]]; then
       echo "WARN: size=$size round=$round failed after retries."
+      FAILED_FINAL_ROUNDS=$((FAILED_FINAL_ROUNDS + 1))
     fi
 
     if (( COOLDOWN_SECS > 0 && round < ROUNDS )); then
@@ -464,9 +485,9 @@ build_median_summary() {
     ok_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" {c++} END{print c+0}' "$ROUND_CSV")"
     fail_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6!="ok" {c++} END{print c+0}' "$ROUND_CSV")"
 
-    t_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $8!="N/A" {print $8}' "$ROUND_CSV")"
+    t_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $11!="N/A" {print $11}' "$ROUND_CSV")"
-    r_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $9!="N/A" {print $9}' "$ROUND_CSV")"
+    r_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $12!="N/A" {print $12}' "$ROUND_CSV")"
-    l_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $11!="N/A" {print $11}' "$ROUND_CSV")"
+    l_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $14!="N/A" {print $14}' "$ROUND_CSV")"
 
     local m_t m_r m_l
     m_t="$(median_from_numbers "$t_vals")"
@@ -559,6 +580,11 @@ main() {
     echo "=== Baseline Compare ==="
     cat "$COMPARE_CSV"
   fi
+
+  if [[ "$DRY_RUN" != "true" && "$FAILED_FINAL_ROUNDS" -gt 0 ]]; then
+    echo "ERROR: ${FAILED_FINAL_ROUNDS} benchmark rounds failed after retries." >&2
+    exit 1
+  fi
 }
 
 main "$@"