test(perf): add PUT after-probes and node telemetry (#6965)

Co-authored-by: heihutu <heihutu@gmail.com>

scripts/run_hotpath_warp_abba.sh

@@ -43,6 +43,14 @@ EXEMPTION_REASON="deliberate correctness tradeoff"
 OUT_DIR="${PROJECT_ROOT}/target/hotpath-abba/$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)"
 DRY_RUN=false
 ALLOW_UNMANAGED_EXTERNAL=false
+AFTER_PROBE=false
+NODE_SSH_TARGETS=()
+NODE_SSH_IDENTITY_FILE=""
+NODE_SSH_TIMEOUT_SECS=30
+REQUIRE_NODE_TELEMETRY=false
+SERVICE_PROMETHEUS_QUERY_URL=""
+SERVICE_PROMETHEUS_QUERY=""
+SERVICE_METRICS_SERVICE_NAME=""
 
 WORKLOADS=(
   "put-4kib|put|4KiB"
@@ -52,7 +60,9 @@ WORKLOADS=(
   "get-10mib|get|10MiB"
   "mixed-256k|mixed|256KiB"
 )
+WORKLOAD_OVERRIDES=()
 DRIVE_SYNC_MATRIX=("sync-on|true" "sync-off|false")
+DRIVE_SYNC_OVERRIDES=()
 
 usage() {
   cat <<'USAGE'
@@ -94,6 +104,22 @@ Production / cluster mode:
                               mode, then write a non-empty evidence file.
   --allow-unmanaged-external Preserve legacy external mode without a deploy
                               hook. Its output is not formal ABBA evidence.
+  --after-probe              Enable per-round PUT HEAD/GET/hash verification.
+  --node-ssh-target <node=host>
+                              Capture node CPU/RSS/IOPS/await/PSI/scheduler telemetry.
+  --node-ssh-identity-file <path>
+                              SSH identity file for node telemetry.
+  --node-ssh-timeout-secs <n>
+                              Bound each remote telemetry command (default 30).
+  --require-node-telemetry    Fail a round if node telemetry is incomplete.
+  --service-prometheus-query-url <url>
+                              Prometheus query endpoint for PUT stage snapshots.
+  --service-prometheus-query <query>
+                              PromQL selector; include internode RPC metrics when needed.
+  --service-metrics-service-name <name>
+                              Optional service.name filter for stage snapshots.
+  --workload <name|mode|size> Restrict the matrix; repeat for multiple workloads.
+  --drive-sync <label|true|false> Restrict durability cells; repeat if needed.
   --health-path <path>      Readiness path (default /health).
 
 Benchmark:
@@ -150,6 +176,23 @@ validate_positive_int() {
   [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]] || die "$name must be a positive integer"
 }
 
+validate_workload_spec() {
+  local spec="$1" name mode size extra
+  IFS='|' read -r name mode size extra <<<"$spec"
+  [[ -n "$name" && -n "$mode" && -n "$size" && -z "${extra:-}" ]] || die "--workload must be name|mode|size"
+  case "$mode" in
+    put|get|mixed) ;;
+    *) die "--workload mode must be put, get, or mixed" ;;
+  esac
+}
+
+validate_drive_sync_spec() {
+  local spec="$1" label value extra
+  IFS='|' read -r label value extra <<<"$spec"
+  [[ -n "$label" && -n "$value" && -z "${extra:-}" ]] || die "--drive-sync must be label|true|false"
+  [[ "$value" == "true" || "$value" == "false" ]] || die "--drive-sync value must be true or false"
+}
+
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --baseline-bin) BASELINE_BIN="$2"; shift 2 ;;
@@ -178,11 +221,34 @@ while [[ $# -gt 0 ]]; do
     --out-dir) OUT_DIR="$2"; shift 2 ;;
     --dry-run) DRY_RUN=true; shift ;;
     --allow-unmanaged-external) ALLOW_UNMANAGED_EXTERNAL=true; shift ;;
+    --after-probe) AFTER_PROBE=true; shift ;;
+    --node-ssh-target) NODE_SSH_TARGETS+=("$2"); shift 2 ;;
+    --node-ssh-identity-file) NODE_SSH_IDENTITY_FILE="$2"; shift 2 ;;
+    --node-ssh-timeout-secs) NODE_SSH_TIMEOUT_SECS="$2"; shift 2 ;;
+    --require-node-telemetry) REQUIRE_NODE_TELEMETRY=true; shift ;;
+    --service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$2"; shift 2 ;;
+    --service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$2"; shift 2 ;;
+    --service-metrics-service-name) SERVICE_METRICS_SERVICE_NAME="$2"; shift 2 ;;
+    --workload) WORKLOAD_OVERRIDES+=("$2"); shift 2 ;;
+    --drive-sync) DRIVE_SYNC_OVERRIDES+=("$2"); shift 2 ;;
     -h|--help) usage; exit 0 ;;
     *) die "unknown argument: $1" ;;
   esac
 done
 
+if ((${#WORKLOAD_OVERRIDES[@]} > 0)); then
+  for workload in "${WORKLOAD_OVERRIDES[@]}"; do
+    validate_workload_spec "$workload"
+  done
+  WORKLOADS=("${WORKLOAD_OVERRIDES[@]}")
+fi
+if ((${#DRIVE_SYNC_OVERRIDES[@]} > 0)); then
+  for drive_sync in "${DRIVE_SYNC_OVERRIDES[@]}"; do
+    validate_drive_sync_spec "$drive_sync"
+  done
+  DRIVE_SYNC_MATRIX=("${DRIVE_SYNC_OVERRIDES[@]}")
+fi
+
 validate_positive_int "$DISKS" "--disks"
 validate_positive_int "$CONCURRENCY" "--concurrency"
 validate_positive_int "$ROUNDS" "--rounds"
@@ -354,6 +420,7 @@ EOF
 
 measure() {
   local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" bucket="$6" baseline_csv="${7:-}"
+  local node_target
   local cell="$OUT_DIR/$workload/$sync_label/$leg"
   local args=(
     --tool warp --warp-bin "$WARP_BIN" --warp-mode "$mode"
@@ -362,6 +429,24 @@ measure() {
     --duration "$DURATION" --rounds "$ROUNDS" --cooldown-secs "$COOLDOWN_SECS"
     --out-dir "$cell"
   )
+  if [[ "$AFTER_PROBE" == "true" && "$mode" == "put" ]]; then
+    args+=(--after-probe)
+  fi
+  if ((${#NODE_SSH_TARGETS[@]} > 0)); then
+    for node_target in "${NODE_SSH_TARGETS[@]}"; do
+      args+=(--node-ssh-target "$node_target")
+    done
+  fi
+  [[ -n "$NODE_SSH_IDENTITY_FILE" ]] && args+=(--node-ssh-identity-file "$NODE_SSH_IDENTITY_FILE")
+  if [[ "$REQUIRE_NODE_TELEMETRY" == "true" ]]; then
+    args+=(--require-node-telemetry)
+  fi
+  args+=(--node-ssh-timeout-secs "$NODE_SSH_TIMEOUT_SECS")
+  if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
+    args+=(--service-prometheus-query-url "$SERVICE_PROMETHEUS_QUERY_URL" --service-metrics-dir "$cell/service_metrics")
+    [[ -n "$SERVICE_PROMETHEUS_QUERY" ]] && args+=(--service-prometheus-query "$SERVICE_PROMETHEUS_QUERY")
+    [[ -n "$SERVICE_METRICS_SERVICE_NAME" ]] && args+=(--service-metrics-service-name "$SERVICE_METRICS_SERVICE_NAME")
+  fi
   if [[ "$mode" != "put" ]]; then
     # Warp defaults to 2,500 setup objects per round. At 10 MiB that writes
     # 25 GiB before every 12-second measurement, so the matrix cannot finish
@@ -462,6 +547,14 @@ external_isolation=$(isolation_mode)
 evidence_mode=$(evidence_mode)
 formal_evidence=$(formal_evidence)
 performance_conclusion=$(performance_conclusion)
+after_probe=$AFTER_PROBE
+node_ssh_target_count=${#NODE_SSH_TARGETS[@]}
+node_ssh_identity_file=${NODE_SSH_IDENTITY_FILE:-N/A}
+node_ssh_timeout_secs=$NODE_SSH_TIMEOUT_SECS
+require_node_telemetry=$REQUIRE_NODE_TELEMETRY
+service_prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL:-N/A}
+service_prometheus_query=${SERVICE_PROMETHEUS_QUERY:-default}
+service_metrics_service_name=${SERVICE_METRICS_SERVICE_NAME:-N/A}
 dataset_namespace=$DATASET_NAMESPACE
 local_run_data_root=$RUN_DATA_ROOT
 bucket_isolation=per-leg

scripts/run_object_batch_bench_enhanced.sh

@@ -61,6 +61,13 @@ REQUIRE_SERVER_PROVENANCE=false
 RUN_LABELS=()
 NODE_METRICS_URLS=()
 NODE_DOCKER_CONTAINERS=()
+NODE_SSH_TARGETS=()
+NODE_SSH_IDENTITY_FILE=""
+NODE_SSH_TIMEOUT_SECS=30
+REQUIRE_NODE_TELEMETRY=false
+NODE_TELEMETRY_FAILED=false
+AFTER_PROBE=false
+AFTER_PROBE_DIR=""
 
 usage() {
   cat <<'USAGE'
@@ -135,6 +142,14 @@ Enhanced options:
   --node-docker-container <node=container>
                                Repeatable Docker container mapped to a node; captures
                                CPU, memory, network, and block I/O before/after each round
+  --node-ssh-target <node=host> Repeatable SSH target; captures process CPU/RSS,
+                               iostat IOPS/await, and /proc/pressure/io before/after
+  --node-ssh-identity-file <path>
+                               Optional SSH private key used for node telemetry
+  --node-ssh-timeout-secs <n> Bound each remote telemetry command (default 30)
+  --require-node-telemetry    Mark a round failed when any node snapshot fails
+  --after-probe                After successful PUT, HEAD and GET one sampled object,
+                               verify length, ETag presence, and repeatable SHA-256
 
 Output files:
   round_results.csv            One row per round attempt (with retry trace)
@@ -144,6 +159,7 @@ Output files:
   node_inventory.csv            Per-node container and immutable image identity
   node_metrics_captures.csv    Per-node metric snapshot inventory
   node_resource_captures.csv   Per-node Docker resource and block I/O snapshots
+  after_probe.csv               Sampled PUT object HEAD/GET length/hash verification
 
 Example:
   scripts/run_object_batch_bench_enhanced.sh \
@@ -171,6 +187,35 @@ normalize_warp_host() {
   echo "$raw"
 }
 
+normalize_s3_endpoint() {
+  local raw="$1"
+  if [[ "$raw" == http://* || "$raw" == https://* ]]; then
+    echo "$raw"
+  else
+    echo "http://$raw"
+  fi
+}
+
+size_to_bytes() {
+  local value="$1" number unit factor
+  if [[ "$value" =~ ^([0-9]+)(B|KB|MB|GB|KiB|MiB|GiB)$ ]]; then
+    number="${BASH_REMATCH[1]}"
+    unit="${BASH_REMATCH[2]}"
+    case "$unit" in
+      B) factor=1 ;;
+      KB) factor=1000 ;;
+      MB) factor=1000000 ;;
+      GB) factor=1000000000 ;;
+      KiB) factor=1024 ;;
+      MiB) factor=1048576 ;;
+      GiB) factor=1073741824 ;;
+    esac
+    echo $((number * factor))
+  else
+    echo N/A
+  fi
+}
+
 parse_args() {
   while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -217,6 +262,11 @@ parse_args() {
       --label) RUN_LABELS+=("$2"); shift 2 ;;
       --node-metrics-url) NODE_METRICS_URLS+=("$2"); shift 2 ;;
       --node-docker-container) NODE_DOCKER_CONTAINERS+=("$2"); shift 2 ;;
+      --node-ssh-target) NODE_SSH_TARGETS+=("$2"); shift 2 ;;
+      --node-ssh-identity-file) NODE_SSH_IDENTITY_FILE="$2"; shift 2 ;;
+      --node-ssh-timeout-secs) NODE_SSH_TIMEOUT_SECS="$2"; shift 2 ;;
+      --require-node-telemetry) REQUIRE_NODE_TELEMETRY=true; shift ;;
+      --after-probe) AFTER_PROBE=true; shift ;;
       --extra-args)
         # shellcheck disable=SC2206
         EXTRA_ARGS=($2)
@@ -317,17 +367,17 @@ default_service_metrics_filter_regex() {
         return
         ;;
       put)
-        echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_write|rustfs_buffer_|rustfs_ec_|rustfs_io_bytespool_"
+        echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_write|rustfs_buffer_|rustfs_ec_|rustfs_io_bytespool_|rustfs_system_network_internode_"
         return
         ;;
       mixed)
-        echo "rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_|rustfs_buffer_|rustfs_ec_"
+        echo "rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_|rustfs_buffer_|rustfs_ec_|rustfs_system_network_internode_"
         return
         ;;
     esac
   fi
 
-  echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_s3_get_object_|rustfs_io_get_object_"
+  echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_system_network_internode_"
 }
 
 default_service_prometheus_query() {
@@ -338,17 +388,17 @@ default_service_prometheus_query() {
         return
         ;;
       put)
-        echo '{__name__=~"rustfs_(s3_put_object|io_put_object|zero_copy_write|buffer|ec|io_bytespool)_.*"}'
+        echo '{__name__=~"rustfs_(s3_put_object|io_put_object|zero_copy_write|buffer|ec|io_bytespool|system_network_internode)_.*"}'
         return
         ;;
       mixed)
-        echo '{__name__=~"rustfs_(s3_get_object|io_get_object|s3_put_object|io_put_object|zero_copy|buffer|ec)_.*"}'
+        echo '{__name__=~"rustfs_(s3_get_object|io_get_object|s3_put_object|io_put_object|zero_copy|buffer|ec|system_network_internode)_.*"}'
         return
         ;;
     esac
   fi
 
-  echo '{__name__=~"rustfs_(s3_put_object|io_put_object|s3_get_object|io_get_object)_.*"}'
+  echo '{__name__=~"rustfs_(s3_put_object|io_put_object|s3_get_object|io_get_object|system_network_internode)_.*"}'
 }
 
 validate_args() {
@@ -370,6 +420,7 @@ validate_args() {
   validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs"
   validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs"
   validate_nonnegative_int "$SERVICE_METRICS_SETTLE_SECS" "--service-metrics-settle-secs"
+  validate_positive_int "$NODE_SSH_TIMEOUT_SECS" "--node-ssh-timeout-secs"
   if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
     echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2
     exit 1
@@ -387,6 +438,11 @@ validate_args() {
   if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)) && [[ "$DRY_RUN" != "true" ]]; then
     require_cmd docker
   fi
+  if [[ "$AFTER_PROBE" == "true" ]] && [[ "$DRY_RUN" != "true" ]]; then
+    require_cmd mc
+    require_cmd jq
+    require_cmd sha256sum
+  fi
   if [[ "$TOOL" == "s3bench" ]]; then
     validate_positive_int "$SAMPLES" "--samples"
   fi
@@ -404,6 +460,15 @@ validate_args() {
   if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then
     validate_named_values "--node-docker-container" "${NODE_DOCKER_CONTAINERS[@]}"
   fi
+  if ((${#NODE_SSH_TARGETS[@]} > 0)); then
+    validate_named_values "--node-ssh-target" "${NODE_SSH_TARGETS[@]}"
+    require_cmd ssh
+    [[ "$DRY_RUN" == "true" ]] || require_cmd timeout
+    if [[ -n "$NODE_SSH_IDENTITY_FILE" && "$DRY_RUN" != "true" && ! -r "$NODE_SSH_IDENTITY_FILE" ]]; then
+      echo "ERROR: --node-ssh-identity-file is not readable: $NODE_SSH_IDENTITY_FILE" >&2
+      exit 1
+    fi
+  fi
   validate_manifest_value "$SERVER_IMAGE_REF" "--server-image-ref"
   validate_manifest_value "$SERVER_IMAGE_DIGEST" "--server-image-digest"
   validate_manifest_value "$SERVER_REVISION" "--server-revision"
@@ -533,6 +598,11 @@ EOF
     fi
     echo "node_metrics_count=${#NODE_METRICS_URLS[@]}"
     echo "node_docker_container_count=${#NODE_DOCKER_CONTAINERS[@]}"
+    echo "node_ssh_target_count=${#NODE_SSH_TARGETS[@]}"
+    echo "node_ssh_identity_file=${NODE_SSH_IDENTITY_FILE:-N/A}"
+    echo "node_ssh_timeout_secs=${NODE_SSH_TIMEOUT_SECS}"
+    echo "require_node_telemetry=${REQUIRE_NODE_TELEMETRY}"
+    echo "after_probe=${AFTER_PROBE}"
   } >>"$manifest_file"
 }
 
@@ -573,6 +643,9 @@ setup_output() {
   if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then
     mkdir -p "$OUT_DIR/node_resources"
   fi
+  if ((${#NODE_SSH_TARGETS[@]} > 0)); then
+    mkdir -p "$OUT_DIR/node_telemetry"
+  fi
 
   ROUND_CSV="$OUT_DIR/round_results.csv"
   MEDIAN_CSV="$OUT_DIR/median_summary.csv"
@@ -580,6 +653,9 @@ setup_output() {
   SERVICE_METRICS_CSV="$OUT_DIR/service_metrics_captures.csv"
   NODE_METRICS_CSV="$OUT_DIR/node_metrics_captures.csv"
   NODE_RESOURCE_CSV="$OUT_DIR/node_resource_captures.csv"
+  NODE_SSH_CSV="$OUT_DIR/node_telemetry_captures.csv"
+  AFTER_PROBE_CSV="$OUT_DIR/after_probe.csv"
+  AFTER_PROBE_DIR="${OUT_DIR}/after_probe"
 
   echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file,req_p90_human,req_p90_ms,req_p99_human,req_p99_ms" > "$ROUND_CSV"
   echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,median_req_p90_ms,median_req_p99_ms" > "$MEDIAN_CSV"
@@ -592,6 +668,14 @@ setup_output() {
   if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then
     echo "size,tool,round,attempt,phase,node,container,status,snapshot_file" > "$NODE_RESOURCE_CSV"
   fi
+  if ((${#NODE_SSH_TARGETS[@]} > 0)); then
+    mkdir -p "$OUT_DIR/node_telemetry"
+    echo "size,tool,round,attempt,phase,node,host,status,snapshot_file" > "$NODE_SSH_CSV"
+  fi
+  if [[ "$AFTER_PROBE" == "true" ]]; then
+    mkdir -p "$AFTER_PROBE_DIR"
+    echo "size,tool,round,attempt,status,bucket,key,expected_size,head_size,get_size,hash1,hash2,etag,error" > "$AFTER_PROBE_CSV"
+  fi
   write_run_manifest
   write_node_inventory
 }
@@ -1088,6 +1172,138 @@ capture_round_node_resources() {
   done
 }
 
+capture_round_node_telemetry() {
+  local size="$1" round="$2" attempt="$3" phase="$4" entry node host token snapshot_file
+  if ((${#NODE_SSH_TARGETS[@]} == 0)); then
+    return
+  fi
+  for entry in "${NODE_SSH_TARGETS[@]}"; do
+    node="${entry%%=*}"
+    host="${entry#*=}"
+    local -a ssh_args=(-o BatchMode=yes -o ConnectTimeout=5)
+    if [[ -n "$NODE_SSH_IDENTITY_FILE" ]]; then
+      ssh_args+=(-i "$NODE_SSH_IDENTITY_FILE")
+    fi
+    token="$(metric_snapshot_token "$size" "$round" "$attempt")"
+    snapshot_file="$OUT_DIR/node_telemetry/${token}_${node}_${phase}.txt"
+    if [[ "$DRY_RUN" == "true" ]]; then
+      : >"$snapshot_file"
+      echo "$size,$TOOL,$round,$attempt,$phase,$node,$host,not_run_dry_run,$snapshot_file" >>"$NODE_SSH_CSV"
+      continue
+    fi
+    if timeout "$NODE_SSH_TIMEOUT_SECS" ssh "${ssh_args[@]}" "$host" 'set -u
+      pid="$(pgrep -o -f /usr/local/bin/rustfs || true)"
+      echo "captured_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+      echo "pid=${pid:-N/A}"
+      if [[ -n "$pid" ]]; then
+        process_metrics="$(ps -p "$pid" -o pcpu=,pmem=,rss=,vsz=,nlwp= | awk '\''NR == 1 {print $1, $2, $3, $4, $5}'\'')"
+        read -r process_cpu process_mem process_rss process_vsz process_threads <<<"$process_metrics"
+        echo "process_cpu_percent=${process_cpu:-N/A}"
+        echo "process_mem_percent=${process_mem:-N/A}"
+        echo "process_rss_kb=${process_rss:-N/A}"
+        echo "process_vsz_kb=${process_vsz:-N/A}"
+        echo "process_threads=${process_threads:-N/A}"
+      else
+        echo "process_status=missing"
+      fi
+      if command -v iostat >/dev/null 2>&1; then
+        echo "iostat_begin"
+        iostat -c -dx 1 1 2>/dev/null || true
+        echo "iostat_end"
+      else
+        echo "iostat_status=unavailable"
+      fi
+      echo "scheduler_begin"
+      echo "loadavg=$(cat /proc/loadavg 2>/dev/null || echo N/A)"
+      awk "/^ctxt / || /^procs_running / || /^procs_blocked / {print}" /proc/stat 2>/dev/null || true
+      if command -v vmstat >/dev/null 2>&1; then
+        vmstat_sample="$(vmstat 1 2 2>/dev/null | tail -n 1 || true)"
+        echo "vmstat_sample=${vmstat_sample:-N/A}"
+        if [[ -n "$vmstat_sample" ]]; then
+          read -r vm_r vm_b vm_swpd vm_free vm_buff vm_cache vm_si vm_so vm_bi vm_bo vm_in vm_cs vm_us vm_sy vm_id vm_wa vm_st <<<"$vmstat_sample"
+          echo "vmstat_r=${vm_r:-N/A}"
+          echo "vmstat_b=${vm_b:-N/A}"
+          echo "vmstat_iowait_pct=${vm_wa:-N/A}"
+          echo "vmstat_steal_pct=${vm_st:-N/A}"
+        fi
+      else
+        echo "vmstat_status=unavailable"
+      fi
+      echo "scheduler_end"
+      echo "io_psi_begin"
+      cat /proc/pressure/io 2>/dev/null || echo "io_psi_status=unavailable"
+      echo "io_psi_end"' >"$snapshot_file" && [[ -s "$snapshot_file" ]]; then
+      echo "$size,$TOOL,$round,$attempt,$phase,$node,$host,ok,$snapshot_file" >>"$NODE_SSH_CSV"
+    else
+      NODE_TELEMETRY_FAILED=true
+      : >"$snapshot_file"
+      echo "$size,$TOOL,$round,$attempt,$phase,$node,$host,capture_failed,$snapshot_file" >>"$NODE_SSH_CSV"
+      echo "WARN: failed to capture node telemetry node=${node} host=${host}" >&2
+    fi
+  done
+}
+
+capture_after_probe() {
+  local size="$1" round="$2" attempt="$3" bucket alias probe_root key stat_json expected_size head_size get_size hash1 hash2 etag error
+  if [[ "$AFTER_PROBE" != "true" || "$TOOL" != "warp" || "$WARP_MODE" != "put" ]]; then
+    return 0
+  fi
+  probe_root="$AFTER_PROBE_DIR/probe-${size}-${round}-${attempt}"
+  mkdir -p "$probe_root"
+  bucket="$(bucket_for_size "$size")"
+  alias="probe"
+  key=""
+  stat_json=""
+  expected_size="$(size_to_bytes "$size")"
+  head_size="N/A"
+  get_size="N/A"
+  hash1="N/A"
+  hash2="N/A"
+  etag="N/A"
+  error=""
+
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "$size,$TOOL,$round,$attempt,not_run_dry_run,$bucket,N/A,$expected_size,N/A,N/A,N/A,N/A,N/A,dry_run" >>"$AFTER_PROBE_CSV"
+    return 0
+  fi
+
+  export MC_CONFIG_DIR="$probe_root/mc-config"
+  if ! mc alias set "$alias" "$(normalize_s3_endpoint "$ENDPOINT")" "$ACCESS_KEY" "$SECRET_KEY" --api S3v4 >/dev/null 2>"$probe_root/alias.err"; then
+    error="alias_setup_failed"
+  else
+    key="$(mc ls --json --recursive "$alias/$bucket" 2>"$probe_root/ls.err" | jq -r 'select(.type == "file") | [(.lastModified // ""), .key] | @tsv' | sort -r | head -n1 | cut -f2- || true)"
+    if [[ -z "$key" ]]; then
+      error="no_object_found"
+    else
+      stat_json="$(mc stat --json "$alias/$bucket/$key" 2>"$probe_root/stat.err" || true)"
+      head_size="$(printf '%s' "$stat_json" | jq -r '.size // empty' 2>/dev/null || true)"
+      etag="$(printf '%s' "$stat_json" | jq -r '.etag // empty' 2>/dev/null || true)"
+      head_size="${head_size:-N/A}"
+      etag="${etag:-N/A}"
+      mc cat "$alias/$bucket/$key" >"$probe_root/get1.bin" 2>"$probe_root/get1.err" || true
+      mc cat "$alias/$bucket/$key" >"$probe_root/get2.bin" 2>"$probe_root/get2.err" || true
+      if [[ -f "$probe_root/get1.bin" ]]; then
+        get_size="$(wc -c <"$probe_root/get1.bin" | tr -d '[:space:]')"
+        hash1="$(sha256sum "$probe_root/get1.bin" | awk '{print $1}')"
+      fi
+      if [[ -f "$probe_root/get2.bin" ]]; then
+        hash2="$(sha256sum "$probe_root/get2.bin" | awk '{print $1}')"
+      fi
+      if [[ "$head_size" != "$expected_size" ]]; then error="head_size_mismatch"; fi
+      if [[ "$get_size" != "$expected_size" ]]; then error="${error:+$error; }get_size_mismatch"; fi
+      if [[ "$hash1" == "N/A" || "$hash1" != "$hash2" ]]; then error="${error:+$error; }hash_mismatch_or_missing"; fi
+    fi
+  fi
+  rm -rf "$MC_CONFIG_DIR"
+  if [[ -z "$error" ]]; then
+    echo "$size,$TOOL,$round,$attempt,ok,$bucket,$key,$expected_size,$head_size,$get_size,$hash1,$hash2,$etag,N/A" >>"$AFTER_PROBE_CSV"
+    return 0
+  fi
+  echo "$size,$TOOL,$round,$attempt,failed,$bucket,$key,$expected_size,$head_size,$get_size,$hash1,$hash2,$etag,$error" >>"$AFTER_PROBE_CSV"
+  echo "WARN: after-probe failed size=${size} round=${round} attempt=${attempt}: ${error}" >&2
+  return 1
+}
+
 median_from_numbers() {
   local values="$1"
   local count
@@ -1118,9 +1334,11 @@ run_one_attempt() {
   local status="ok"
   local exit_code=0
   local started_at_utc finished_at_utc
+  NODE_TELEMETRY_FAILED=false
   capture_round_service_metrics "$size" "$round" "$attempt" before
   capture_round_node_metrics "$size" "$round" "$attempt" before
   capture_round_node_resources "$size" "$round" "$attempt" before
+  capture_round_node_telemetry "$size" "$round" "$attempt" before
   started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 
   if [[ "$TOOL" == "warp" ]]; then
@@ -1140,6 +1358,9 @@ run_one_attempt() {
       "--no-color"
       "--analyze.v"
     )
+    if [[ "$AFTER_PROBE" == "true" && "$WARP_MODE" == "put" ]]; then
+      cmd+=("--noclear")
+    fi
     if [[ "$INSECURE" == "true" ]]; then
       cmd+=("--insecure")
     fi
@@ -1198,6 +1419,15 @@ run_one_attempt() {
   capture_round_service_metrics "$size" "$round" "$attempt" after
   capture_round_node_metrics "$size" "$round" "$attempt" after
   capture_round_node_resources "$size" "$round" "$attempt" after
+  capture_round_node_telemetry "$size" "$round" "$attempt" after
+  if [[ "$REQUIRE_NODE_TELEMETRY" == "true" && "$NODE_TELEMETRY_FAILED" == "true" ]]; then
+    status="failed"
+    exit_code=1
+  fi
+  if ! capture_after_probe "$size" "$round" "$attempt"; then
+    status="failed"
+    exit_code=1
+  fi
 
   local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms
   if [[ "$DRY_RUN" == "true" ]]; then

scripts/test_hotpath_warp_abba.sh

@@ -69,6 +69,32 @@ if rg -q -- 'rustfs-bench' "$TRACE_FILE"; then
   echo "unexpected rustfs-bench command" >&2
   exit 1
 fi
+
+FOCUSED_OUT_DIR="${TMP_DIR}/focused"
+"$RUNNER" \
+  --baseline-bin /usr/bin/true \
+  --candidate-bin /usr/bin/true \
+  --baseline-revision baseline-test \
+  --candidate-revision candidate-test \
+  --warp-bin /usr/bin/true \
+  --rounds 3 \
+  --workload 'put-128kib|put|128KiB' \
+  --workload 'put-512kib|put|512KiB' \
+  --drive-sync 'sync-on|true' \
+  --out-dir "$FOCUSED_OUT_DIR" \
+  --dry-run >/dev/null 2>&1
+awk -F',' 'NR == 1 {next} {seen[$3]++; if ($1 != "sync-on" || $3 !~ /^put-(128|512)kib$/) exit 1} END {exit (seen["put-128kib"] == 4 && seen["put-512kib"] == 4) ? 0 : 1}' "$FOCUSED_OUT_DIR/abba_schedule.csv"
+rg -Fxq 'workloads=put-128kib|put|128KiB;put-512kib|put|512KiB' "$FOCUSED_OUT_DIR/manifest.env"
+rg -Fxq 'drive_sync_matrix=sync-on|true' "$FOCUSED_OUT_DIR/manifest.env"
+
+if "$RUNNER" --baseline-bin /usr/bin/true --candidate-bin /usr/bin/true --baseline-revision baseline-test --candidate-revision candidate-test --warp-bin /usr/bin/true --rounds 3 --workload 'bad|delete|1KiB' --out-dir "${TMP_DIR}/bad-workload" --dry-run >/dev/null 2>&1; then
+  echo "expected invalid --workload mode to fail" >&2
+  exit 1
+fi
+if "$RUNNER" --baseline-bin /usr/bin/true --candidate-bin /usr/bin/true --baseline-revision baseline-test --candidate-revision candidate-test --warp-bin /usr/bin/true --rounds 3 --drive-sync 'sync-on|maybe' --out-dir "${TMP_DIR}/bad-drive-sync" --dry-run >/dev/null 2>&1; then
+  echo "expected invalid --drive-sync value to fail" >&2
+  exit 1
+fi
 rg -qF -- '--labeled-compare-csv sync-on/put-4kib/B1-vs-A1' "$TRACE_FILE"
 rg -qF -- '--labeled-compare-csv sync-off/put-4kib/B1-vs-A1' "$TRACE_FILE"
 rg -qF -- '--labeled-compare-csv sync-on/put-4kib/A2-vs-A1' "$TRACE_FILE"

scripts/test_object_batch_bench_enhanced.sh

@@ -23,7 +23,9 @@ trap cleanup EXIT
   --duration 1s \
   --out-dir "$OUT_DIR" \
   --warp-bin true \
+  --warp-mode put \
   --dry-run \
+  --after-probe \
   --service-metrics-dir "${OUT_DIR}/metrics" \
   --server-image-ref rustfs/rustfs:bench \
   --server-image-digest sha256:0123456789abcdef \
@@ -32,6 +34,10 @@ trap cleanup EXIT
   --label topology=4x2 \
   --label workload=get \
   --node-metrics-url node1=http://127.0.0.1:9001/metrics \
+  --node-ssh-target node1=localhost \
+  --node-ssh-identity-file /tmp/nonexistent-key \
+  --node-ssh-timeout-secs 7 \
+  --require-node-telemetry \
   --node-docker-container node1=rustfs-bench-1 >/dev/null
 
 rg -qx 'server_image_ref=rustfs/rustfs:bench' "${OUT_DIR}/run_manifest.env"
@@ -43,6 +49,29 @@ rg -qx 'run_label_workload=get' "${OUT_DIR}/run_manifest.env"
 rg -q '^node1,rustfs-bench-1,not_run_dry_run,N/A,N/A,N/A,N/A$' "${OUT_DIR}/node_inventory.csv"
 rg -q '^1MiB,warp,1,1,before,node1,not_run_dry_run,' "${OUT_DIR}/node_metrics_captures.csv"
 rg -q '^1MiB,warp,1,1,after,node1,rustfs-bench-1,not_run_dry_run,' "${OUT_DIR}/node_resource_captures.csv"
+rg -qx 'after_probe=true' "${OUT_DIR}/run_manifest.env"
+rg -qx 'node_ssh_identity_file=/tmp/nonexistent-key' "${OUT_DIR}/run_manifest.env"
+rg -qx 'node_ssh_timeout_secs=7' "${OUT_DIR}/run_manifest.env"
+rg -qx 'require_node_telemetry=true' "${OUT_DIR}/run_manifest.env"
+rg -q '^1MiB,warp,1,1,before,node1,localhost,not_run_dry_run,' "${OUT_DIR}/node_telemetry_captures.csv"
+rg -q '^1MiB,warp,1,1,not_run_dry_run,rustfs-bench,N/A,1048576,N/A,N/A,N/A,N/A,N/A,dry_run$' "${OUT_DIR}/after_probe.csv"
+
+"$RUNNER" \
+  --tool warp \
+  --endpoint http://127.0.0.1:9000 \
+  --access-key test-access \
+  --secret-key test-secret \
+  --sizes 1MB \
+  --rounds 1 \
+  --retry-per-round 1 \
+  --cooldown-secs 0 \
+  --duration 1s \
+  --out-dir "${TMP_DIR}/decimal-size" \
+  --warp-bin true \
+  --warp-mode put \
+  --dry-run \
+  --after-probe >/dev/null
+rg -q '^1MB,warp,1,1,not_run_dry_run,rustfs-bench,N/A,1000000,N/A,N/A,N/A,N/A,N/A,dry_run$' "${TMP_DIR}/decimal-size/after_probe.csv"
 
 if "$RUNNER" \
   --tool warp \