feat(storage): optimize gt1g get read path (#3860)

* feat(storage): tune gt1g get read path * docs(ops): add gt1g get benchmark guide * feat(scripts): add gt1g get fallback runner * test(storage): trim gt1g get helper warnings * test(storage): reduce gt1g get helper type complexity
2026-08-06 05:17:42 +00:00 · 2026-06-25 20:12:08 +08:00
parent a30357d21e
commit 96b1f5c373
8 changed files with 1776 additions and 21 deletions
@@ -0,0 +1,249 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ENDPOINT=""
+ACCESS_KEY=""
+SECRET_KEY=""
+BUCKET=""
+PREFIX="bench/issue713"
+OBJECTS="1GiB=plain-1g.bin,2GiB=plain-2g.bin"
+REGION="us-east-1"
+MC_BIN="${MC_BIN:-mc}"
+FORCE=false
+INSECURE=false
+DRY_RUN=false
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/prepare_gt1g_get_test_objects.sh \
+    --endpoint <url> --access-key <ak> --secret-key <sk> --bucket <bucket> [options]
+
+Required:
+  --endpoint <url>
+  --access-key <ak>
+  --secret-key <sk>
+  --bucket <bucket>
+
+Options:
+  --prefix <path>              Default: bench/issue713
+  --objects <csv>              Default: 1GiB=plain-1g.bin,2GiB=plain-2g.bin
+                               Format: size=object-name,size=object-name
+  --region <name>              Default: us-east-1
+  --mc-bin <path>              Default: mc
+  --force                      Re-upload even if object already exists
+  --insecure                   Allow insecure TLS
+  --dry-run
+  -h, --help
+
+Examples:
+  scripts/prepare_gt1g_get_test_objects.sh \
+    --endpoint http://127.0.0.1:9000 \
+    --access-key rustfsadmin \
+    --secret-key rustfsadmin \
+    --bucket rustfs-bench \
+    --prefix bench/issue713/plain
+USAGE
+}
+
+arg_value() {
+  local flag="$1"
+  local value="${2:-}"
+  if [[ -z "$value" || "$value" == --* ]]; then
+    echo "ERROR: missing value for $flag" >&2
+    exit 1
+  fi
+  printf '%s\n' "$value"
+}
+
+require_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "ERROR: command not found: $1" >&2
+    exit 1
+  fi
+}
+
+run_rust_helper_fallback() {
+  local objects_arg=""
+  local helper_out_dir="$OUT_DIR"
+  local raw_spec
+  IFS=',' read -r -a object_specs <<< "$OBJECTS"
+  for raw_spec in "${object_specs[@]}"; do
+    local spec size object_name
+    spec="$(echo "$raw_spec" | awk '{$1=$1;print}')"
+    [[ -z "$spec" ]] && continue
+    size="${spec%%=*}"
+    object_name="${spec#*=}"
+    objects_arg+="${size}=${PREFIX%/}/${object_name},"
+  done
+  objects_arg="${objects_arg%,}"
+
+  if [[ -n "$helper_out_dir" && "$helper_out_dir" != /* ]]; then
+    helper_out_dir="$PWD/$helper_out_dir"
+  fi
+
+  echo "mc not found; falling back to rustfs/tests/gt1g_get_benchmark_tool.rs"
+  GT1G_GET_ACTION=prepare \
+  GT1G_GET_ENDPOINT="$ENDPOINT" \
+  GT1G_GET_ACCESS_KEY="$ACCESS_KEY" \
+  GT1G_GET_SECRET_KEY="$SECRET_KEY" \
+  GT1G_GET_BUCKET="$BUCKET" \
+  GT1G_GET_REGION="$REGION" \
+  GT1G_GET_OBJECTS="$objects_arg" \
+  GT1G_GET_OUT_DIR="${helper_out_dir:-$PWD/target/bench/issue713-prepare-helper}" \
+  GT1G_GET_FORCE="$([[ "$FORCE" == "true" ]] && echo true || echo false)" \
+  cargo test -p rustfs --test gt1g_get_benchmark_tool gt1g_get_benchmark_tool -- --ignored --nocapture
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --bucket) BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --prefix) PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --objects) OBJECTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --mc-bin) MC_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --force) FORCE=true; shift ;;
+      --insecure) INSECURE=true; shift ;;
+      --dry-run) DRY_RUN=true; shift ;;
+      -h|--help) usage; exit 0 ;;
+      *)
+        echo "ERROR: unknown arg: $1" >&2
+        usage
+        exit 1
+        ;;
+    esac
+  done
+}
+
+validate_args() {
+  if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" || -z "$BUCKET" ]]; then
+    echo "ERROR: --endpoint, --access-key, --secret-key, and --bucket are required" >&2
+    exit 1
+  fi
+}
+
+size_to_bytes() {
+  local size="$1"
+  case "$size" in
+    *GiB)
+      local n="${size%GiB}"
+      echo $((n * 1024 * 1024 * 1024))
+      ;;
+    *MiB)
+      local n="${size%MiB}"
+      echo $((n * 1024 * 1024))
+      ;;
+    *KiB)
+      local n="${size%KiB}"
+      echo $((n * 1024))
+      ;;
+    *B)
+      echo "${size%B}"
+      ;;
+    *)
+      echo "ERROR"
+      ;;
+  esac
+}
+
+create_sparse_file() {
+  local file_path="$1"
+  local bytes="$2"
+  if command -v mkfile >/dev/null 2>&1; then
+    mkfile -n "$bytes" "$file_path"
+  else
+    truncate -s "$bytes" "$file_path"
+  fi
+}
+
+object_exists() {
+  local alias_path="$1"
+  local -a cmd=("$MC_BIN" stat "$alias_path")
+  if [[ "$INSECURE" == "true" ]]; then
+    cmd=("$MC_BIN" --insecure stat "$alias_path")
+  fi
+
+  "${cmd[@]}" >/dev/null 2>&1
+}
+
+main() {
+  parse_args "$@"
+  validate_args
+
+  if ! command -v "$MC_BIN" >/dev/null 2>&1; then
+    run_rust_helper_fallback
+    exit 0
+  fi
+
+  local tmp_root
+  tmp_root="$(mktemp -d "${TMPDIR:-/tmp}/issue713-gt1g-get.XXXXXX")"
+  trap 'rm -rf "$tmp_root"' EXIT
+
+  local mc_config_dir="$tmp_root/mc"
+  mkdir -p "$mc_config_dir"
+
+  local -a mc_alias_cmd=("$MC_BIN" --config-dir "$mc_config_dir")
+  if [[ "$INSECURE" == "true" ]]; then
+    mc_alias_cmd+=("--insecure")
+  fi
+
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "[DRY-RUN] ${mc_alias_cmd[*]} alias set issue713 ${ENDPOINT} REDACTED REDACTED"
+  else
+    "${mc_alias_cmd[@]}" alias set issue713 "$ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null
+  fi
+
+  IFS=',' read -r -a object_specs <<< "$OBJECTS"
+  for raw_spec in "${object_specs[@]}"; do
+    local spec size object_name bytes local_file object_key alias_path
+    spec="$(echo "$raw_spec" | awk '{$1=$1;print}')"
+    [[ -z "$spec" ]] && continue
+
+    size="${spec%%=*}"
+    object_name="${spec#*=}"
+    if [[ -z "$size" || -z "$object_name" || "$size" == "$object_name" ]]; then
+      echo "ERROR: invalid object spec: $spec" >&2
+      exit 1
+    fi
+
+    bytes="$(size_to_bytes "$size")"
+    if [[ "$bytes" == "ERROR" ]]; then
+      echo "ERROR: unsupported size label: $size" >&2
+      exit 1
+    fi
+
+    object_key="${PREFIX%/}/${object_name}"
+    alias_path="issue713/${BUCKET}/${object_key}"
+
+    if [[ "$FORCE" != "true" && "$DRY_RUN" != "true" ]] && object_exists "$alias_path"; then
+      echo "skip existing: s3://${BUCKET}/${object_key}"
+      continue
+    fi
+
+    local_file="${tmp_root}/${object_name}"
+    create_sparse_file "$local_file" "$bytes"
+
+    if [[ "$DRY_RUN" == "true" ]]; then
+      echo "[DRY-RUN] create sparse file ${local_file} (${bytes} bytes)"
+      echo "[DRY-RUN] ${mc_alias_cmd[*]} cp ${local_file} ${alias_path}"
+    else
+      echo "uploading: s3://${BUCKET}/${object_key} (${size})"
+      "${mc_alias_cmd[@]}" cp "$local_file" "$alias_path" >/dev/null
+    fi
+  done
+
+  echo "prepared objects:"
+  for raw_spec in "${object_specs[@]}"; do
+    local spec object_name
+    spec="$(echo "$raw_spec" | awk '{$1=$1;print}')"
+    [[ -z "$spec" ]] && continue
+    object_name="${spec#*=}"
+    echo "  s3://${BUCKET}/${PREFIX%/}/${object_name}"
+  done
+}
+
+main "$@"
@@ -0,0 +1,501 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ENDPOINT=""
+ACCESS_KEY=""
+SECRET_KEY=""
+BUCKET=""
+OBJECTS=""
+REGION="us-east-1"
+MODES="sequential,ranged_parallel"
+CONCURRENCIES="1,4,8"
+RANGE_WORKERS="4"
+ROUNDS=3
+COOLDOWN_SECS=15
+PRESIGN_EXPIRY="2h"
+OUT_DIR=""
+MC_BIN="${MC_BIN:-mc}"
+CURL_BIN="${CURL_BIN:-curl}"
+JQ_BIN="${JQ_BIN:-jq}"
+INSECURE=false
+DRY_RUN=false
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/run_gt1g_get_http_matrix.sh \
+    --endpoint <url> --access-key <ak> --secret-key <sk> \
+    --bucket <bucket> --objects <csv> [options]
+
+Required:
+  --endpoint <url>
+  --access-key <ak>
+  --secret-key <sk>
+  --bucket <bucket>
+  --objects <csv>               Format: label=object-key,label=object-key
+
+Options:
+  --region <name>               Default: us-east-1
+  --modes <csv>                 Default: sequential,ranged_parallel
+  --concurrencies <csv>         Default: 1,4,8
+  --range-workers <csv>         Default: 4
+                                 Used only by ranged_parallel mode
+  --rounds <n>                  Default: 3
+  --cooldown-secs <n>           Default: 15
+  --presign-expiry <dur>        Default: 2h
+  --out-dir <dir>               Default: target/bench/gt1g-get-http-<timestamp>
+  --mc-bin <path>               Default: mc
+  --curl-bin <path>             Default: curl
+  --jq-bin <path>               Default: jq
+  --insecure                    Allow insecure TLS
+  --dry-run
+  -h, --help
+
+Examples:
+  scripts/run_gt1g_get_http_matrix.sh \
+    --endpoint http://127.0.0.1:9000 \
+    --access-key rustfsadmin \
+    --secret-key rustfsadmin \
+    --bucket rustfs-bench \
+    --objects '1GiB=bench/issue713/plain-1g.bin,2GiB=bench/issue713/plain-2g.bin'
+USAGE
+}
+
+arg_value() {
+  local flag="$1"
+  local value="${2:-}"
+  if [[ -z "$value" || "$value" == --* ]]; then
+    echo "ERROR: missing value for $flag" >&2
+    exit 1
+  fi
+  printf '%s\n' "$value"
+}
+
+require_cmd() {
+  if ! command -v "$1" >/dev/null 2>&1; then
+    echo "ERROR: command not found: $1" >&2
+    exit 1
+  fi
+}
+
+run_rust_helper_fallback() {
+  local helper_out_dir="$OUT_DIR"
+  if [[ -n "$helper_out_dir" && "$helper_out_dir" != /* ]]; then
+    helper_out_dir="$PWD/$helper_out_dir"
+  fi
+
+  echo "mc not found; falling back to rustfs/tests/gt1g_get_benchmark_tool.rs"
+
+  GT1G_GET_ACTION=bench \
+  GT1G_GET_ENDPOINT="$ENDPOINT" \
+  GT1G_GET_ACCESS_KEY="$ACCESS_KEY" \
+  GT1G_GET_SECRET_KEY="$SECRET_KEY" \
+  GT1G_GET_BUCKET="$BUCKET" \
+  GT1G_GET_REGION="$REGION" \
+  GT1G_GET_OBJECTS="$OBJECTS" \
+  GT1G_GET_MODES="$MODES" \
+  GT1G_GET_CONCURRENCIES="$CONCURRENCIES" \
+  GT1G_GET_RANGE_WORKERS="$RANGE_WORKERS" \
+  GT1G_GET_ROUNDS="$ROUNDS" \
+  GT1G_GET_COOLDOWN_SECS="$COOLDOWN_SECS" \
+  GT1G_GET_OUT_DIR="$helper_out_dir" \
+  cargo test -p rustfs --test gt1g_get_benchmark_tool gt1g_get_benchmark_tool -- --ignored --nocapture
+}
+
+parse_args() {
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --bucket) BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --objects) OBJECTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --modes) MODES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --range-workers) RANGE_WORKERS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --rounds) ROUNDS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --presign-expiry) PRESIGN_EXPIRY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --mc-bin) MC_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --curl-bin) CURL_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --jq-bin) JQ_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
+      --insecure) INSECURE=true; shift ;;
+      --dry-run) DRY_RUN=true; shift ;;
+      -h|--help) usage; exit 0 ;;
+      *)
+        echo "ERROR: unknown arg: $1" >&2
+        usage
+        exit 1
+        ;;
+    esac
+  done
+}
+
+validate_positive_int() {
+  local value="$1"
+  local label="$2"
+  if ! [[ "$value" =~ ^[0-9]+$ ]] || (( value <= 0 )); then
+    echo "ERROR: $label must be a positive integer, got: $value" >&2
+    exit 1
+  fi
+}
+
+validate_nonnegative_int() {
+  local value="$1"
+  local label="$2"
+  if ! [[ "$value" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: $label must be a nonnegative integer, got: $value" >&2
+    exit 1
+  fi
+}
+
+trim() {
+  echo "$1" | awk '{$1=$1;print}'
+}
+
+setup_output() {
+  if [[ -z "$OUT_DIR" ]]; then
+    OUT_DIR="target/bench/gt1g-get-http-$(date -u +%Y%m%dT%H%M%SZ)"
+  fi
+  LOG_DIR="${OUT_DIR}/logs"
+  mkdir -p "$LOG_DIR"
+  ROUND_CSV="${OUT_DIR}/round_results.csv"
+  MEDIAN_CSV="${OUT_DIR}/median_summary.csv"
+  cat > "$ROUND_CSV" <<'EOF'
+object_label,object_key,mode,range_workers,concurrency,round,status,total_bytes,elapsed_ms,throughput_bps,avg_client_latency_ms,log_file
+EOF
+  cat > "$MEDIAN_CSV" <<'EOF'
+object_label,object_key,mode,range_workers,concurrency,successful_rounds,failed_rounds,median_total_bytes,median_elapsed_ms,median_throughput_bps,median_avg_client_latency_ms
+EOF
+  cat > "${OUT_DIR}/artifact_layout.txt" <<'EOF'
+round_results.csv
+median_summary.csv
+artifact_layout.txt
+logs/
+- <label>_<mode>_rw<workers>_c<concurrency>_r<round>.log
+EOF
+}
+
+now_ms() {
+  perl -MTime::HiRes=time -e 'printf "%.0f\n", time()*1000'
+}
+
+csv_to_lines() {
+  local csv="$1"
+  local raw item
+  IFS=',' read -r -a parts <<< "$csv"
+  for raw in "${parts[@]}"; do
+    item="$(trim "$raw")"
+    [[ -z "$item" ]] && continue
+    echo "$item"
+  done
+}
+
+cooldown_sleep() {
+  local secs="$1"
+  if (( secs <= 0 )); then
+    return 0
+  fi
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "[DRY-RUN] sleep ${secs}s"
+  else
+    sleep "$secs"
+  fi
+}
+
+mc_cmd() {
+  if [[ "$INSECURE" == "true" ]]; then
+    "$MC_BIN" --config-dir "$MC_CONFIG_DIR" --insecure "$@"
+  else
+    "$MC_BIN" --config-dir "$MC_CONFIG_DIR" "$@"
+  fi
+}
+
+setup_mc_alias() {
+  MC_CONFIG_DIR="$(mktemp -d "${TMPDIR:-/tmp}/issue713-gt1g-get-mc.XXXXXX")"
+  trap 'rm -rf "$MC_CONFIG_DIR"' EXIT
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "[DRY-RUN] mc alias set issue713 ${ENDPOINT} REDACTED REDACTED"
+  else
+    mc_cmd alias set issue713 "$ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null
+  fi
+}
+
+get_object_size_bytes() {
+  local object_key="$1"
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo 1073741824
+    return
+  fi
+  mc_cmd stat --json "issue713/${BUCKET}/${object_key}" | "$JQ_BIN" -r '.size'
+}
+
+presign_object_url() {
+  local object_key="$1"
+  if [[ "$DRY_RUN" == "true" ]]; then
+    echo "${ENDPOINT%/}/${BUCKET}/${object_key}"
+    return
+  fi
+  mc_cmd share download --json --expire "$PRESIGN_EXPIRY" "issue713/${BUCKET}/${object_key}" \
+    | "$JQ_BIN" -r '.share // .url // .download // empty'
+}
+
+sum_metric_files() {
+  local total=0 value
+  for metric_file in "$@"; do
+    [[ -f "$metric_file" ]] || continue
+    value="$(tr -d '\r\n' < "$metric_file")"
+    [[ -z "$value" ]] && continue
+    total="$(awk -v a="$total" -v b="$value" 'BEGIN { printf "%.0f\n", a + b }')"
+  done
+  echo "$total"
+}
+
+run_sequential_round() {
+  local url="$1"
+  local object_size="$2"
+  local concurrency="$3"
+  local log_file="$4"
+  local object_label="$5"
+  local round="$6"
+
+  local -a pids=()
+  local -a metric_files=()
+  local task_index metric_file
+  local curl_rc=0
+  local start_ms end_ms elapsed_ms total_bytes avg_latency_ms throughput_bps expected_total_bytes
+
+  start_ms="$(now_ms)"
+  for ((task_index = 1; task_index <= concurrency; task_index++)); do
+    metric_file="${LOG_DIR}/${object_label}_sequential_c${concurrency}_r${round}_t${task_index}.bytes"
+    metric_files+=("$metric_file")
+    (
+      if [[ "$INSECURE" == "true" ]]; then
+        "$CURL_BIN" -fsSL -k "$url" -o /dev/null -w '%{size_download}\n' > "$metric_file"
+      else
+        "$CURL_BIN" -fsSL "$url" -o /dev/null -w '%{size_download}\n' > "$metric_file"
+      fi
+    ) >>"$log_file" 2>&1 &
+    pids+=("$!")
+  done
+
+  for pid in "${pids[@]}"; do
+    if ! wait "$pid"; then
+      curl_rc=1
+    fi
+  done
+
+  end_ms="$(now_ms)"
+  elapsed_ms=$((end_ms - start_ms))
+  total_bytes="$(sum_metric_files "${metric_files[@]}")"
+  expected_total_bytes=$((object_size * concurrency))
+  avg_latency_ms="$(awk -v elapsed="$elapsed_ms" -v conc="$concurrency" 'BEGIN { printf "%.3f\n", elapsed / conc }')"
+  throughput_bps="$(awk -v bytes="$total_bytes" -v elapsed="$elapsed_ms" 'BEGIN { if (elapsed <= 0) printf "0.000000\n"; else printf "%.6f\n", bytes / (elapsed / 1000.0) }')"
+
+  if [[ "$curl_rc" -ne 0 || "$total_bytes" -ne "$expected_total_bytes" ]]; then
+    echo "failed,${total_bytes},${elapsed_ms},${throughput_bps},${avg_latency_ms}"
+  else
+    echo "ok,${total_bytes},${elapsed_ms},${throughput_bps},${avg_latency_ms}"
+  fi
+
+  rm -f "${metric_files[@]}"
+}
+
+run_ranged_parallel_round() {
+  local url="$1"
+  local object_size="$2"
+  local concurrency="$3"
+  local range_workers="$4"
+  local log_file="$5"
+  local object_label="$6"
+  local round="$7"
+
+  local chunk_size=$(( (object_size + range_workers - 1) / range_workers ))
+  local -a pids=()
+  local -a metric_files=()
+  local curl_rc=0
+  local start_ms end_ms elapsed_ms total_bytes avg_latency_ms throughput_bps expected_total_bytes
+  local client_index worker_index range_start range_end metric_file
+
+  start_ms="$(now_ms)"
+  for ((client_index = 1; client_index <= concurrency; client_index++)); do
+    for ((worker_index = 0; worker_index < range_workers; worker_index++)); do
+      range_start=$((worker_index * chunk_size))
+      if (( range_start >= object_size )); then
+        continue
+      fi
+      range_end=$((range_start + chunk_size - 1))
+      if (( range_end >= object_size )); then
+        range_end=$((object_size - 1))
+      fi
+      metric_file="${LOG_DIR}/${object_label}_ranged_rw${range_workers}_c${concurrency}_r${round}_t${client_index}_w${worker_index}.bytes"
+      metric_files+=("$metric_file")
+      (
+        if [[ "$INSECURE" == "true" ]]; then
+          "$CURL_BIN" -fsSL -k -H "Range: bytes=${range_start}-${range_end}" "$url" -o /dev/null -w '%{size_download}\n' > "$metric_file"
+        else
+          "$CURL_BIN" -fsSL -H "Range: bytes=${range_start}-${range_end}" "$url" -o /dev/null -w '%{size_download}\n' > "$metric_file"
+        fi
+      ) >>"$log_file" 2>&1 &
+      pids+=("$!")
+    done
+  done
+
+  for pid in "${pids[@]}"; do
+    if ! wait "$pid"; then
+      curl_rc=1
+    fi
+  done
+
+  end_ms="$(now_ms)"
+  elapsed_ms=$((end_ms - start_ms))
+  total_bytes="$(sum_metric_files "${metric_files[@]}")"
+  expected_total_bytes=$((object_size * concurrency))
+  avg_latency_ms="$(awk -v elapsed="$elapsed_ms" -v conc="$concurrency" 'BEGIN { printf "%.3f\n", elapsed / conc }')"
+  throughput_bps="$(awk -v bytes="$total_bytes" -v elapsed="$elapsed_ms" 'BEGIN { if (elapsed <= 0) printf "0.000000\n"; else printf "%.6f\n", bytes / (elapsed / 1000.0) }')"
+
+  if [[ "$curl_rc" -ne 0 || "$total_bytes" -ne "$expected_total_bytes" ]]; then
+    echo "failed,${total_bytes},${elapsed_ms},${throughput_bps},${avg_latency_ms}"
+  else
+    echo "ok,${total_bytes},${elapsed_ms},${throughput_bps},${avg_latency_ms}"
+  fi
+
+  rm -f "${metric_files[@]}"
+}
+
+median_from_numbers() {
+  local values="$1"
+  local count
+  count="$(printf '%s\n' "$values" | awk 'NF { c++ } END { print c + 0 }')"
+  if [[ "$count" -eq 0 ]]; then
+    echo "N/A"
+    return
+  fi
+
+  printf '%s\n' "$values" | awk 'NF' | sort -n | awk '
+    { a[NR] = $1 }
+    END {
+      n = NR
+      if (n == 0) { print "N/A"; exit }
+      if (n % 2 == 1) {
+        printf "%.6f\n", a[(n + 1) / 2]
+      } else {
+        printf "%.6f\n", (a[n / 2] + a[n / 2 + 1]) / 2
+      }
+    }'
+}
+
+build_median_summary() {
+  local object_label object_key mode range_workers concurrency
+  while IFS=',' read -r object_label object_key mode range_workers concurrency; do
+    local ok_rounds fail_rounds bytes_vals elapsed_vals throughput_vals latency_vals
+    ok_rounds="$(awk -F',' -v l="$object_label" -v o="$object_key" -v m="$mode" -v rw="$range_workers" -v c="$concurrency" 'NR>1 && $1==l && $2==o && $3==m && $4==rw && $5==c && $7=="ok" {count++} END{print count+0}' "$ROUND_CSV")"
+    fail_rounds="$(awk -F',' -v l="$object_label" -v o="$object_key" -v m="$mode" -v rw="$range_workers" -v c="$concurrency" 'NR>1 && $1==l && $2==o && $3==m && $4==rw && $5==c && $7!="ok" {count++} END{print count+0}' "$ROUND_CSV")"
+
+    bytes_vals="$(awk -F',' -v l="$object_label" -v o="$object_key" -v m="$mode" -v rw="$range_workers" -v c="$concurrency" 'NR>1 && $1==l && $2==o && $3==m && $4==rw && $5==c && $7=="ok" {print $8}' "$ROUND_CSV")"
+    elapsed_vals="$(awk -F',' -v l="$object_label" -v o="$object_key" -v m="$mode" -v rw="$range_workers" -v c="$concurrency" 'NR>1 && $1==l && $2==o && $3==m && $4==rw && $5==c && $7=="ok" {print $9}' "$ROUND_CSV")"
+    throughput_vals="$(awk -F',' -v l="$object_label" -v o="$object_key" -v m="$mode" -v rw="$range_workers" -v c="$concurrency" 'NR>1 && $1==l && $2==o && $3==m && $4==rw && $5==c && $7=="ok" {print $10}' "$ROUND_CSV")"
+    latency_vals="$(awk -F',' -v l="$object_label" -v o="$object_key" -v m="$mode" -v rw="$range_workers" -v c="$concurrency" 'NR>1 && $1==l && $2==o && $3==m && $4==rw && $5==c && $7=="ok" {print $11}' "$ROUND_CSV")"
+
+    echo "${object_label},${object_key},${mode},${range_workers},${concurrency},${ok_rounds},${fail_rounds},$(median_from_numbers "$bytes_vals"),$(median_from_numbers "$elapsed_vals"),$(median_from_numbers "$throughput_vals"),$(median_from_numbers "$latency_vals")" \
+      >> "$MEDIAN_CSV"
+  done < <(
+    awk -F',' 'NR>1 { print $1 "," $2 "," $3 "," $4 "," $5 }' "$ROUND_CSV" | sort -u
+  )
+}
+
+main() {
+  parse_args "$@"
+  if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" || -z "$BUCKET" || -z "$OBJECTS" ]]; then
+    echo "ERROR: --endpoint, --access-key, --secret-key, --bucket, and --objects are required" >&2
+    exit 1
+  fi
+  validate_positive_int "$ROUNDS" "--rounds"
+  validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs"
+  if ! command -v "$MC_BIN" >/dev/null 2>&1; then
+    setup_output
+    run_rust_helper_fallback
+    exit 0
+  fi
+
+  require_cmd "$MC_BIN"
+  require_cmd "$CURL_BIN"
+  require_cmd "$JQ_BIN"
+  require_cmd awk
+  require_cmd sort
+  require_cmd perl
+
+  setup_output
+  setup_mc_alias
+
+  IFS=',' read -r -a object_specs <<< "$OBJECTS"
+  for raw_object_spec in "${object_specs[@]}"; do
+    local object_spec object_label object_key object_size presigned_url
+    object_spec="$(trim "$raw_object_spec")"
+    [[ -z "$object_spec" ]] && continue
+    object_label="${object_spec%%=*}"
+    object_key="${object_spec#*=}"
+    if [[ -z "$object_label" || -z "$object_key" || "$object_label" == "$object_key" ]]; then
+      echo "ERROR: invalid object spec: $object_spec" >&2
+      exit 1
+    fi
+
+    object_size="$(get_object_size_bytes "$object_key")"
+    presigned_url="$(presign_object_url "$object_key")"
+    if [[ -z "$presigned_url" ]]; then
+      echo "ERROR: failed to generate presigned URL for ${object_key}" >&2
+      exit 1
+    fi
+
+    for mode in $(csv_to_lines "$MODES"); do
+      for concurrency in $(csv_to_lines "$CONCURRENCIES"); do
+        validate_positive_int "$concurrency" "--concurrencies item"
+
+        if [[ "$mode" == "sequential" ]]; then
+          local round result status total_bytes elapsed_ms throughput_bps avg_latency_ms log_file
+          for ((round = 1; round <= ROUNDS; round++)); do
+            log_file="${LOG_DIR}/${object_label}_${mode}_rw1_c${concurrency}_r${round}.log"
+            result="$(run_sequential_round "$presigned_url" "$object_size" "$concurrency" "$log_file" "$object_label" "$round")"
+            IFS=',' read -r status total_bytes elapsed_ms throughput_bps avg_latency_ms <<< "$result"
+            echo "${object_label},${object_key},${mode},1,${concurrency},${round},${status},${total_bytes},${elapsed_ms},${throughput_bps},${avg_latency_ms},${log_file}" >> "$ROUND_CSV"
+            if (( COOLDOWN_SECS > 0 && round < ROUNDS )); then
+              cooldown_sleep "$COOLDOWN_SECS"
+            fi
+          done
+          continue
+        fi
+
+        if [[ "$mode" == "ranged_parallel" ]]; then
+          for range_workers in $(csv_to_lines "$RANGE_WORKERS"); do
+            validate_positive_int "$range_workers" "--range-workers item"
+            local round result status total_bytes elapsed_ms throughput_bps avg_latency_ms log_file
+            for ((round = 1; round <= ROUNDS; round++)); do
+              log_file="${LOG_DIR}/${object_label}_${mode}_rw${range_workers}_c${concurrency}_r${round}.log"
+              result="$(run_ranged_parallel_round "$presigned_url" "$object_size" "$concurrency" "$range_workers" "$log_file" "$object_label" "$round")"
+              IFS=',' read -r status total_bytes elapsed_ms throughput_bps avg_latency_ms <<< "$result"
+              echo "${object_label},${object_key},${mode},${range_workers},${concurrency},${round},${status},${total_bytes},${elapsed_ms},${throughput_bps},${avg_latency_ms},${log_file}" >> "$ROUND_CSV"
+              if (( COOLDOWN_SECS > 0 && round < ROUNDS )); then
+                cooldown_sleep "$COOLDOWN_SECS"
+              fi
+            done
+          done
+          continue
+        fi
+
+        echo "ERROR: unsupported mode: $mode" >&2
+        exit 1
+      done
+    done
+  done
+
+  build_median_summary
+
+  echo "Artifacts written to: $OUT_DIR"
+  echo "Summaries:"
+  echo "  - $ROUND_CSV"
+  echo "  - $MEDIAN_CSV"
+}
+
+main "$@"