gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(perf): add large PUT tuning and encode optimization (#3816)

Browse Source
This commit is contained in:
houseme
2026-06-24 19:04:04 +08:00
committed by GitHub
parent 3c6e9acfcb
commit efebcb66be
14 changed files with 3120 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/collect_put_large_stage_breakdown_artifacts.sh
Executable
+468
View File
@@ -0,0 +1,468 @@
#!/usr/bin/env bash
set -euo pipefail
# Helper artifact collector for rustfs/backlog#706 large-object PUT stage-breakdown runs.
# Designed to pair with scripts/run_put_large_stage_breakdown.sh and store
# supporting evidence under:
# <run-root>/captures/<label>/
#
# Captures:
# - health and readiness snapshots
# - signed admin metrics snapshots per endpoint
# - optional plain Prometheus text snapshots
# - optional host telemetry (pidstat/iostat/mpstat)
# - per-sample process snapshots when a rustfs pid is available
RUN_ROOT=""
LABEL=""
ENDPOINT=""
ACCESS_KEY="${RUSTFS_ACCESS_KEY:-}"
SECRET_KEY=""
SECRET_KEY_ENV="RUSTFS_SECRET_KEY"
REGION="us-east-1"
METRICS_ENDPOINTS=""
PROM_METRICS_URLS=""
DURATION_SECS=180
INTERVAL_SECS=15
RUSTFS_PID=""
SKIP_HOST_TELEMETRY=false
AWSCURL_BIN="awscurl"
CURL_BIN="curl"
JQ_BIN="jq"
DRY_RUN=false
usage() {
cat <<'USAGE'
Usage:
scripts/collect_put_large_stage_breakdown_artifacts.sh --run-root <dir> \
--endpoint <url> [options]
Required:
--run-root <dir> Benchmark run root, usually from
scripts/run_put_large_stage_breakdown.sh
--endpoint <url> RustFS endpoint, e.g. http://127.0.0.1:9000
Auth / signed admin metrics:
--access-key <ak> Override RUSTFS_ACCESS_KEY
--secret-key-env <var> Secret-key environment variable
(default: RUSTFS_SECRET_KEY)
--region <name> SigV4 region (default: us-east-1)
Capture options:
--label <name> Capture label (default: capture-<timestamp>)
--metrics-endpoints <csv> Signed admin metrics endpoints. If omitted,
defaults to --endpoint.
--prom-metrics-urls <csv> Optional plain Prometheus text endpoints
captured with curl.
--duration-secs <n> Total capture duration (default: 180)
--interval-secs <n> Sample interval (default: 15)
--rustfs-pid <pid> RustFS pid. Auto-detect with pidof rustfs if omitted.
--skip-host-telemetry Skip pidstat/iostat/mpstat collection.
--dry-run Print intended actions without live capture.
Output:
<run-root>/captures/<label>/
capture-meta.txt
capture-layout.txt
capture-report.md
health/
metrics/
prom/
host/
USAGE
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "ERROR: command not found: $1" >&2
exit 1
fi
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s\n' "$value"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--run-root) RUN_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--label) LABEL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--secret-key-env) SECRET_KEY_ENV="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--metrics-endpoints) METRICS_ENDPOINTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--prom-metrics-urls) PROM_METRICS_URLS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--duration-secs) DURATION_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--interval-secs) INTERVAL_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--rustfs-pid) RUSTFS_PID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--skip-host-telemetry) SKIP_HOST_TELEMETRY=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
is_nonnegative_integer() {
[[ "$1" =~ ^[0-9]+$ ]]
}
is_positive_integer() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
validate_args() {
if [[ -z "$RUN_ROOT" || -z "$ENDPOINT" ]]; then
echo "ERROR: --run-root and --endpoint are required" >&2
exit 1
fi
if ! [[ "$SECRET_KEY_ENV" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
echo "ERROR: --secret-key-env must be a valid environment variable name" >&2
exit 1
fi
if ! is_nonnegative_integer "$DURATION_SECS" || ! is_positive_integer "$INTERVAL_SECS"; then
echo "ERROR: --duration-secs must be >= 0 and --interval-secs must be > 0" >&2
exit 1
fi
SECRET_KEY="${!SECRET_KEY_ENV:-}"
if [[ -z "$LABEL" ]]; then
LABEL="capture-$(date -u +%Y%m%dT%H%M%SZ)"
fi
}
setup_output() {
CAPTURE_ROOT="${RUN_ROOT}/captures/${LABEL}"
HEALTH_DIR="${CAPTURE_ROOT}/health"
METRICS_DIR="${CAPTURE_ROOT}/metrics"
PROM_DIR="${CAPTURE_ROOT}/prom"
HOST_DIR="${CAPTURE_ROOT}/host"
mkdir -p "$HEALTH_DIR" "$METRICS_DIR" "$PROM_DIR" "$HOST_DIR"
}
endpoint_label() {
local endpoint="$1"
local label
label="${endpoint#*://}"
label="${label%%/*}"
printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_'
}
csv_to_lines() {
local csv="$1"
local raw item
IFS=',' read -r -a arr <<< "$csv"
for raw in "${arr[@]}"; do
item="$(echo "$raw" | awk '{$1=$1;print}')"
[[ -z "$item" ]] && continue
echo "$item"
done
}
resolve_metrics_endpoints() {
if [[ -n "$METRICS_ENDPOINTS" ]]; then
csv_to_lines "$METRICS_ENDPOINTS"
else
printf '%s\n' "$ENDPOINT"
fi
}
resolve_pid() {
if [[ -n "$RUSTFS_PID" ]]; then
echo "$RUSTFS_PID"
return
fi
pidof rustfs 2>/dev/null | awk '{print $1}' || true
}
health_url() {
printf '%s/health\n' "${1%/}"
}
ready_url() {
printf '%s/health/ready\n' "${1%/}"
}
admin_metrics_url() {
printf '%s/rustfs/admin/v3/metrics?types=1&by-host=true&n=1\n' "${1%/}"
}
write_layout_file() {
cat > "${CAPTURE_ROOT}/capture-layout.txt" <<'EOF'
capture-meta.txt
capture-layout.txt
capture-report.md
health/
- health.<endpoint>.<index>.<timestamp>.txt
- ready.<endpoint>.<index>.<timestamp>.txt
metrics/
- admin-metrics.<endpoint>.<index>.<timestamp>.ndjson
prom/
- prometheus.<endpoint>.<index>.<timestamp>.prom
host/
- pidstat.txt
- iostat.txt
- mpstat.txt
- proc-status.<index>.<timestamp>.txt
- proc-io.<index>.<timestamp>.txt
- ps.<index>.<timestamp>.txt
EOF
}
write_meta_file() {
local git_commit git_branch
git_commit="$(git rev-parse HEAD 2>/dev/null || echo "unknown")"
git_branch="$(git branch --show-current 2>/dev/null || echo "unknown")"
cat > "${CAPTURE_ROOT}/capture-meta.txt" <<EOF
created_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
run_root=${RUN_ROOT}
label=${LABEL}
endpoint=${ENDPOINT}
metrics_endpoints=${METRICS_ENDPOINTS:-$ENDPOINT}
prom_metrics_urls=${PROM_METRICS_URLS:-N/A}
region=${REGION}
duration_secs=${DURATION_SECS}
interval_secs=${INTERVAL_SECS}
rustfs_pid=${RUSTFS_PID:-auto}
resolved_pid=${RESOLVED_PID:-N/A}
skip_host_telemetry=${SKIP_HOST_TELEMETRY}
dry_run=${DRY_RUN}
git_branch=${git_branch}
git_commit=${git_commit}
access_key_present=$([[ -n "$ACCESS_KEY" ]] && echo true || echo false)
secret_key_env=${SECRET_KEY_ENV}
EOF
}
capture_health_sample() {
local index="$1"
local ts="$2"
local endpoint="$3"
local label file
label="$(endpoint_label "$endpoint")"
file="${HEALTH_DIR}/health.${label}.${index}.${ts}.txt"
if [[ "$DRY_RUN" == "true" ]]; then
echo "dry run" > "$file"
else
"$CURL_BIN" -fsS "$(health_url "$endpoint")" > "$file" 2>&1 || true
fi
file="${HEALTH_DIR}/ready.${label}.${index}.${ts}.txt"
if [[ "$DRY_RUN" == "true" ]]; then
echo "dry run" > "$file"
else
"$CURL_BIN" -fsS "$(ready_url "$endpoint")" > "$file" 2>&1 || true
fi
}
capture_admin_metrics_sample() {
local index="$1"
local ts="$2"
local endpoint="$3"
local label file
label="$(endpoint_label "$endpoint")"
file="${METRICS_DIR}/admin-metrics.${label}.${index}.${ts}.ndjson"
if [[ "$DRY_RUN" == "true" ]]; then
echo "dry run" > "$file"
return
fi
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
echo "missing access/secret for signed metrics capture" > "$file"
return
fi
AWS_ACCESS_KEY_ID="$ACCESS_KEY" \
AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \
AWS_DEFAULT_REGION="$REGION" \
"$AWSCURL_BIN" \
--service s3 \
--region "$REGION" \
--request GET \
"$(admin_metrics_url "$endpoint")" \
> "$file" 2>&1 || true
}
capture_prometheus_sample() {
local index="$1"
local ts="$2"
local endpoint="$3"
local label file
label="$(endpoint_label "$endpoint")"
file="${PROM_DIR}/prometheus.${label}.${index}.${ts}.prom"
if [[ "$DRY_RUN" == "true" ]]; then
echo "dry run" > "$file"
else
"$CURL_BIN" -fsS "$endpoint" > "$file" 2>&1 || true
fi
}
capture_proc_snapshot() {
local index="$1"
local ts="$2"
local pid="$3"
[[ -n "$pid" ]] || return 0
if [[ -r "/proc/${pid}/status" ]]; then
cp "/proc/${pid}/status" "${HOST_DIR}/proc-status.${index}.${ts}.txt" || true
fi
if [[ -r "/proc/${pid}/io" ]]; then
cp "/proc/${pid}/io" "${HOST_DIR}/proc-io.${index}.${ts}.txt" || true
fi
if command -v ps >/dev/null 2>&1; then
ps -p "$pid" -o pid,ppid,stat,pcpu,pmem,rss,vsz,etime,args > "${HOST_DIR}/ps.${index}.${ts}.txt" 2>&1 || true
fi
}
start_host_telemetry() {
HOST_TELEMETRY_PIDS=()
if [[ "$SKIP_HOST_TELEMETRY" == "true" ]]; then
return
fi
local count
count=$(( DURATION_SECS / INTERVAL_SECS + 1 ))
(( count < 1 )) && count=1
if [[ -n "$RESOLVED_PID" ]] && command -v pidstat >/dev/null 2>&1; then
pidstat -durwh -p "$RESOLVED_PID" "$INTERVAL_SECS" "$count" > "${HOST_DIR}/pidstat.txt" 2>&1 &
HOST_TELEMETRY_PIDS+=("$!")
fi
if command -v iostat >/dev/null 2>&1; then
iostat -xz "$INTERVAL_SECS" "$count" > "${HOST_DIR}/iostat.txt" 2>&1 &
HOST_TELEMETRY_PIDS+=("$!")
fi
if command -v mpstat >/dev/null 2>&1; then
mpstat "$INTERVAL_SECS" "$count" > "${HOST_DIR}/mpstat.txt" 2>&1 &
HOST_TELEMETRY_PIDS+=("$!")
fi
}
wait_host_telemetry() {
local pid
for pid in "${HOST_TELEMETRY_PIDS[@]:-}"; do
wait "$pid" || true
done
}
capture_one_sample() {
local index="$1"
local ts endpoint prom_url
ts="$(date -u +%Y%m%dT%H%M%SZ)"
while IFS= read -r endpoint; do
capture_health_sample "$index" "$ts" "$endpoint"
capture_admin_metrics_sample "$index" "$ts" "$endpoint"
done < <(resolve_metrics_endpoints)
if [[ -n "$PROM_METRICS_URLS" ]]; then
while IFS= read -r prom_url; do
capture_prometheus_sample "$index" "$ts" "$prom_url"
done < <(csv_to_lines "$PROM_METRICS_URLS")
fi
if [[ "$DRY_RUN" != "true" ]]; then
capture_proc_snapshot "$index" "$ts" "$RESOLVED_PID"
fi
}
capture_series() {
local total_samples index
total_samples=$(( DURATION_SECS / INTERVAL_SECS + 1 ))
(( total_samples < 1 )) && total_samples=1
for ((index = 1; index <= total_samples; index++)); do
capture_one_sample "$index"
if (( index < total_samples )); then
sleep "$INTERVAL_SECS"
fi
done
}
count_artifacts() {
local dir="$1"
local pattern="$2"
find "$dir" -type f -name "$pattern" 2>/dev/null | wc -l | tr -d ' '
}
write_report() {
local health_count metrics_count prom_count proc_count
health_count="$(count_artifacts "$HEALTH_DIR" 'health.*.txt')"
metrics_count="$(count_artifacts "$METRICS_DIR" 'admin-metrics.*.ndjson')"
prom_count="$(count_artifacts "$PROM_DIR" 'prometheus.*.prom')"
proc_count="$(count_artifacts "$HOST_DIR" 'proc-status.*.txt')"
cat > "${CAPTURE_ROOT}/capture-report.md" <<EOF
## Large PUT Stage-Breakdown Capture Report
Run root: ${RUN_ROOT}
Label: ${LABEL}
Endpoint: ${ENDPOINT}
Duration seconds: ${DURATION_SECS}
Interval seconds: ${INTERVAL_SECS}
Resolved pid: ${RESOLVED_PID:-N/A}
## Artifact Summary
- Health snapshots: ${health_count}
- Signed admin metrics snapshots: ${metrics_count}
- Plain Prometheus snapshots: ${prom_count}
- Process snapshots: ${proc_count}
- Host telemetry present: $([[ -s "${HOST_DIR}/pidstat.txt" || -s "${HOST_DIR}/iostat.txt" || -s "${HOST_DIR}/mpstat.txt" ]] && echo yes || echo no)
## Review Checklist
- Align capture timestamps with benchmark windows in \`runs/cXX/\`.
- Check whether health or readiness degraded during the benchmark window.
- Compare admin metrics snapshots with the Prometheus / Grafana queries used for stage interpretation.
- Attach pidstat/iostat/mpstat when evaluating CPU, RSS, and disk pressure.
- Use the capture directory together with \`aggregate_median_summary.csv\` when filling the stage-breakdown report template.
EOF
}
main() {
parse_args "$@"
validate_args
require_cmd bash
require_cmd "$CURL_BIN"
require_cmd awk
if [[ "$DRY_RUN" != "true" ]]; then
require_cmd date
fi
if [[ "$DRY_RUN" != "true" && -n "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
require_cmd "$AWSCURL_BIN"
fi
setup_output
RESOLVED_PID="$(resolve_pid)"
write_layout_file
write_meta_file
start_host_telemetry
capture_series
wait_host_telemetry
write_report
echo "Capture artifacts written to ${CAPTURE_ROOT}"
}
main "$@"
scripts/run_object_batch_bench_enhanced.sh
+38
View File
@@ -34,6 +34,7 @@ SAMPLES=20000
ROUNDS=3
RETRY_PER_ROUND=2
RETRY_SLEEP_SECS=2
COOLDOWN_SECS=0
BASELINE_CSV=""
EXTRA_ARGS=()
@@ -71,6 +72,7 @@ Enhanced options:
--rounds Benchmark rounds per size (default: 3)
--retry-per-round Retry count per failed round (default: 2)
--retry-sleep-secs Sleep seconds between retries (default: 2)
--cooldown-secs Sleep seconds between rounds/sizes (default: 0)
--baseline-csv Baseline median CSV to compare
--extra-args Extra args appended to tool command, quoted as one string
@@ -127,6 +129,7 @@ parse_args() {
--rounds) ROUNDS="$2"; shift 2 ;;
--retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;;
--retry-sleep-secs) RETRY_SLEEP_SECS="$2"; shift 2 ;;
--cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
--baseline-csv) BASELINE_CSV="$2"; shift 2 ;;
--extra-args)
# shellcheck disable=SC2206
@@ -152,6 +155,27 @@ validate_positive_int() {
fi
}
validate_nonnegative_int() {
local v="$1"
local n="$2"
if ! [[ "$v" =~ ^[0-9]+$ ]]; then
echo "ERROR: $n must be a nonnegative integer, got: $v" >&2
exit 1
fi
}
cooldown_sleep() {
local secs="$1"
if (( secs <= 0 )); then
return 0
fi
if [[ "$DRY_RUN" == "true" ]]; then
echo "[DRY-RUN] sleep ${secs}"
else
sleep "$secs"
fi
}
validate_args() {
if [[ "$TOOL" != "warp" && "$TOOL" != "s3bench" ]]; then
echo "ERROR: --tool must be warp or s3bench" >&2
@@ -165,6 +189,7 @@ validate_args() {
validate_positive_int "$ROUNDS" "--rounds"
validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round"
validate_positive_int "$RETRY_SLEEP_SECS" "--retry-sleep-secs"
validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs"
if [[ "$TOOL" == "s3bench" ]]; then
validate_positive_int "$SAMPLES" "--samples"
fi
@@ -416,6 +441,11 @@ run_size() {
if [[ "$success" == "no" ]]; then
echo "WARN: size=$size round=$round failed after retries."
fi
if (( COOLDOWN_SECS > 0 && round < ROUNDS )); then
echo "Cooldown after size=$size round=$round: ${COOLDOWN_SECS}s"
cooldown_sleep "$COOLDOWN_SECS"
fi
done
}
@@ -498,12 +528,20 @@ main() {
echo "Concurrency: $CONCURRENCY"
echo "Rounds: $ROUNDS"
echo "Retry per round: $RETRY_PER_ROUND"
echo "Cooldown secs: $COOLDOWN_SECS"
IFS=',' read -r -a size_arr <<< "$SIZES"
local total_sizes="${#size_arr[@]}"
local size_index=0
for raw in "${size_arr[@]}"; do
size="$(trim "$raw")"
[[ -z "$size" ]] && continue
size_index=$(( size_index + 1 ))
run_size "$size"
if (( COOLDOWN_SECS > 0 && size_index < total_sizes )); then
echo "Cooldown after size=$size: ${COOLDOWN_SECS}s"
cooldown_sleep "$COOLDOWN_SECS"
fi
done
build_median_summary
scripts/run_put_large_stage_breakdown.sh
Executable
+460
View File
@@ -0,0 +1,460 @@
#!/usr/bin/env bash
set -euo pipefail
# Large-object PUT stage-breakdown runner for rustfs/backlog#706.
# This wrapper standardizes:
# - object sizes and concurrency matrix for 16MiB / 32MiB PUT runs
# - output directory layout under target/bench/
# - artifact naming and run manifest capture
# - optional baseline comparison reuse across per-concurrency runs
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
ENHANCED_BENCH_SCRIPT="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh"
TOOL="warp"
ENDPOINT=""
ACCESS_KEY=""
SECRET_KEY=""
REGION="us-east-1"
BUCKET_PREFIX="rustfs-put-large"
CONCURRENCIES="16,32,64,96,128"
SIZES="16MiB,32MiB"
DURATION="120s"
ROUNDS=3
RETRY_PER_ROUND=1
RETRY_SLEEP_SECS=2
COOLDOWN_SECS=0
WARP_BIN="${WARP_BIN:-warp}"
OUT_DIR=""
BASELINE_ROOT=""
EXTRA_ARGS=""
INSECURE=false
DRY_RUN=false
TOPOLOGY_NODES=""
TOPOLOGY_DISKS_PER_NODE=""
TOPOLOGY_TOTAL_DISKS=""
TOPOLOGY_CPU_PER_NODE=""
TOPOLOGY_MEM_PER_NODE=""
TOPOLOGY_NETWORK=""
TOPOLOGY_ENDPOINT_MODE=""
TOPOLOGY_ERASURE_SET_DRIVE_COUNT=""
CLIENT_HOST=""
WORKLOAD_LABEL="backlog-706-large-put-stage-breakdown"
usage() {
cat <<'USAGE'
Usage:
scripts/run_put_large_stage_breakdown.sh --endpoint <url> \
--access-key <ak> --secret-key <sk> [options]
Required:
--endpoint <url> S3 endpoint
--access-key <ak> Access key
--secret-key <sk> Secret key
Core options:
--bucket-prefix <prefix> Bucket prefix (default: rustfs-put-large)
--region <name> Region (default: us-east-1)
--sizes <csv> Object sizes (default: 16MiB,32MiB)
--concurrencies <csv> Concurrency matrix (default: 16,32,64,96,128)
--duration <dur> Per-run duration (default: 120s)
--rounds <n> Rounds per size (default: 3)
--retry-per-round <n> Retries per failed round (default: 1)
--retry-sleep-secs <n> Sleep between retries (default: 2)
--cooldown-secs <n> Sleep between rounds/sizes (default: 0)
--out-dir <dir> Output root (default: target/bench/put-large-stage-breakdown-<timestamp>)
--baseline-root <dir> Existing root from a previous run of this script
--extra-args "<args>" Extra args passed to run_object_batch_bench_enhanced.sh
--warp-bin <path> warp binary (default: warp)
--insecure Pass --insecure to warp
--dry-run Print commands only
Topology metadata (optional but recommended):
--nodes <n>
--disks-per-node <n>
--total-disks <n>
--cpu-per-node <text>
--mem-per-node <text>
--network <text>
--endpoint-mode <direct|lb>
--erasure-set-drive-count <n>
--client-host <text>
--workload-label <text>
Output layout:
<out-dir>/
run_manifest.txt
run_matrix.csv
aggregate_median_summary.csv
aggregate_baseline_compare.csv
artifact_layout.txt
runs/
c16/
c32/
...
Each runs/cXX directory is a direct output directory of:
scripts/run_object_batch_bench_enhanced.sh
USAGE
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "ERROR: command not found: $1" >&2
exit 1
fi
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s\n' "$value"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--bucket-prefix) BUCKET_PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--sizes) SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--rounds) ROUNDS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--retry-per-round) RETRY_PER_ROUND="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--retry-sleep-secs) RETRY_SLEEP_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--baseline-root) BASELINE_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--extra-args) EXTRA_ARGS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--warp-bin) WARP_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--nodes) TOPOLOGY_NODES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--disks-per-node) TOPOLOGY_DISKS_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--total-disks) TOPOLOGY_TOTAL_DISKS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cpu-per-node) TOPOLOGY_CPU_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--mem-per-node) TOPOLOGY_MEM_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--network) TOPOLOGY_NETWORK="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--endpoint-mode) TOPOLOGY_ENDPOINT_MODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--erasure-set-drive-count) TOPOLOGY_ERASURE_SET_DRIVE_COUNT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--client-host) CLIENT_HOST="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--workload-label) WORKLOAD_LABEL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--insecure) INSECURE=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
is_positive_int() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
is_nonnegative_int() {
[[ "$1" =~ ^[0-9]+$ ]]
}
cooldown_sleep() {
local secs="$1"
if (( secs <= 0 )); then
return 0
fi
if [[ "$DRY_RUN" == "true" ]]; then
echo "[DRY-RUN] sleep ${secs}"
else
sleep "$secs"
fi
}
validate_args() {
if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
echo "ERROR: --endpoint, --access-key, and --secret-key are required" >&2
exit 1
fi
if [[ "$TOOL" != "warp" ]]; then
echo "ERROR: only warp is supported by this wrapper" >&2
exit 1
fi
if ! is_positive_int "$ROUNDS" || ! is_positive_int "$RETRY_PER_ROUND" || ! is_positive_int "$RETRY_SLEEP_SECS" || ! is_nonnegative_int "$COOLDOWN_SECS"; then
echo "ERROR: --rounds, --retry-per-round, and --retry-sleep-secs must be positive integers; --cooldown-secs must be a nonnegative integer" >&2
exit 1
fi
if [[ -n "$BASELINE_ROOT" && ! -d "$BASELINE_ROOT" ]]; then
echo "ERROR: --baseline-root does not exist: $BASELINE_ROOT" >&2
exit 1
fi
}
setup_output() {
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="target/bench/put-large-stage-breakdown-$(date -u +%Y%m%dT%H%M%SZ)"
fi
RUNS_DIR="${OUT_DIR}/runs"
AGG_MEDIAN_CSV="${OUT_DIR}/aggregate_median_summary.csv"
AGG_COMPARE_CSV="${OUT_DIR}/aggregate_baseline_compare.csv"
RUN_MATRIX_CSV="${OUT_DIR}/run_matrix.csv"
mkdir -p "$RUNS_DIR"
echo "concurrency,size,tool,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,bucket,run_dir" > "$AGG_MEDIAN_CSV"
echo "concurrency,size,tool,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct,run_dir" > "$AGG_COMPARE_CSV"
echo "concurrency,bucket,run_dir,baseline_csv,status" > "$RUN_MATRIX_CSV"
}
join_bool() {
if [[ "$1" == "true" ]]; then
echo "true"
else
echo "false"
fi
}
write_manifest() {
local git_commit git_branch git_dirty rustc_version
git_commit="$(git -C "$PROJECT_ROOT" rev-parse HEAD 2>/dev/null || echo "unknown")"
git_branch="$(git -C "$PROJECT_ROOT" branch --show-current 2>/dev/null || echo "unknown")"
if [[ -n "$(git -C "$PROJECT_ROOT" status --porcelain 2>/dev/null || true)" ]]; then
git_dirty="true"
else
git_dirty="false"
fi
rustc_version="$(rustc --version 2>/dev/null || echo "unknown")"
{
echo "created_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "git_commit=${git_commit}"
echo "git_branch=${git_branch}"
echo "git_dirty=${git_dirty}"
echo "rustc_version=${rustc_version}"
echo "kernel=$(uname -srvmo 2>/dev/null || echo "unknown")"
echo "tool=${TOOL}"
echo "endpoint=${ENDPOINT}"
echo "region=${REGION}"
echo "bucket_prefix=${BUCKET_PREFIX}"
echo "sizes=${SIZES}"
echo "concurrencies=${CONCURRENCIES}"
echo "duration=${DURATION}"
echo "rounds=${ROUNDS}"
echo "retry_per_round=${RETRY_PER_ROUND}"
echo "retry_sleep_secs=${RETRY_SLEEP_SECS}"
echo "cooldown_secs=${COOLDOWN_SECS}"
echo "insecure=$(join_bool "$INSECURE")"
echo "dry_run=$(join_bool "$DRY_RUN")"
echo "baseline_root=${BASELINE_ROOT:-N/A}"
echo "extra_args_present=$([[ -n "$EXTRA_ARGS" ]] && echo true || echo false)"
echo "workload_label=${WORKLOAD_LABEL}"
echo "nodes=${TOPOLOGY_NODES:-N/A}"
echo "disks_per_node=${TOPOLOGY_DISKS_PER_NODE:-N/A}"
echo "total_disks=${TOPOLOGY_TOTAL_DISKS:-N/A}"
echo "cpu_per_node=${TOPOLOGY_CPU_PER_NODE:-N/A}"
echo "mem_per_node=${TOPOLOGY_MEM_PER_NODE:-N/A}"
echo "network=${TOPOLOGY_NETWORK:-N/A}"
echo "endpoint_mode=${TOPOLOGY_ENDPOINT_MODE:-N/A}"
echo "erasure_set_drive_count=${TOPOLOGY_ERASURE_SET_DRIVE_COUNT:-N/A}"
echo "client_host=${CLIENT_HOST:-N/A}"
echo "access_key=REDACTED"
echo "secret_key=REDACTED"
} > "${OUT_DIR}/run_manifest.txt"
}
write_artifact_layout() {
cat > "${OUT_DIR}/artifact_layout.txt" <<'EOF'
Top-level artifacts:
- run_manifest.txt: run metadata, git revision, topology notes, and redacted execution context
- run_matrix.csv: one row per concurrency run, including bucket and baseline linkage
- aggregate_median_summary.csv: merged median_summary rows from every concurrency run
- aggregate_baseline_compare.csv: merged baseline_compare rows when a matching baseline exists
- runs/cXX/: direct output directory of scripts/run_object_batch_bench_enhanced.sh
Per-concurrency directory:
- round_results.csv
- median_summary.csv
- baseline_compare.csv (only when a baseline CSV is supplied)
- logs/
EOF
}
trim() {
echo "$1" | awk '{$1=$1;print}'
}
csv_to_lines() {
local csv="$1"
IFS=',' read -r -a arr <<< "$csv"
for raw in "${arr[@]}"; do
local item
item="$(trim "$raw")"
[[ -z "$item" ]] && continue
echo "$item"
done
}
sanitize_bucket_component() {
echo "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/^-+//; s/-+$//; s/-{2,}/-/g'
}
build_bucket_name() {
local concurrency="$1"
local run_id prefix raw
run_id="$(date -u +%Y%m%d%H%M%S)"
prefix="$(sanitize_bucket_component "$BUCKET_PREFIX")"
raw="${prefix}-${run_id}-c${concurrency}"
raw="$(sanitize_bucket_component "$raw")"
echo "${raw:0:63}"
}
resolve_baseline_csv() {
local concurrency="$1"
local candidate=""
if [[ -z "$BASELINE_ROOT" ]]; then
return 0
fi
if [[ -f "${BASELINE_ROOT}/runs/c${concurrency}/median_summary.csv" ]]; then
candidate="${BASELINE_ROOT}/runs/c${concurrency}/median_summary.csv"
elif [[ -f "${BASELINE_ROOT}/c${concurrency}/median_summary.csv" ]]; then
candidate="${BASELINE_ROOT}/c${concurrency}/median_summary.csv"
fi
if [[ -n "$candidate" ]]; then
printf '%s\n' "$candidate"
fi
}
append_aggregate_rows() {
local concurrency="$1"
local bucket="$2"
local run_dir="$3"
local median_csv="$4"
local compare_csv="$5"
awk -F',' -v c="$concurrency" -v b="$bucket" -v rd="$run_dir" 'NR>1 {print c "," $1 "," $2 "," $4 "," $5 "," $6 "," $7 "," $8 "," b "," rd}' "$median_csv" >> "$AGG_MEDIAN_CSV"
if [[ -f "$compare_csv" ]]; then
awk -F',' -v c="$concurrency" -v rd="$run_dir" 'NR>1 {print c "," $1 "," $2 "," $4 "," $5 "," $6 "," $7 "," $8 "," $9 "," $10 "," $11 "," $12 "," rd}' "$compare_csv" >> "$AGG_COMPARE_CSV"
fi
}
run_concurrency() {
local concurrency="$1"
local run_dir bucket baseline_csv status
run_dir="${RUNS_DIR}/c${concurrency}"
bucket="$(build_bucket_name "$concurrency")"
baseline_csv="$(resolve_baseline_csv "$concurrency" || true)"
status="pending"
local -a cmd=(
bash "$ENHANCED_BENCH_SCRIPT"
--tool "$TOOL"
--endpoint "$ENDPOINT"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--bucket "$bucket"
--region "$REGION"
--warp-bin "$WARP_BIN"
--warp-mode put
--sizes "$SIZES"
--concurrency "$concurrency"
--duration "$DURATION"
--rounds "$ROUNDS"
--retry-per-round "$RETRY_PER_ROUND"
--retry-sleep-secs "$RETRY_SLEEP_SECS"
--cooldown-secs "$COOLDOWN_SECS"
--out-dir "$run_dir"
)
if [[ "$INSECURE" == "true" ]]; then
cmd+=(--insecure)
fi
if [[ -n "$baseline_csv" ]]; then
cmd+=(--baseline-csv "$baseline_csv")
fi
if [[ -n "$EXTRA_ARGS" ]]; then
cmd+=(--extra-args "$EXTRA_ARGS")
fi
if [[ "$DRY_RUN" == "true" ]]; then
cmd+=(--dry-run)
fi
echo "==== concurrency=${concurrency} bucket=${bucket} ===="
printf 'Command:'
printf ' %q' "${cmd[@]}"
printf '\n'
if "${cmd[@]}"; then
status="ok"
else
status="failed"
fi
echo "${concurrency},${bucket},${run_dir},${baseline_csv:-N/A},${status}" >> "$RUN_MATRIX_CSV"
if [[ "$status" != "ok" ]]; then
echo "ERROR: concurrency=${concurrency} failed; aborting remaining matrix runs" >&2
exit 1
fi
append_aggregate_rows "$concurrency" "$bucket" "$run_dir" "${run_dir}/median_summary.csv" "${run_dir}/baseline_compare.csv"
}
main() {
parse_args "$@"
validate_args
require_cmd bash
require_cmd git
require_cmd awk
require_cmd sed
require_cmd sort
require_cmd "$WARP_BIN"
if [[ ! -x "$ENHANCED_BENCH_SCRIPT" && ! -f "$ENHANCED_BENCH_SCRIPT" ]]; then
echo "ERROR: missing dependency script: $ENHANCED_BENCH_SCRIPT" >&2
exit 1
fi
setup_output
write_manifest
write_artifact_layout
echo "Output dir: $OUT_DIR"
echo "Concurrencies: $CONCURRENCIES"
echo "Sizes: $SIZES"
echo "Duration: $DURATION"
echo "Rounds: $ROUNDS"
echo "Cooldown secs: $COOLDOWN_SECS"
local conc_count conc_index
conc_count="$(csv_to_lines "$CONCURRENCIES" | awk 'END{print NR+0}')"
conc_index=0
while IFS= read -r concurrency; do
conc_index=$(( conc_index + 1 ))
run_concurrency "$concurrency"
if (( COOLDOWN_SECS > 0 && conc_index < conc_count )); then
echo "Cooldown after concurrency=${concurrency}: ${COOLDOWN_SECS}s"
cooldown_sleep "$COOLDOWN_SECS"
fi
done < <(csv_to_lines "$CONCURRENCIES")
echo
echo "Stage-breakdown run finished."
echo "Artifacts written to: $OUT_DIR"
echo "Top-level summaries:"
echo " - $RUN_MATRIX_CSV"
echo " - $AGG_MEDIAN_CSV"
if [[ -s "$AGG_COMPARE_CSV" ]]; then
echo " - $AGG_COMPARE_CSV"
fi
}
main "$@"
scripts/run_put_large_stage_breakdown_with_capture.sh
Executable
+406
View File
@@ -0,0 +1,406 @@
#!/usr/bin/env bash
set -euo pipefail
# One-shot wrapper for rustfs/backlog#706:
# - runs the large PUT stage-breakdown benchmark matrix
# - starts a parallel artifact collector that covers the benchmark window
# - stores both outputs under the same run root
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
BENCH_SCRIPT="${PROJECT_ROOT}/scripts/run_put_large_stage_breakdown.sh"
CAPTURE_SCRIPT="${PROJECT_ROOT}/scripts/collect_put_large_stage_breakdown_artifacts.sh"
ENDPOINT=""
ACCESS_KEY=""
SECRET_KEY=""
REGION="us-east-1"
BUCKET_PREFIX="rustfs-put-large"
CONCURRENCIES="16,32,64,96,128"
SIZES="16MiB,32MiB"
DURATION="120s"
ROUNDS=3
RETRY_PER_ROUND=1
RETRY_SLEEP_SECS=2
COOLDOWN_SECS=0
OUT_DIR=""
BASELINE_ROOT=""
EXTRA_ARGS=""
INSECURE=false
DRY_RUN=false
WARP_BIN="${WARP_BIN:-warp}"
TOPOLOGY_NODES=""
TOPOLOGY_DISKS_PER_NODE=""
TOPOLOGY_TOTAL_DISKS=""
TOPOLOGY_CPU_PER_NODE=""
TOPOLOGY_MEM_PER_NODE=""
TOPOLOGY_NETWORK=""
TOPOLOGY_ENDPOINT_MODE=""
TOPOLOGY_ERASURE_SET_DRIVE_COUNT=""
CLIENT_HOST=""
WORKLOAD_LABEL="backlog-706-large-put-stage-breakdown"
CAPTURE_LABEL="benchmark-window"
CAPTURE_METRICS_ENDPOINTS=""
CAPTURE_PROM_METRICS_URLS=""
CAPTURE_DURATION_SECS=""
CAPTURE_INTERVAL_SECS=15
CAPTURE_RUSTFS_PID=""
CAPTURE_SKIP_HOST_TELEMETRY=false
SKIP_CAPTURE=false
usage() {
cat <<'USAGE'
Usage:
scripts/run_put_large_stage_breakdown_with_capture.sh --endpoint <url> \
--access-key <ak> --secret-key <sk> [options]
Benchmark options:
--endpoint <url>
--access-key <ak>
--secret-key <sk>
--bucket-prefix <prefix> Default: rustfs-put-large
--region <name> Default: us-east-1
--sizes <csv> Default: 16MiB,32MiB
--concurrencies <csv> Default: 16,32,64,96,128
--duration <dur> Default: 120s
--rounds <n> Default: 3
--retry-per-round <n> Default: 1
--retry-sleep-secs <n> Default: 2
--cooldown-secs <n> Sleep between rounds/sizes (default: 0)
--out-dir <dir> Default: target/bench/put-large-stage-breakdown-<timestamp>
--baseline-root <dir>
--extra-args "<args>"
--warp-bin <path> Default: warp
--insecure
--dry-run
Topology metadata:
--nodes <n>
--disks-per-node <n>
--total-disks <n>
--cpu-per-node <text>
--mem-per-node <text>
--network <text>
--endpoint-mode <direct|lb>
--erasure-set-drive-count <n>
--client-host <text>
--workload-label <text>
Capture options:
--capture-label <name> Default: benchmark-window
--capture-metrics-endpoints <csv> Signed admin metrics endpoints
--capture-prom-metrics-urls <csv> Optional plain Prometheus text endpoints
--capture-duration-secs <n> Override auto-derived capture window
--capture-interval-secs <n> Default: 15
--capture-rustfs-pid <pid>
--capture-skip-host-telemetry
--skip-capture Run benchmark only and do not start capture
Behavior:
- If --capture-duration-secs is omitted, the wrapper derives a nominal
capture window from the benchmark matrix and adds a safety slack.
- The collector runs in the background while the benchmark matrix is in the foreground.
USAGE
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "ERROR: command not found: $1" >&2
exit 1
fi
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s\n' "$value"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--bucket-prefix) BUCKET_PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--sizes) SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--rounds) ROUNDS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--retry-per-round) RETRY_PER_ROUND="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--retry-sleep-secs) RETRY_SLEEP_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--baseline-root) BASELINE_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--extra-args) EXTRA_ARGS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--warp-bin) WARP_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--nodes) TOPOLOGY_NODES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--disks-per-node) TOPOLOGY_DISKS_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--total-disks) TOPOLOGY_TOTAL_DISKS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cpu-per-node) TOPOLOGY_CPU_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--mem-per-node) TOPOLOGY_MEM_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--network) TOPOLOGY_NETWORK="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--endpoint-mode) TOPOLOGY_ENDPOINT_MODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--erasure-set-drive-count) TOPOLOGY_ERASURE_SET_DRIVE_COUNT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--client-host) CLIENT_HOST="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--workload-label) WORKLOAD_LABEL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-label) CAPTURE_LABEL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-metrics-endpoints) CAPTURE_METRICS_ENDPOINTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-prom-metrics-urls) CAPTURE_PROM_METRICS_URLS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-duration-secs) CAPTURE_DURATION_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-interval-secs) CAPTURE_INTERVAL_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-rustfs-pid) CAPTURE_RUSTFS_PID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-skip-host-telemetry) CAPTURE_SKIP_HOST_TELEMETRY=true; shift ;;
--skip-capture) SKIP_CAPTURE=true; shift ;;
--insecure) INSECURE=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
is_positive_int() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
is_nonnegative_int() {
[[ "$1" =~ ^[0-9]+$ ]]
}
count_csv_items() {
local csv="$1"
local count=0 raw item
IFS=',' read -r -a arr <<< "$csv"
for raw in "${arr[@]}"; do
item="$(echo "$raw" | awk '{$1=$1;print}')"
[[ -z "$item" ]] && continue
count=$((count + 1))
done
echo "$count"
}
duration_to_seconds() {
local value="$1"
if [[ "$value" =~ ^([0-9]+)s$ ]]; then
echo "${BASH_REMATCH[1]}"
elif [[ "$value" =~ ^([0-9]+)m$ ]]; then
echo $(( BASH_REMATCH[1] * 60 ))
elif [[ "$value" =~ ^([0-9]+)h$ ]]; then
echo $(( BASH_REMATCH[1] * 3600 ))
elif [[ "$value" =~ ^[0-9]+$ ]]; then
echo "$value"
else
echo "ERROR"
fi
}
validate_args() {
if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
echo "ERROR: --endpoint, --access-key, and --secret-key are required" >&2
exit 1
fi
if ! is_positive_int "$ROUNDS" || ! is_positive_int "$RETRY_PER_ROUND" || ! is_positive_int "$RETRY_SLEEP_SECS" || ! is_nonnegative_int "$COOLDOWN_SECS"; then
echo "ERROR: --rounds, --retry-per-round, and --retry-sleep-secs must be positive integers; --cooldown-secs must be a nonnegative integer" >&2
exit 1
fi
if ! is_positive_int "$CAPTURE_INTERVAL_SECS"; then
echo "ERROR: --capture-interval-secs must be a positive integer" >&2
exit 1
fi
if [[ -n "$CAPTURE_DURATION_SECS" && ! "$CAPTURE_DURATION_SECS" =~ ^[0-9]+$ ]]; then
echo "ERROR: --capture-duration-secs must be a nonnegative integer" >&2
exit 1
fi
if [[ -n "$BASELINE_ROOT" && ! -d "$BASELINE_ROOT" ]]; then
echo "ERROR: --baseline-root does not exist: $BASELINE_ROOT" >&2
exit 1
fi
local duration_secs
duration_secs="$(duration_to_seconds "$DURATION")"
if [[ "$duration_secs" == "ERROR" ]]; then
echo "ERROR: unsupported --duration format: $DURATION (expected e.g. 120s, 2m, 1h)" >&2
exit 1
fi
}
setup_output() {
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="target/bench/put-large-stage-breakdown-$(date -u +%Y%m%dT%H%M%SZ)"
fi
}
derive_capture_duration() {
if [[ -n "$CAPTURE_DURATION_SECS" ]]; then
echo "$CAPTURE_DURATION_SECS"
return
fi
local sizes_count conc_count duration_secs nominal bench_secs retry_slack cooldown_slack fixed_slack
sizes_count="$(count_csv_items "$SIZES")"
conc_count="$(count_csv_items "$CONCURRENCIES")"
duration_secs="$(duration_to_seconds "$DURATION")"
bench_secs=$(( sizes_count * conc_count * ROUNDS * duration_secs ))
retry_slack=$(( conc_count * RETRY_PER_ROUND * RETRY_SLEEP_SECS + conc_count * 30 ))
cooldown_slack=$(( (conc_count * (sizes_count * (ROUNDS - 1) + (sizes_count - 1)) + (conc_count - 1)) * COOLDOWN_SECS ))
fixed_slack=120
nominal=$(( bench_secs + retry_slack + cooldown_slack + fixed_slack ))
echo "$nominal"
}
run_capture() {
local capture_duration="$1"
local -a cmd=(
bash "$CAPTURE_SCRIPT"
--run-root "$OUT_DIR"
--endpoint "$ENDPOINT"
--label "$CAPTURE_LABEL"
--access-key "$ACCESS_KEY"
--secret-key-env RUSTFS_CAPTURE_SECRET_KEY
--region "$REGION"
--duration-secs "$capture_duration"
--interval-secs "$CAPTURE_INTERVAL_SECS"
)
if [[ -n "$CAPTURE_METRICS_ENDPOINTS" ]]; then
cmd+=(--metrics-endpoints "$CAPTURE_METRICS_ENDPOINTS")
fi
if [[ -n "$CAPTURE_PROM_METRICS_URLS" ]]; then
cmd+=(--prom-metrics-urls "$CAPTURE_PROM_METRICS_URLS")
fi
if [[ -n "$CAPTURE_RUSTFS_PID" ]]; then
cmd+=(--rustfs-pid "$CAPTURE_RUSTFS_PID")
fi
if [[ "$CAPTURE_SKIP_HOST_TELEMETRY" == "true" ]]; then
cmd+=(--skip-host-telemetry)
fi
if [[ "$DRY_RUN" == "true" ]]; then
cmd+=(--dry-run)
fi
printf 'Capture command:'
printf ' %q' "${cmd[@]}"
printf '\n'
RUSTFS_CAPTURE_SECRET_KEY="$SECRET_KEY" "${cmd[@]}" &
CAPTURE_PID=$!
}
run_benchmark() {
local -a cmd=(
bash "$BENCH_SCRIPT"
--endpoint "$ENDPOINT"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--bucket-prefix "$BUCKET_PREFIX"
--region "$REGION"
--sizes "$SIZES"
--concurrencies "$CONCURRENCIES"
--duration "$DURATION"
--rounds "$ROUNDS"
--retry-per-round "$RETRY_PER_ROUND"
--retry-sleep-secs "$RETRY_SLEEP_SECS"
--cooldown-secs "$COOLDOWN_SECS"
--out-dir "$OUT_DIR"
--warp-bin "$WARP_BIN"
--workload-label "$WORKLOAD_LABEL"
)
if [[ -n "$BASELINE_ROOT" ]]; then
cmd+=(--baseline-root "$BASELINE_ROOT")
fi
if [[ -n "$EXTRA_ARGS" ]]; then
cmd+=(--extra-args "$EXTRA_ARGS")
fi
if [[ -n "$TOPOLOGY_NODES" ]]; then
cmd+=(--nodes "$TOPOLOGY_NODES")
fi
if [[ -n "$TOPOLOGY_DISKS_PER_NODE" ]]; then
cmd+=(--disks-per-node "$TOPOLOGY_DISKS_PER_NODE")
fi
if [[ -n "$TOPOLOGY_TOTAL_DISKS" ]]; then
cmd+=(--total-disks "$TOPOLOGY_TOTAL_DISKS")
fi
if [[ -n "$TOPOLOGY_CPU_PER_NODE" ]]; then
cmd+=(--cpu-per-node "$TOPOLOGY_CPU_PER_NODE")
fi
if [[ -n "$TOPOLOGY_MEM_PER_NODE" ]]; then
cmd+=(--mem-per-node "$TOPOLOGY_MEM_PER_NODE")
fi
if [[ -n "$TOPOLOGY_NETWORK" ]]; then
cmd+=(--network "$TOPOLOGY_NETWORK")
fi
if [[ -n "$TOPOLOGY_ENDPOINT_MODE" ]]; then
cmd+=(--endpoint-mode "$TOPOLOGY_ENDPOINT_MODE")
fi
if [[ -n "$TOPOLOGY_ERASURE_SET_DRIVE_COUNT" ]]; then
cmd+=(--erasure-set-drive-count "$TOPOLOGY_ERASURE_SET_DRIVE_COUNT")
fi
if [[ -n "$CLIENT_HOST" ]]; then
cmd+=(--client-host "$CLIENT_HOST")
fi
if [[ "$INSECURE" == "true" ]]; then
cmd+=(--insecure)
fi
if [[ "$DRY_RUN" == "true" ]]; then
cmd+=(--dry-run)
fi
printf 'Benchmark command:'
printf ' %q' "${cmd[@]}"
printf '\n'
"${cmd[@]}"
}
cleanup_capture() {
if [[ -n "${CAPTURE_PID:-}" ]] && kill -0 "$CAPTURE_PID" >/dev/null 2>&1; then
kill "$CAPTURE_PID" >/dev/null 2>&1 || true
wait "$CAPTURE_PID" >/dev/null 2>&1 || true
fi
}
main() {
parse_args "$@"
validate_args
require_cmd bash
require_cmd awk
require_cmd "$WARP_BIN"
setup_output
echo "Output dir: $OUT_DIR"
if [[ "$SKIP_CAPTURE" != "true" ]]; then
local capture_duration
capture_duration="$(derive_capture_duration)"
echo "Derived capture duration seconds: $capture_duration"
echo "Capture label: $CAPTURE_LABEL"
trap cleanup_capture EXIT
run_capture "$capture_duration"
else
echo "Capture: skipped"
fi
run_benchmark
if [[ "$SKIP_CAPTURE" != "true" ]]; then
wait "$CAPTURE_PID"
CAPTURE_PID=""
fi
echo
echo "One-shot benchmark + capture run finished."
echo "Run root: $OUT_DIR"
}
main "$@"
scripts/run_put_large_tuning_matrix.sh
Executable
+430
View File
@@ -0,0 +1,430 @@
#!/usr/bin/env bash
set -euo pipefail
# rustfs/backlog#708 tuning matrix runner for large-object PUT.
# This controller:
# - defines high-priority tuning profiles for 16MiB / 32MiB PUT
# - writes an env snapshot per profile
# - optionally calls an apply hook after switching profile env
# - reuses scripts/run_put_large_stage_breakdown_with_capture.sh for each profile
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RUNNER_SCRIPT="${SCRIPT_DIR}/run_put_large_stage_breakdown_with_capture.sh"
GROUP="all" # all|baseline|inflight|io-buffer|runtime|duplex
ENDPOINT=""
ACCESS_KEY=""
SECRET_KEY=""
REGION="us-east-1"
BUCKET_PREFIX="rustfs-put-large"
CONCURRENCIES="16,32,64,96,128"
SIZES="16MiB,32MiB"
DURATION="120s"
ROUNDS=3
OUT_ROOT=""
WARP_BIN="warp"
EXTRA_ARGS=""
INSECURE=false
DRY_RUN=false
COOLDOWN_SECS=30
TOPOLOGY_NODES=""
TOPOLOGY_DISKS_PER_NODE=""
TOPOLOGY_TOTAL_DISKS=""
TOPOLOGY_CPU_PER_NODE=""
TOPOLOGY_MEM_PER_NODE=""
TOPOLOGY_NETWORK=""
TOPOLOGY_ENDPOINT_MODE=""
TOPOLOGY_ERASURE_SET_DRIVE_COUNT=""
CLIENT_HOST=""
CAPTURE_METRICS_ENDPOINTS=""
CAPTURE_PROM_METRICS_URLS=""
CAPTURE_INTERVAL_SECS=15
CAPTURE_RUSTFS_PID=""
CAPTURE_SKIP_HOST_TELEMETRY=false
SKIP_CAPTURE=false
APPLY_CMD=""
APPLY_CMD_ARR=()
APPLY_WAIT_SECS=20
BASE_OBJECT_IO_BUFFER_SIZE=262144
BASE_OBJECT_DUPLEX_BUFFER_SIZE=8388608
BASE_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
BASE_RUNTIME_WORKER_THREADS=12
BASE_RUNTIME_MAX_BLOCKING_THREADS=512
usage() {
cat <<'USAGE'
Usage:
scripts/run_put_large_tuning_matrix.sh --endpoint <url> \
--access-key <ak> --secret-key <sk> [options]
Required:
--endpoint <url>
--access-key <ak>
--secret-key <sk>
Core options:
--group <name> all|baseline|inflight|io-buffer|runtime|duplex
--bucket-prefix <prefix> Default: rustfs-put-large
--region <name> Default: us-east-1
--concurrencies <csv> Default: 16,32,64,96,128
--sizes <csv> Default: 16MiB,32MiB
--duration <dur> Default: 120s
--rounds <n> Default: 3
--out-root <dir> Default: target/bench/put-large-tuning-<timestamp>
--warp-bin <path> Default: warp
--extra-args "<args>"
--cooldown-secs <n> Sleep between profiles (default: 30)
--insecure
--dry-run
Topology metadata:
--nodes <n>
--disks-per-node <n>
--total-disks <n>
--cpu-per-node <text>
--mem-per-node <text>
--network <text>
--endpoint-mode <direct|lb>
--erasure-set-drive-count <n>
--client-host <text>
Capture options:
--capture-metrics-endpoints <csv>
--capture-prom-metrics-urls <csv>
--capture-interval-secs <n> Default: 15
--capture-rustfs-pid <pid>
--capture-skip-host-telemetry
--skip-capture Reuse benchmark runner without starting capture
Optional apply hook:
--apply-cmd "<cmd>" Optional plain command + args, no shell operators.
Useful for restarting/reloading RustFS after env changes.
--apply-wait-secs <n> Wait after apply command (default: 20)
Behavior:
- Each profile writes: <out-root>/<profile>/env_snapshot.env
- Each profile then calls scripts/run_put_large_stage_breakdown_with_capture.sh
- Non-baseline profiles automatically compare against <out-root>/baseline
USAGE
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "ERROR: command not found: $1" >&2
exit 1
fi
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s\n' "$value"
}
parse_apply_cmd() {
local raw="$1"
if [[ "$raw" == *';'* || "$raw" == *'&&'* || "$raw" == *'||'* || "$raw" == *'|'* || "$raw" == *'<'* || "$raw" == *'>'* || "$raw" == *'`'* || "$raw" == *'$'* ]]; then
echo "ERROR: --apply-cmd does not allow shell operators or expansions; pass a plain command and args only" >&2
exit 1
fi
IFS=$' \t\n' read -r -a APPLY_CMD_ARR <<< "$raw"
if [[ "${#APPLY_CMD_ARR[@]}" -eq 0 ]]; then
echo "ERROR: --apply-cmd must not be empty" >&2
exit 1
fi
require_cmd "${APPLY_CMD_ARR[0]}"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--access-key) ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--secret-key) SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--group) GROUP="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--bucket-prefix) BUCKET_PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--region) REGION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--sizes) SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--rounds) ROUNDS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-root) OUT_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--warp-bin) WARP_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--extra-args) EXTRA_ARGS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--nodes) TOPOLOGY_NODES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--disks-per-node) TOPOLOGY_DISKS_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--total-disks) TOPOLOGY_TOTAL_DISKS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cpu-per-node) TOPOLOGY_CPU_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--mem-per-node) TOPOLOGY_MEM_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--network) TOPOLOGY_NETWORK="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--endpoint-mode) TOPOLOGY_ENDPOINT_MODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--erasure-set-drive-count) TOPOLOGY_ERASURE_SET_DRIVE_COUNT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--client-host) CLIENT_HOST="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-metrics-endpoints) CAPTURE_METRICS_ENDPOINTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-prom-metrics-urls) CAPTURE_PROM_METRICS_URLS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-interval-secs) CAPTURE_INTERVAL_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-rustfs-pid) CAPTURE_RUSTFS_PID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--capture-skip-host-telemetry) CAPTURE_SKIP_HOST_TELEMETRY=true; shift ;;
--skip-capture) SKIP_CAPTURE=true; shift ;;
--apply-cmd) APPLY_CMD="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--apply-wait-secs) APPLY_WAIT_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--insecure) INSECURE=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
is_positive_int() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
validate_args() {
if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
echo "ERROR: --endpoint, --access-key, and --secret-key are required" >&2
exit 1
fi
case "$GROUP" in
all|baseline|inflight|io-buffer|runtime|duplex) ;;
*) echo "ERROR: --group must be all|baseline|inflight|io-buffer|runtime|duplex" >&2; exit 1 ;;
esac
if ! is_positive_int "$ROUNDS" || ! is_positive_int "$CAPTURE_INTERVAL_SECS" || ! is_positive_int "$APPLY_WAIT_SECS" || ! [[ "$COOLDOWN_SECS" =~ ^[0-9]+$ ]]; then
echo "ERROR: --rounds, --capture-interval-secs, and --apply-wait-secs must be positive integers; --cooldown-secs must be a nonnegative integer" >&2
exit 1
fi
if [[ -n "$APPLY_CMD" ]]; then
parse_apply_cmd "$APPLY_CMD"
fi
}
setup_out_root() {
if [[ -z "$OUT_ROOT" ]]; then
OUT_ROOT="target/bench/put-large-tuning-$(date -u +%Y%m%dT%H%M%SZ)"
fi
mkdir -p "$OUT_ROOT"
}
apply_profile() {
local profile="$1"
export RUSTFS_OBJECT_IO_BUFFER_SIZE="$BASE_OBJECT_IO_BUFFER_SIZE"
export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE="$BASE_OBJECT_DUPLEX_BUFFER_SIZE"
export RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES="$BASE_ERASURE_ENCODE_MAX_INFLIGHT_BYTES"
export RUSTFS_RUNTIME_WORKER_THREADS="$BASE_RUNTIME_WORKER_THREADS"
export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS="$BASE_RUNTIME_MAX_BLOCKING_THREADS"
case "$profile" in
baseline) ;;
inflight-32m) export RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432 ;;
inflight-48m) export RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648 ;;
inflight-64m) export RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=67108864 ;;
io-buffer-512k) export RUSTFS_OBJECT_IO_BUFFER_SIZE=524288 ;;
io-buffer-1m) export RUSTFS_OBJECT_IO_BUFFER_SIZE=1048576 ;;
runtime-w16-b768)
export RUSTFS_RUNTIME_WORKER_THREADS=16
export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768
;;
runtime-w20-b1024)
export RUSTFS_RUNTIME_WORKER_THREADS=20
export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
;;
duplex-16m) export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216 ;;
*)
echo "ERROR: unsupported profile $profile" >&2
exit 1
;;
esac
}
write_env_snapshot() {
local out_file="$1"
cat > "$out_file" <<EOF
RUSTFS_OBJECT_IO_BUFFER_SIZE=${RUSTFS_OBJECT_IO_BUFFER_SIZE}
RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=${RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE}
RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=${RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES}
RUSTFS_RUNTIME_WORKER_THREADS=${RUSTFS_RUNTIME_WORKER_THREADS}
RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=${RUSTFS_RUNTIME_MAX_BLOCKING_THREADS}
EOF
}
run_apply_hook_if_needed() {
local profile="$1"
local env_file="$2"
if [[ "${#APPLY_CMD_ARR[@]}" -eq 0 ]]; then
return
fi
echo "[${profile}] running apply command..."
if [[ "$DRY_RUN" == "true" ]]; then
printf '[DRY-RUN] RUSTFS_TUNING_ENV_FILE=%q ' "$env_file"
printf '%q ' "${APPLY_CMD_ARR[@]}"
printf '\n'
echo "[DRY-RUN] sleep $APPLY_WAIT_SECS"
else
RUSTFS_TUNING_ENV_FILE="$env_file" "${APPLY_CMD_ARR[@]}"
echo "[${profile}] waiting ${APPLY_WAIT_SECS}s for service readiness..."
sleep "$APPLY_WAIT_SECS"
fi
}
profiles_for_group() {
case "$1" in
baseline) echo "baseline" ;;
inflight) echo "baseline inflight-32m inflight-48m inflight-64m" ;;
io-buffer) echo "baseline io-buffer-512k io-buffer-1m" ;;
runtime) echo "baseline runtime-w16-b768 runtime-w20-b1024" ;;
duplex) echo "baseline duplex-16m" ;;
all) echo "baseline inflight-32m inflight-48m inflight-64m io-buffer-512k io-buffer-1m runtime-w16-b768 runtime-w20-b1024 duplex-16m" ;;
esac
}
run_profile() {
local profile="$1"
local out_dir env_file baseline_root
out_dir="${OUT_ROOT}/${profile}"
mkdir -p "$out_dir"
apply_profile "$profile"
env_file="${out_dir}/env_snapshot.env"
write_env_snapshot "$env_file"
run_apply_hook_if_needed "$profile" "$env_file"
baseline_root=""
if [[ "$profile" != "baseline" ]]; then
baseline_root="${OUT_ROOT}/baseline"
fi
local -a cmd=(
bash "$RUNNER_SCRIPT"
--endpoint "$ENDPOINT"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--bucket-prefix "${BUCKET_PREFIX}-${profile}"
--region "$REGION"
--concurrencies "$CONCURRENCIES"
--sizes "$SIZES"
--duration "$DURATION"
--rounds "$ROUNDS"
--out-dir "$out_dir"
--warp-bin "$WARP_BIN"
)
if [[ -n "$baseline_root" ]]; then
cmd+=(--baseline-root "$baseline_root")
fi
if [[ -n "$EXTRA_ARGS" ]]; then
cmd+=(--extra-args "$EXTRA_ARGS")
fi
if [[ -n "$TOPOLOGY_NODES" ]]; then
cmd+=(--nodes "$TOPOLOGY_NODES")
fi
if [[ -n "$TOPOLOGY_DISKS_PER_NODE" ]]; then
cmd+=(--disks-per-node "$TOPOLOGY_DISKS_PER_NODE")
fi
if [[ -n "$TOPOLOGY_TOTAL_DISKS" ]]; then
cmd+=(--total-disks "$TOPOLOGY_TOTAL_DISKS")
fi
if [[ -n "$TOPOLOGY_CPU_PER_NODE" ]]; then
cmd+=(--cpu-per-node "$TOPOLOGY_CPU_PER_NODE")
fi
if [[ -n "$TOPOLOGY_MEM_PER_NODE" ]]; then
cmd+=(--mem-per-node "$TOPOLOGY_MEM_PER_NODE")
fi
if [[ -n "$TOPOLOGY_NETWORK" ]]; then
cmd+=(--network "$TOPOLOGY_NETWORK")
fi
if [[ -n "$TOPOLOGY_ENDPOINT_MODE" ]]; then
cmd+=(--endpoint-mode "$TOPOLOGY_ENDPOINT_MODE")
fi
if [[ -n "$TOPOLOGY_ERASURE_SET_DRIVE_COUNT" ]]; then
cmd+=(--erasure-set-drive-count "$TOPOLOGY_ERASURE_SET_DRIVE_COUNT")
fi
if [[ -n "$CLIENT_HOST" ]]; then
cmd+=(--client-host "$CLIENT_HOST")
fi
if [[ -n "$CAPTURE_METRICS_ENDPOINTS" ]]; then
cmd+=(--capture-metrics-endpoints "$CAPTURE_METRICS_ENDPOINTS")
fi
if [[ -n "$CAPTURE_PROM_METRICS_URLS" ]]; then
cmd+=(--capture-prom-metrics-urls "$CAPTURE_PROM_METRICS_URLS")
fi
if [[ -n "$CAPTURE_RUSTFS_PID" ]]; then
cmd+=(--capture-rustfs-pid "$CAPTURE_RUSTFS_PID")
fi
if [[ "$CAPTURE_SKIP_HOST_TELEMETRY" == "true" ]]; then
cmd+=(--capture-skip-host-telemetry)
fi
if [[ "$SKIP_CAPTURE" == "true" ]]; then
cmd+=(--skip-capture)
fi
if [[ "$INSECURE" == "true" ]]; then
cmd+=(--insecure)
fi
if [[ "$DRY_RUN" == "true" ]]; then
cmd+=(--dry-run)
fi
echo
echo "===== Running profile ${profile} ====="
echo "Output: $out_dir"
echo "Env snapshot: $env_file"
if [[ "$DRY_RUN" == "true" ]]; then
printf '[DRY-RUN] '
printf '%q ' "${cmd[@]}"
printf '\n'
else
"${cmd[@]}"
fi
}
main() {
parse_args "$@"
validate_args
require_cmd bash
require_cmd awk
if [[ ! -x "$RUNNER_SCRIPT" && ! -f "$RUNNER_SCRIPT" ]]; then
echo "ERROR: missing dependency script: $RUNNER_SCRIPT" >&2
exit 1
fi
setup_out_root
echo "Tuning output root: $OUT_ROOT"
echo "Group: $GROUP"
echo "Concurrencies: $CONCURRENCIES"
echo "Sizes: $SIZES"
echo "Cooldown between profiles: ${COOLDOWN_SECS}s"
local profiles profile index total_profiles
profiles=($(profiles_for_group "$GROUP"))
total_profiles="${#profiles[@]}"
for ((index = 0; index < total_profiles; index++)); do
profile="${profiles[$index]}"
run_profile "$profile"
if (( index + 1 < total_profiles )) && (( COOLDOWN_SECS > 0 )); then
echo
echo "[${profile}] cooldown ${COOLDOWN_SECS}s before next profile..."
if [[ "$DRY_RUN" == "true" ]]; then
echo "[DRY-RUN] sleep ${COOLDOWN_SECS}"
else
sleep "$COOLDOWN_SECS"
fi
fi
done
echo
echo "Done. Profile outputs are under: $OUT_ROOT"
}
main "$@"