gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
rustfs/scripts/run_object_batch_bench_enhanced.sh
T
houseme 14f31b797a feat(bench): capture node attribution metadata (#5158) 
Co-authored-by: heihutu <heihutu@gmail.com>
2026-07-23 15:56:48 +00:00

1364 lines
48 KiB
Bash
Executable File
Raw Permalink Blame History

#!/usr/bin/env bash
set -euo pipefail
# Enhanced batch object benchmark runner for warp/s3bench:
# - Multi-round execution (default 3 rounds)
# - Retry failed round attempts automatically
# - Median aggregation per object size
# - Optional baseline CSV comparison
DEFAULT_SIZES="1KiB,4KiB,8KiB,16KiB,32KiB,100KiB,512KiB,1MiB,2MiB,5MiB,10MiB"
TOOL="warp"
ENDPOINT=""
ACCESS_KEY=""
SECRET_KEY=""
BUCKET="rustfs-bench"
BUCKET_SIZE_SUFFIX=false
REGION="us-east-1"
CONCURRENCY=128
SIZES="$DEFAULT_SIZES"
OUT_DIR=""
INSECURE=false
DRY_RUN=false
PYTHON_BIN="python3"
# warp options
WARP_BIN="warp"
WARP_MODE="mixed"
DURATION="60s"
# s3bench options
S3BENCH_BIN="s3bench"
SAMPLES=20000
# enhancement options
ROUNDS=3
RETRY_PER_ROUND=2
RETRY_SLEEP_SECS=2
COOLDOWN_SECS=20
BASELINE_CSV=""
EXTRA_ARGS=()
FAILED_FINAL_ROUNDS=0
SERVICE_METRICS_URL=""
SERVICE_METRICS_DIR=""
SERVICE_METRICS_CAPTURE_ATTEMPTS=3
SERVICE_METRICS_CAPTURE_RETRY_SECS=1
SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2
SERVICE_METRICS_MAX_TIME_SECS=15
SERVICE_METRICS_SETTLE_SECS=0
SERVICE_METRICS_FILTER_REGEX=""
SERVICE_METRICS_FILTER_REGEX_EXPLICIT=false
SERVICE_PROMETHEUS_QUERY_URL=""
SERVICE_PROMETHEUS_QUERY=""
SERVICE_PROMETHEUS_QUERY_EXPLICIT=false
SERVICE_METRICS_SERVICE_NAME=""
SERVER_IMAGE_REF=""
SERVER_IMAGE_DIGEST=""
SERVER_REVISION=""
REQUIRE_SERVER_PROVENANCE=false
RUN_LABELS=()
NODE_METRICS_URLS=()
NODE_DOCKER_CONTAINERS=()
usage() {
cat <<'USAGE'
Usage:
scripts/run_object_batch_bench_enhanced.sh --tool <warp|s3bench> --endpoint <host:port|url> \
--access-key <ak> --secret-key <sk> [options]
Required:
--tool warp | s3bench
--endpoint S3 endpoint
--access-key S3 access key
--secret-key S3 secret key
Core options:
--bucket Bucket name (default: rustfs-bench)
--bucket-size-suffix Append a sanitized object size suffix to the
bucket for each size
--region Region (default: us-east-1)
--concurrency Concurrency for all sizes (default: 128)
--sizes Comma-separated sizes (default: 1KiB..10MiB matrix)
--out-dir Output directory (default: target/bench/object-batch-enhanced-<timestamp>)
--insecure Allow insecure TLS
--dry-run Print commands only, do not execute
--python-bin Python binary for Prometheus query capture (default: python3)
Warp options:
--warp-bin warp binary (default: warp)
--warp-mode get|put|mixed (default: mixed)
--duration e.g. 60s/2m (default: 60s)
s3bench options:
--s3bench-bin s3bench binary (default: s3bench)
--samples numSamples (default: 20000)
Enhanced options:
--rounds Benchmark rounds per size (default: 3)
--retry-per-round Retry count per failed round (default: 2)
--retry-sleep-secs Sleep seconds between retries (default: 2)
--cooldown-secs Sleep seconds between rounds/sizes (default: 20)
--round-cooldown-secs Compatibility alias for --cooldown-secs
--baseline-csv Baseline median CSV to compare
--extra-args Extra args appended to tool command, quoted as one string
--service-metrics-url Optional Prometheus scrape URL captured before/after each round attempt
--service-prometheus-query-url
Optional Prometheus HTTP API /api/v1/query URL for OTLP-exported metrics
--service-prometheus-query PromQL used with --service-prometheus-query-url
(default: auto by tool/mode)
--service-metrics-service-name
Optional service.name/service_name label filter for Prometheus query results
--service-metrics-dir Output directory for per-round service metric snapshots
--service-metrics-attempts Capture attempts for each snapshot (default: 3)
--service-metrics-retry-secs Sleep seconds between failed capture attempts (default: 1)
--service-metrics-connect-timeout-secs
Curl connect timeout for each metrics capture (default: 2)
--service-metrics-max-time-secs
Curl max time for each metrics capture (default: 15)
--service-metrics-settle-secs
Sleep seconds before after-snapshot capture (default: 0)
--service-metrics-filter-regex
Regex for retained metrics lines after scrape
(default: auto by tool/mode)
--server-image-ref Image reference of the benchmarked server
--server-image-digest Immutable image digest of the benchmarked server
--server-revision Source revision built into the benchmarked server
--require-server-provenance Fail unless the three server identity fields are set
--label <key=value> Repeatable run label written to run_manifest.env
--node-metrics-url <node=url>
Repeatable node-specific Prometheus scrape URL captured
before/after each round attempt
--node-docker-container <node=container>
Repeatable Docker container mapped to a node; captures
CPU, memory, network, and block I/O before/after each round
Output files:
round_results.csv One row per round attempt (with retry trace)
median_summary.csv Median metrics per object size
baseline_compare.csv Delta vs baseline (if --baseline-csv is set)
<service-metrics-dir>/*.prom Optional per-round service metric snapshots
node_inventory.csv Per-node container and immutable image identity
node_metrics_captures.csv Per-node metric snapshot inventory
node_resource_captures.csv Per-node Docker resource and block I/O snapshots
Example:
scripts/run_object_batch_bench_enhanced.sh \
--tool warp --endpoint http://127.0.0.1:9000 \
--access-key minioadmin --secret-key minioadmin \
--bucket bench-obj --concurrency 128 --duration 90s \
--rounds 3 --retry-per-round 2 --baseline-csv old/median_summary.csv
USAGE
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
echo "ERROR: command not found: $1" >&2
exit 1
fi
}
normalize_warp_host() {
local raw="$1"
raw="${raw#http://}"
raw="${raw#https://}"
raw="${raw%%/*}"
raw="${raw%%\?*}"
raw="${raw%%\#*}"
echo "$raw"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--tool) TOOL="$2"; shift 2 ;;
--endpoint) ENDPOINT="$2"; shift 2 ;;
--access-key) ACCESS_KEY="$2"; shift 2 ;;
--secret-key) SECRET_KEY="$2"; shift 2 ;;
--bucket) BUCKET="$2"; shift 2 ;;
--bucket-size-suffix) BUCKET_SIZE_SUFFIX=true; shift ;;
--region) REGION="$2"; shift 2 ;;
--concurrency) CONCURRENCY="$2"; shift 2 ;;
--sizes) SIZES="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--insecure) INSECURE=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
--python-bin) PYTHON_BIN="$2"; shift 2 ;;
--warp-bin) WARP_BIN="$2"; shift 2 ;;
--warp-mode) WARP_MODE="$2"; shift 2 ;;
--duration) DURATION="$2"; shift 2 ;;
--s3bench-bin) S3BENCH_BIN="$2"; shift 2 ;;
--samples) SAMPLES="$2"; shift 2 ;;
--rounds) ROUNDS="$2"; shift 2 ;;
--retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;;
--retry-sleep-secs) RETRY_SLEEP_SECS="$2"; shift 2 ;;
--cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
--round-cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
--baseline-csv) BASELINE_CSV="$2"; shift 2 ;;
--service-metrics-url) SERVICE_METRICS_URL="$2"; shift 2 ;;
--service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$2"; shift 2 ;;
--service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$2"; SERVICE_PROMETHEUS_QUERY_EXPLICIT=true; shift 2 ;;
--service-metrics-service-name) SERVICE_METRICS_SERVICE_NAME="$2"; shift 2 ;;
--service-metrics-dir) SERVICE_METRICS_DIR="$2"; shift 2 ;;
--service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;;
--service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;;
--service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;;
--service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$2"; shift 2 ;;
--service-metrics-settle-secs) SERVICE_METRICS_SETTLE_SECS="$2"; shift 2 ;;
--service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; SERVICE_METRICS_FILTER_REGEX_EXPLICIT=true; shift 2 ;;
--server-image-ref) SERVER_IMAGE_REF="$2"; shift 2 ;;
--server-image-digest) SERVER_IMAGE_DIGEST="$2"; shift 2 ;;
--server-revision) SERVER_REVISION="$2"; shift 2 ;;
--require-server-provenance) REQUIRE_SERVER_PROVENANCE=true; shift ;;
--label) RUN_LABELS+=("$2"); shift 2 ;;
--node-metrics-url) NODE_METRICS_URLS+=("$2"); shift 2 ;;
--node-docker-container) NODE_DOCKER_CONTAINERS+=("$2"); shift 2 ;;
--extra-args)
# shellcheck disable=SC2206
EXTRA_ARGS=($2)
shift 2
;;
-h|--help) usage; exit 0 ;;
*)
echo "ERROR: unknown arg: $1" >&2
usage
exit 1
;;
esac
done
}
validate_positive_int() {
local v="$1"
local n="$2"
if ! [[ "$v" =~ ^[0-9]+$ ]] || [[ "$v" -le 0 ]]; then
echo "ERROR: $n must be a positive integer, got: $v" >&2
exit 1
fi
}
validate_nonnegative_int() {
local v="$1"
local n="$2"
if ! [[ "$v" =~ ^[0-9]+$ ]]; then
echo "ERROR: $n must be a nonnegative integer, got: $v" >&2
exit 1
fi
}
validate_named_value() {
local value="$1"
local option_name="$2"
if ! [[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*=.+$ ]] || [[ "$value" == *$'\n'* || "$value" == *,* ]]; then
echo "ERROR: $option_name must be name=value without commas or newlines, got: $value" >&2
exit 1
fi
}
validate_named_values() {
local option_name="$1"
shift
local entry name seen_names=""
for entry in "$@"; do
validate_named_value "$entry" "$option_name"
name="${entry%%=*}"
if [[ " $seen_names " == *" $name "* ]]; then
echo "ERROR: $option_name repeats name: $name" >&2
exit 1
fi
seen_names="${seen_names} ${name}"
done
}
validate_manifest_labels() {
local entry key
if ((${#RUN_LABELS[@]} == 0)); then
return
fi
for entry in "${RUN_LABELS[@]}"; do
key="${entry%%=*}"
if ! [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
echo "ERROR: --label name must be a shell-compatible manifest key, got: $key" >&2
exit 1
fi
done
}
validate_manifest_value() {
local value="$1"
local option_name="$2"
if [[ "$value" == *$'\n'* || "$value" == *,* ]]; then
echo "ERROR: $option_name must not contain commas or newlines" >&2
exit 1
fi
}
cooldown_sleep() {
local secs="$1"
if (( secs <= 0 )); then
return 0
fi
if [[ "$DRY_RUN" == "true" ]]; then
echo "[DRY-RUN] sleep ${secs}"
else
sleep "$secs"
fi
}
default_service_metrics_filter_regex() {
if [[ "$TOOL" == "warp" ]]; then
case "$WARP_MODE" in
get)
echo "rustfs_io_get_object_|rustfs_s3_get_object_|rustfs_zero_copy_read|rustfs_mmap_|rustfs_get_object_"
return
;;
put)
echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_write|rustfs_buffer_|rustfs_ec_|rustfs_io_bytespool_"
return
;;
mixed)
echo "rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_|rustfs_buffer_|rustfs_ec_"
return
;;
esac
fi
echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_s3_get_object_|rustfs_io_get_object_"
}
default_service_prometheus_query() {
if [[ "$TOOL" == "warp" ]]; then
case "$WARP_MODE" in
get)
echo '{__name__=~"rustfs_(io_get_object|s3_get_object|zero_copy_read|mmap|get_object)_.*"}'
return
;;
put)
echo '{__name__=~"rustfs_(s3_put_object|io_put_object|zero_copy_write|buffer|ec|io_bytespool)_.*"}'
return
;;
mixed)
echo '{__name__=~"rustfs_(s3_get_object|io_get_object|s3_put_object|io_put_object|zero_copy|buffer|ec)_.*"}'
return
;;
esac
fi
echo '{__name__=~"rustfs_(s3_put_object|io_put_object|s3_get_object|io_get_object)_.*"}'
}
validate_args() {
if [[ "$TOOL" != "warp" && "$TOOL" != "s3bench" ]]; then
echo "ERROR: --tool must be warp or s3bench" >&2
exit 1
fi
if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
echo "ERROR: --endpoint/--access-key/--secret-key are required" >&2
exit 1
fi
validate_positive_int "$CONCURRENCY" "--concurrency"
validate_positive_int "$ROUNDS" "--rounds"
validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round"
validate_positive_int "$RETRY_SLEEP_SECS" "--retry-sleep-secs"
validate_nonnegative_int "$COOLDOWN_SECS" "--cooldown-secs"
validate_positive_int "$SERVICE_METRICS_CAPTURE_ATTEMPTS" "--service-metrics-attempts"
validate_nonnegative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" "--service-metrics-retry-secs"
validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs"
validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs"
validate_nonnegative_int "$SERVICE_METRICS_SETTLE_SECS" "--service-metrics-settle-secs"
if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2
exit 1
fi
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" || ${#NODE_METRICS_URLS[@]} -gt 0 ]] && [[ -z "$SERVICE_METRICS_DIR" ]]; then
echo "ERROR: --service-metrics-dir is required when service metrics capture is enabled" >&2
exit 1
fi
if [[ -n "$SERVICE_METRICS_URL" || ${#NODE_METRICS_URLS[@]} -gt 0 ]]; then
require_cmd curl
fi
if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
require_cmd "$PYTHON_BIN"
fi
if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)) && [[ "$DRY_RUN" != "true" ]]; then
require_cmd docker
fi
if [[ "$TOOL" == "s3bench" ]]; then
validate_positive_int "$SAMPLES" "--samples"
fi
if [[ -n "$BASELINE_CSV" && ! -f "$BASELINE_CSV" ]]; then
echo "ERROR: --baseline-csv does not exist: $BASELINE_CSV" >&2
exit 1
fi
if ((${#RUN_LABELS[@]} > 0)); then
validate_named_values "--label" "${RUN_LABELS[@]}"
fi
validate_manifest_labels
if ((${#NODE_METRICS_URLS[@]} > 0)); then
validate_named_values "--node-metrics-url" "${NODE_METRICS_URLS[@]}"
fi
if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then
validate_named_values "--node-docker-container" "${NODE_DOCKER_CONTAINERS[@]}"
fi
validate_manifest_value "$SERVER_IMAGE_REF" "--server-image-ref"
validate_manifest_value "$SERVER_IMAGE_DIGEST" "--server-image-digest"
validate_manifest_value "$SERVER_REVISION" "--server-revision"
if [[ "$REQUIRE_SERVER_PROVENANCE" == "true" ]] && { [[ -z "$SERVER_IMAGE_REF" ]] || [[ -z "$SERVER_IMAGE_DIGEST" ]] || [[ -z "$SERVER_REVISION" ]]; }; then
echo "ERROR: --require-server-provenance needs --server-image-ref, --server-image-digest, and --server-revision" >&2
exit 1
fi
if [[ "$TOOL" == "warp" ]]; then
if [[ "$WARP_MODE" != "get" && "$WARP_MODE" != "put" && "$WARP_MODE" != "mixed" ]]; then
echo "ERROR: --warp-mode must be get, put, or mixed" >&2
exit 1
fi
local warp_host
warp_host="$(normalize_warp_host "$ENDPOINT")"
if [[ -z "$warp_host" ]]; then
echo "ERROR: invalid --endpoint for warp: $ENDPOINT" >&2
exit 1
fi
fi
if [[ "$SERVICE_METRICS_FILTER_REGEX_EXPLICIT" != "true" && -z "$SERVICE_METRICS_FILTER_REGEX" ]]; then
SERVICE_METRICS_FILTER_REGEX="$(default_service_metrics_filter_regex)"
fi
if [[ "$SERVICE_PROMETHEUS_QUERY_EXPLICIT" != "true" && -z "$SERVICE_PROMETHEUS_QUERY" ]]; then
SERVICE_PROMETHEUS_QUERY="$(default_service_prometheus_query)"
fi
}
git_value() {
local default_value="$1"
shift
git "$@" 2>/dev/null || echo "$default_value"
}
git_dirty_state() {
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "unknown"
return
fi
if ! git diff --quiet --ignore-submodules --; then
echo "dirty"
return
fi
if ! git diff --cached --quiet --ignore-submodules --; then
echo "dirty"
return
fi
echo "clean"
}
write_manifest_entry() {
printf '%s=%q\n' "$1" "$2"
}
write_run_manifest() {
local manifest_file="$OUT_DIR/run_manifest.env"
local started_at_utc git_commit git_branch git_dirty rustc_version uname_s service_metrics_csv
started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
git_commit="$(git_value unknown rev-parse --short HEAD)"
git_branch="$(git_value unknown branch --show-current)"
git_dirty="$(git_dirty_state)"
rustc_version="$(rustc --version 2>/dev/null || echo unknown)"
uname_s="$(uname -a 2>/dev/null || echo unknown)"
service_metrics_csv="${SERVICE_METRICS_CSV:-}"
cat >"$manifest_file" <<EOF
started_at_utc=${started_at_utc}
manifest_schema_version=2
git_commit=${git_commit}
git_branch=${git_branch}
git_dirty=${git_dirty}
runner_git_commit=${git_commit}
runner_git_branch=${git_branch}
runner_git_dirty=${git_dirty}
rustc_version=${rustc_version}
uname=${uname_s}
tool=${TOOL}
endpoint=${ENDPOINT}
access_key=REDACTED
secret_key=REDACTED
bucket=${BUCKET}
bucket_size_suffix=${BUCKET_SIZE_SUFFIX}
region=${REGION}
concurrency=${CONCURRENCY}
sizes=${SIZES}
rounds=${ROUNDS}
retry_per_round=${RETRY_PER_ROUND}
retry_sleep_secs=${RETRY_SLEEP_SECS}
cooldown_secs=${COOLDOWN_SECS}
warp_bin=${WARP_BIN}
warp_mode=${WARP_MODE}
duration=${DURATION}
s3bench_bin=${S3BENCH_BIN}
samples=${SAMPLES}
baseline_csv=${BASELINE_CSV}
service_metrics_url=${SERVICE_METRICS_URL}
service_prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL}
service_prometheus_query=${SERVICE_PROMETHEUS_QUERY}
service_prometheus_query_explicit=${SERVICE_PROMETHEUS_QUERY_EXPLICIT}
service_metrics_service_name=${SERVICE_METRICS_SERVICE_NAME}
service_metrics_dir=${SERVICE_METRICS_DIR}
service_metrics_csv=${service_metrics_csv}
service_metrics_filter_regex=${SERVICE_METRICS_FILTER_REGEX}
service_metrics_filter_regex_explicit=${SERVICE_METRICS_FILTER_REGEX_EXPLICIT}
service_metrics_capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}
service_metrics_retry_secs=${SERVICE_METRICS_CAPTURE_RETRY_SECS}
service_metrics_connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}
service_metrics_max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
service_metrics_settle_secs=${SERVICE_METRICS_SETTLE_SECS}
dry_run=${DRY_RUN}
EOF
{
write_manifest_entry server_image_ref "${SERVER_IMAGE_REF:-unknown}"
write_manifest_entry server_image_digest "${SERVER_IMAGE_DIGEST:-unknown}"
write_manifest_entry server_revision "${SERVER_REVISION:-unknown}"
echo "require_server_provenance=${REQUIRE_SERVER_PROVENANCE}"
echo "run_label_count=${#RUN_LABELS[@]}"
if ((${#RUN_LABELS[@]} > 0)); then
local label
for label in "${RUN_LABELS[@]}"; do
write_manifest_entry "run_label_${label%%=*}" "${label#*=}"
done
fi
echo "node_metrics_count=${#NODE_METRICS_URLS[@]}"
echo "node_docker_container_count=${#NODE_DOCKER_CONTAINERS[@]}"
} >>"$manifest_file"
}
write_node_inventory() {
local inventory_csv="$OUT_DIR/node_inventory.csv"
local entry node container container_id image_id image_ref image_digest status
if ((${#NODE_DOCKER_CONTAINERS[@]} == 0)); then
return
fi
echo "node,container,status,container_id,image_id,image_ref,image_digest" > "$inventory_csv"
for entry in "${NODE_DOCKER_CONTAINERS[@]}"; do
node="${entry%%=*}"
container="${entry#*=}"
if [[ "$DRY_RUN" == "true" ]]; then
echo "$node,$container,not_run_dry_run,N/A,N/A,N/A,N/A" >> "$inventory_csv"
continue
fi
if ! container_id="$(docker inspect --format '{{.Id}}' "$container" 2>/dev/null)" || ! image_id="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null)" || ! image_ref="$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null)"; then
echo "$node,$container,capture_failed,N/A,N/A,N/A,N/A" >> "$inventory_csv"
echo "WARN: failed to inspect Docker node=${node} container=${container}" >&2
continue
fi
image_digest="$(docker image inspect --format '{{join .RepoDigests ";"}}' "$image_id" 2>/dev/null || echo N/A)"
status=ok
echo "$node,$container,$status,$container_id,$image_id,$image_ref,${image_digest:-N/A}" >> "$inventory_csv"
done
}
setup_output() {
if [[ -z "$OUT_DIR" ]]; then
OUT_DIR="target/bench/object-batch-enhanced-$(date +%Y%m%d-%H%M%S)"
fi
mkdir -p "$OUT_DIR/logs"
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" || ${#NODE_METRICS_URLS[@]} -gt 0 ]]; then
mkdir -p "$SERVICE_METRICS_DIR"
fi
if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then
mkdir -p "$OUT_DIR/node_resources"
fi
ROUND_CSV="$OUT_DIR/round_results.csv"
MEDIAN_CSV="$OUT_DIR/median_summary.csv"
COMPARE_CSV="$OUT_DIR/baseline_compare.csv"
SERVICE_METRICS_CSV="$OUT_DIR/service_metrics_captures.csv"
NODE_METRICS_CSV="$OUT_DIR/node_metrics_captures.csv"
NODE_RESOURCE_CSV="$OUT_DIR/node_resource_captures.csv"
echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file,req_p90_human,req_p90_ms,req_p99_human,req_p99_ms" > "$ROUND_CSV"
echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,median_req_p90_ms,median_req_p99_ms" > "$MEDIAN_CSV"
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "size,tool,round,attempt,phase,source,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex,prometheus_query" > "$SERVICE_METRICS_CSV"
fi
if ((${#NODE_METRICS_URLS[@]} > 0)); then
echo "size,tool,round,attempt,phase,node,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex" > "$NODE_METRICS_CSV"
fi
if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then
echo "size,tool,round,attempt,phase,node,container,status,snapshot_file" > "$NODE_RESOURCE_CSV"
fi
write_run_manifest
write_node_inventory
}
trim() {
echo "$1" | awk '{$1=$1;print}'
}
sanitize_bucket_suffix() {
printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-'
}
bucket_for_size() {
local size="$1"
if [[ "$BUCKET_SIZE_SUFFIX" != "true" ]]; then
echo "$BUCKET"
return
fi
echo "${BUCKET}-$(sanitize_bucket_suffix "$size")"
}
to_bps() {
local human="$1"
local number unit factor
if [[ "$human" == "N/A" || -z "$human" ]]; then
echo "N/A"
return
fi
# Normalize "123MiB/s" → "123 MiB/s" when no space separator exists.
human="$(echo "$human" | sed -E 's/^([0-9]+(\.[0-9]+)?)([A-Za-zµ])/\1 \3/')"
number="$(echo "$human" | awk '{print $1}')"
unit="$(echo "$human" | awk '{print $2}')"
case "$unit" in
GiB/s) factor=1073741824 ;;
MiB/s) factor=1048576 ;;
KiB/s) factor=1024 ;;
GB/s) factor=1000000000 ;;
MB/s) factor=1000000 ;;
KB/s) factor=1000 ;;
B/s) factor=1 ;;
*)
echo "N/A"
return
;;
esac
awk -v n="$number" -v f="$factor" 'BEGIN { printf "%.6f\n", n * f }'
}
to_ms() {
local human="$1"
local number unit factor
if [[ "$human" == "N/A" || -z "$human" ]]; then
echo "N/A"
return
fi
# Normalize "50ms" → "50 ms" when no space separator exists.
human="$(echo "$human" | sed -E 's/^([0-9]+(\.[0-9]+)?)([A-Za-zµ])/\1 \3/')"
number="$(echo "$human" | awk '{print $1}')"
unit="$(echo "$human" | awk '{print $2}')"
case "$unit" in
s) factor=1000 ;;
ms) factor=1 ;;
us|µs) factor=0.001 ;;
*)
echo "N/A"
return
;;
esac
awk -v n="$number" -v f="$factor" 'BEGIN { printf "%.6f\n", n * f }'
}
extract_first() {
local regex="$1"
local file="$2"
rg -o "$regex" "$file" | head -n1 || true
}
normalize_duration_metric() {
local value="$1"
value="$(trim "${value:-N/A}")"
if [[ -z "$value" || "$value" == "N/A" ]]; then
echo "N/A"
return
fi
if [[ "$value" =~ ^([0-9]+(\.[0-9]+)?)(ms|us|µs|s)$ ]]; then
echo "${BASH_REMATCH[1]} ${BASH_REMATCH[3]}"
return
fi
echo "$value" | awk '{ if ($1 == "" || $1 == "N/A") print "N/A"; else print $1" "$2 }'
}
extract_metrics() {
local log_file="$1"
local throughput reqps latency req_p90 req_p99
throughput="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")"
reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")"
latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')"
req_p90="$(rg -o '90%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^90%:[[:space:]]+//')"
req_p99="$(rg -o '99%:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^99%:[[:space:]]+//')"
if [[ -z "$latency" ]]; then
latency="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(ms|us|µs|s)' "$log_file")"
fi
throughput="$(trim "${throughput:-N/A}")"
reqps="$(trim "${reqps:-N/A}")"
latency="$(trim "${latency:-N/A}")"
req_p90="$(trim "${req_p90:-N/A}")"
req_p99="$(trim "${req_p99:-N/A}")"
# Normalize to "<num> <unit>" shape for downstream conversion helpers.
latency="$(normalize_duration_metric "$latency")"
req_p90="$(normalize_duration_metric "$req_p90")"
req_p99="$(normalize_duration_metric "$req_p99")"
reqps_num="$(echo "$reqps" | awk '{print $1}')"
echo "$throughput,${reqps_num:-N/A},$latency,$req_p90,$req_p99"
}
metric_snapshot_token() {
local size="$1"
local round="$2"
local attempt="$3"
local safe_size
safe_size="$(printf '%s' "$size" | tr -c '[:alnum:]_.-' '_')"
printf '%s_%s_r%s_a%s' "$TOOL" "$safe_size" "$round" "$attempt"
}
filter_service_metrics_snapshot() {
local input_file="$1"
local output_file="$2"
if [[ -z "$SERVICE_METRICS_FILTER_REGEX" ]]; then
mv "$input_file" "$output_file"
return 0
fi
awk -v pattern="$SERVICE_METRICS_FILTER_REGEX" '$0 ~ pattern { print }' "$input_file" >"$output_file"
if [[ ! -s "$output_file" ]]; then
mv "$input_file" "$output_file"
else
rm -f "$input_file"
fi
}
settle_before_after_metrics_capture() {
if (( SERVICE_METRICS_SETTLE_SECS <= 0 )); then
return 0
fi
if [[ "$DRY_RUN" == "true" ]]; then
echo "[DRY-RUN] sleep ${SERVICE_METRICS_SETTLE_SECS}"
else
sleep "$SERVICE_METRICS_SETTLE_SECS"
fi
}
capture_prometheus_query_snapshot() {
local phase="$1"
local snapshot_file="$2"
local status_file="$3"
local capture_attempt="$4"
"$PYTHON_BIN" - "$SERVICE_PROMETHEUS_QUERY_URL" "$SERVICE_PROMETHEUS_QUERY" "$phase" "$snapshot_file" "$status_file" "$SERVICE_METRICS_SERVICE_NAME" "$SERVICE_METRICS_SETTLE_SECS" "$SERVICE_METRICS_MAX_TIME_SECS" "$SERVICE_METRICS_FILTER_REGEX" "$capture_attempt" <<'PY'
import json
import pathlib
import sys
import urllib.parse
import urllib.request
query_url, query, phase, snapshot_raw, status_raw, service_name, settle_secs, timeout_secs_raw, filter_regex, capture_attempt = sys.argv[1:]
snapshot_path = pathlib.Path(snapshot_raw)
status_path = pathlib.Path(status_raw)
timeout_secs = float(timeout_secs_raw)
def write_status(status):
status_path.write_text(
"\n".join(
[
f"phase={phase}",
f"status={status}",
f"url={query_url}",
f"query={query}",
f"service_name={service_name}",
f"settle_secs={settle_secs}",
f"max_time_secs={timeout_secs_raw}",
f"filter_regex={filter_regex}",
f"capture_attempt={capture_attempt}",
"",
]
),
encoding="utf-8",
)
def escape_label(value):
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
def sample_to_line(sample):
metric = dict(sample.get("metric", {}))
name = metric.pop("__name__", "")
if not name:
return None
labels = ",".join(f'{key}="{escape_label(value)}"' for key, value in sorted(metric.items()))
value = sample.get("value", [None, None])[1]
if value is None:
return None
if labels:
return f"{name}{{{labels}}} {value}"
return f"{name} {value}"
try:
separator = "&" if "?" in query_url else "?"
url = f"{query_url}{separator}{urllib.parse.urlencode({'query': query})}"
request = urllib.request.Request(url, headers={"Accept": "application/json"})
with urllib.request.urlopen(request, timeout=timeout_secs) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception as err: # noqa: BLE001 - shell harness reports failures in status files.
snapshot_path.write_text(f"# prometheus_query_failed error={err}\n", encoding="utf-8")
write_status("query_failed")
raise SystemExit(0)
if payload.get("status") != "success":
snapshot_path.write_text(f"# prometheus_query_failed payload_status={payload.get('status')}\n", encoding="utf-8")
write_status("query_failed")
raise SystemExit(0)
samples = [
sample
for sample in payload.get("data", {}).get("result", [])
if not service_name
or sample.get("metric", {}).get("service.name") == service_name
or sample.get("metric", {}).get("service_name") == service_name
]
lines = [line for sample in samples if (line := sample_to_line(sample))]
if not lines:
snapshot_path.write_text(f"# no_matching_metrics service_name={service_name}\n", encoding="utf-8")
write_status("no_matching_metrics")
raise SystemExit(0)
snapshot_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
write_status("ok")
PY
}
capture_round_service_metrics() {
local size="$1"
local round="$2"
local attempt="$3"
local phase="$4"
if [[ -z "$SERVICE_METRICS_URL" && -z "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
return 0
fi
local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes capture_status
token="$(metric_snapshot_token "$size" "$round" "$attempt")"
snapshot_file="${SERVICE_METRICS_DIR}/${token}_${phase}.prom"
status_file="${SERVICE_METRICS_DIR}/${token}_${phase}.status"
tmp_file="${snapshot_file}.tmp"
filtered_file="${snapshot_file}.filtered.tmp"
if [[ "$DRY_RUN" == "true" ]]; then
: >"$snapshot_file"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
status=not_run_dry_run
url=${SERVICE_METRICS_URL}
prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
prometheus_query=${SERVICE_PROMETHEUS_QUERY}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
echo "$size,$TOOL,$round,$attempt,$phase,dry_run,not_run_dry_run,0,0,0,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
return 0
fi
if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
for ((capture_attempt=1; capture_attempt<=SERVICE_METRICS_CAPTURE_ATTEMPTS; capture_attempt++)); do
capture_prometheus_query_snapshot "$phase" "$tmp_file" "$status_file" "$capture_attempt"
capture_status="$(awk -F= '$1=="status" {print $2; exit}' "$status_file")"
if [[ "${capture_status:-unknown}" == "ok" && -s "$tmp_file" ]]; then
raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')"
filter_service_metrics_snapshot "$tmp_file" "$filtered_file"
mv "$filtered_file" "$snapshot_file"
snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')"
{
echo "raw_bytes=${raw_bytes}"
echo "snapshot_bytes=${snapshot_bytes}"
} >>"$status_file"
echo "$size,$TOOL,$round,$attempt,$phase,prometheus_query,ok,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
return 0
fi
rm -f "$tmp_file" "$filtered_file"
if (( capture_attempt < SERVICE_METRICS_CAPTURE_ATTEMPTS && SERVICE_METRICS_CAPTURE_RETRY_SECS > 0 )); then
sleep "$SERVICE_METRICS_CAPTURE_RETRY_SECS"
fi
done
: >"$snapshot_file"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
status=capture_failed
capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}
url=${SERVICE_PROMETHEUS_QUERY_URL}
query=${SERVICE_PROMETHEUS_QUERY}
max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
echo "$size,$TOOL,$round,$attempt,$phase,prometheus_query,capture_failed,$SERVICE_METRICS_CAPTURE_ATTEMPTS,N/A,N/A,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
echo "WARN: failed to capture service metrics size=${size} round=${round} attempt=${attempt} phase=${phase}" >&2
return 0
fi
capture_direct_metrics_snapshot "$size" "$round" "$attempt" "$phase" service "$SERVICE_METRICS_URL"
}
append_direct_metrics_capture() {
local size="$1" round="$2" attempt="$3" phase="$4" node="$5" status="$6" capture_attempt="$7" raw_bytes="$8" snapshot_bytes="$9" status_file="${10}" snapshot_file="${11}"
if [[ "$node" == "service" ]]; then
echo "$size,$TOOL,$round,$attempt,$phase,prometheus_text,$status,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
return
fi
echo "$size,$TOOL,$round,$attempt,$phase,$node,$status,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX" >> "$NODE_METRICS_CSV"
}
capture_direct_metrics_snapshot() {
local size="$1" round="$2" attempt="$3" phase="$4" node="$5" metrics_url="$6"
local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes
token="$(metric_snapshot_token "$size" "$round" "$attempt")"
if [[ "$node" == "service" ]]; then
snapshot_file="${SERVICE_METRICS_DIR}/${token}_${phase}.prom"
else
snapshot_file="${SERVICE_METRICS_DIR}/${token}_${node}_${phase}.prom"
fi
status_file="${SERVICE_METRICS_DIR}/${token}_${node}_${phase}.status"
tmp_file="${snapshot_file}.tmp"
filtered_file="${snapshot_file}.filtered.tmp"
if [[ "$DRY_RUN" == "true" ]]; then
: >"$snapshot_file"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
node=${node}
status=not_run_dry_run
url=${metrics_url}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
EOF
append_direct_metrics_capture "$size" "$round" "$attempt" "$phase" "$node" not_run_dry_run 0 0 0 "$status_file" "$snapshot_file"
return
fi
for ((capture_attempt=1; capture_attempt<=SERVICE_METRICS_CAPTURE_ATTEMPTS; capture_attempt++)); do
if curl -fsS --noproxy '*' \
--connect-timeout "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" \
--max-time "$SERVICE_METRICS_MAX_TIME_SECS" \
"$metrics_url" >"$tmp_file" && [[ -s "$tmp_file" ]]; then
raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')"
filter_service_metrics_snapshot "$tmp_file" "$filtered_file"
mv "$filtered_file" "$snapshot_file"
snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
node=${node}
status=ok
capture_attempt=${capture_attempt}
url=${metrics_url}
connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}
max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
raw_bytes=${raw_bytes}
snapshot_bytes=${snapshot_bytes}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
append_direct_metrics_capture "$size" "$round" "$attempt" "$phase" "$node" ok "$capture_attempt" "$raw_bytes" "$snapshot_bytes" "$status_file" "$snapshot_file"
return
fi
rm -f "$tmp_file" "$filtered_file"
if (( capture_attempt < SERVICE_METRICS_CAPTURE_ATTEMPTS && SERVICE_METRICS_CAPTURE_RETRY_SECS > 0 )); then
sleep "$SERVICE_METRICS_CAPTURE_RETRY_SECS"
fi
done
: >"$snapshot_file"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
node=${node}
status=capture_failed
capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}
url=${metrics_url}
connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}
max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
append_direct_metrics_capture "$size" "$round" "$attempt" "$phase" "$node" capture_failed "$SERVICE_METRICS_CAPTURE_ATTEMPTS" N/A N/A "$status_file" "$snapshot_file"
echo "WARN: failed to capture metrics size=${size} round=${round} attempt=${attempt} phase=${phase} node=${node}" >&2
}
capture_round_node_metrics() {
local size="$1" round="$2" attempt="$3" phase="$4" entry node metrics_url
if ((${#NODE_METRICS_URLS[@]} == 0)); then
return
fi
for entry in "${NODE_METRICS_URLS[@]}"; do
node="${entry%%=*}"
metrics_url="${entry#*=}"
capture_direct_metrics_snapshot "$size" "$round" "$attempt" "$phase" "$node" "$metrics_url"
done
}
capture_round_node_resources() {
local size="$1" round="$2" attempt="$3" phase="$4" entry node container token snapshot_file
if ((${#NODE_DOCKER_CONTAINERS[@]} == 0)); then
return
fi
for entry in "${NODE_DOCKER_CONTAINERS[@]}"; do
node="${entry%%=*}"
container="${entry#*=}"
token="$(metric_snapshot_token "$size" "$round" "$attempt")"
snapshot_file="$OUT_DIR/node_resources/${token}_${node}_${phase}.json"
if [[ "$DRY_RUN" == "true" ]]; then
: >"$snapshot_file"
echo "$size,$TOOL,$round,$attempt,$phase,$node,$container,not_run_dry_run,$snapshot_file" >> "$NODE_RESOURCE_CSV"
elif docker stats --no-stream --format '{{json .}}' "$container" >"$snapshot_file" && [[ -s "$snapshot_file" ]]; then
echo "$size,$TOOL,$round,$attempt,$phase,$node,$container,ok,$snapshot_file" >> "$NODE_RESOURCE_CSV"
else
: >"$snapshot_file"
echo "$size,$TOOL,$round,$attempt,$phase,$node,$container,capture_failed,$snapshot_file" >> "$NODE_RESOURCE_CSV"
echo "WARN: failed to capture Docker resources node=${node} container=${container}" >&2
fi
done
}
median_from_numbers() {
local values="$1"
local count
count="$(printf '%s\n' "$values" | awk 'NF{c++} END{print c+0}')"
if [[ "$count" -eq 0 ]]; then
echo "N/A"
return
fi
printf '%s\n' "$values" | awk 'NF' | sort -n | awk '
{a[NR]=$1}
END{
n=NR
if (n==0) { print "N/A"; exit }
if (n%2==1) {
printf "%.6f\n", a[(n+1)/2]
} else {
printf "%.6f\n", (a[n/2]+a[n/2+1])/2
}
}'
}
run_one_attempt() {
local size="$1"
local round="$2"
local attempt="$3"
local log_file="$OUT_DIR/logs/${TOOL}_${size}_r${round}_a${attempt}.log"
local status="ok"
local exit_code=0
local started_at_utc finished_at_utc
capture_round_service_metrics "$size" "$round" "$attempt" before
capture_round_node_metrics "$size" "$round" "$attempt" before
capture_round_node_resources "$size" "$round" "$attempt" before
started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
if [[ "$TOOL" == "warp" ]]; then
local warp_host bucket
warp_host="$(normalize_warp_host "$ENDPOINT")"
bucket="$(bucket_for_size "$size")"
local cmd=(
"$WARP_BIN" "$WARP_MODE"
"--host" "$warp_host"
"--access-key" "$ACCESS_KEY"
"--secret-key" "$SECRET_KEY"
"--bucket" "$bucket"
"--obj.size" "$size"
"--concurrent" "$CONCURRENCY"
"--duration" "$DURATION"
"--region" "$REGION"
)
if [[ "$INSECURE" == "true" ]]; then
cmd+=("--insecure")
fi
if ((${#EXTRA_ARGS[@]} > 0)); then
cmd+=("${EXTRA_ARGS[@]}")
fi
if [[ "$DRY_RUN" == "true" ]]; then
printf '[DRY-RUN] %q ' "${cmd[@]}"
printf '\n'
echo "dry run" > "$log_file"
else
if "${cmd[@]}" 2>&1 | tee "$log_file" >&2; then
:
else
exit_code=$?
status="failed"
fi
fi
else
local bucket
bucket="$(bucket_for_size "$size")"
local cmd=(
"$S3BENCH_BIN"
"-accessKey=$ACCESS_KEY"
"-secretKey=$SECRET_KEY"
"-bucket=$bucket"
"-endpoint=$ENDPOINT"
"-region=$REGION"
"-numClients=$CONCURRENCY"
"-numSamples=$SAMPLES"
"-objectSize=$size"
)
if [[ "$INSECURE" == "true" ]]; then
cmd+=("-insecure")
fi
if ((${#EXTRA_ARGS[@]} > 0)); then
cmd+=("${EXTRA_ARGS[@]}")
fi
if [[ "$DRY_RUN" == "true" ]]; then
printf '[DRY-RUN] %q ' "${cmd[@]}"
printf '\n'
echo "dry run" > "$log_file"
else
if "${cmd[@]}" 2>&1 | tee "$log_file" >&2; then
:
else
exit_code=$?
status="failed"
fi
fi
fi
finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
settle_before_after_metrics_capture
capture_round_service_metrics "$size" "$round" "$attempt" after
capture_round_node_metrics "$size" "$round" "$attempt" after
capture_round_node_resources "$size" "$round" "$attempt" after
local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms
if [[ "$DRY_RUN" == "true" ]]; then
throughput_human="N/A"
reqps="N/A"
latency_human="N/A"
throughput_bps="N/A"
latency_ms="N/A"
req_p90_human="N/A"
req_p90_ms="N/A"
req_p99_human="N/A"
req_p99_ms="N/A"
else
metrics="$(extract_metrics "$log_file")"
throughput_human="$(echo "$metrics" | cut -d',' -f1)"
reqps="$(echo "$metrics" | cut -d',' -f2)"
latency_human="$(echo "$metrics" | cut -d',' -f3)"
req_p90_human="$(echo "$metrics" | cut -d',' -f4)"
req_p99_human="$(echo "$metrics" | cut -d',' -f5)"
throughput_bps="$(to_bps "$throughput_human")"
latency_ms="$(to_ms "$latency_human")"
req_p90_ms="$(to_ms "$req_p90_human")"
req_p99_ms="$(to_ms "$req_p99_human")"
fi
if [[ "$DRY_RUN" != "true" && "$status" == "ok" ]]; then
if [[ "$throughput_bps" == "N/A" && "$reqps" == "N/A" ]]; then
status="failed"
exit_code=1
fi
fi
echo "$size,$TOOL,$round,$attempt,$CONCURRENCY,$status,$exit_code,$started_at_utc,$finished_at_utc,$throughput_human,$throughput_bps,$reqps,$latency_human,$latency_ms,$log_file,$req_p90_human,$req_p90_ms,$req_p99_human,$req_p99_ms" >> "$ROUND_CSV"
echo "$status"
}
run_size() {
local size="$1"
local round success attempt rc max_attempts
max_attempts=$((RETRY_PER_ROUND + 1))
for ((round=1; round<=ROUNDS; round++)); do
success="no"
for ((attempt=1; attempt<=max_attempts; attempt++)); do
echo "==== size=$size round=$round attempt=$attempt/$max_attempts ===="
rc="$(run_one_attempt "$size" "$round" "$attempt")"
if [[ "$rc" == "ok" || "$DRY_RUN" == "true" ]]; then
success="yes"
break
fi
if (( attempt < RETRY_PER_ROUND+1 )); then
echo "Round failed, retry in ${RETRY_SLEEP_SECS}s..."
cooldown_sleep "$RETRY_SLEEP_SECS"
fi
done
if [[ "$success" == "no" ]]; then
echo "WARN: size=$size round=$round failed after retries."
FAILED_FINAL_ROUNDS=$((FAILED_FINAL_ROUNDS + 1))
fi
if (( COOLDOWN_SECS > 0 && round < ROUNDS )); then
echo "Cooldown after size=$size round=$round: ${COOLDOWN_SECS}s"
cooldown_sleep "$COOLDOWN_SECS"
fi
done
}
build_median_summary() {
local sizes_arr size
IFS=',' read -r -a sizes_arr <<< "$SIZES"
for raw in "${sizes_arr[@]}"; do
size="$(trim "$raw")"
[[ -z "$size" ]] && continue
local ok_rounds fail_rounds t_vals r_vals l_vals p90_vals p99_vals
ok_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" {c++} END{print c+0}' "$ROUND_CSV")"
fail_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6!="ok" {c++} END{print c+0}' "$ROUND_CSV")"
t_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $11!="N/A" {print $11}' "$ROUND_CSV")"
r_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $12!="N/A" {print $12}' "$ROUND_CSV")"
l_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $14!="N/A" {print $14}' "$ROUND_CSV")"
p90_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $17!="N/A" {print $17}' "$ROUND_CSV")"
p99_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $19!="N/A" {print $19}' "$ROUND_CSV")"
local m_t m_r m_l m_p90 m_p99
m_t="$(median_from_numbers "$t_vals")"
m_r="$(median_from_numbers "$r_vals")"
m_l="$(median_from_numbers "$l_vals")"
m_p90="$(median_from_numbers "$p90_vals")"
m_p99="$(median_from_numbers "$p99_vals")"
echo "$size,$TOOL,$CONCURRENCY,$ok_rounds,$fail_rounds,$m_t,$m_r,$m_l,$m_p90,$m_p99" >> "$MEDIAN_CSV"
done
}
compare_baseline() {
if [[ -z "$BASELINE_CSV" ]]; then
return
fi
echo "size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct" > "$COMPARE_CSV"
awk -F',' '
NR==FNR {
if (FNR==1) next
key=$1
b_req[key]=$7
b_lat[key]=$8
b_thr[key]=$6
next
}
FNR==1 {next}
{
key=$1
n_thr=$6; n_req=$7; n_lat=$8
br=(key in b_req)?b_req[key]:"N/A"
bl=(key in b_lat)?b_lat[key]:"N/A"
bt=(key in b_thr)?b_thr[key]:"N/A"
dr="N/A"; dl="N/A"; dt="N/A"
if (br!="N/A" && n_req!="N/A" && br+0!=0) dr=sprintf("%.2f", ((n_req-br)/br)*100)
if (bl!="N/A" && n_lat!="N/A" && bl+0!=0) dl=sprintf("%.2f", ((n_lat-bl)/bl)*100)
if (bt!="N/A" && n_thr!="N/A" && bt+0!=0) dt=sprintf("%.2f", ((n_thr-bt)/bt)*100)
print key "," $2 "," $3 "," n_req "," br "," dr "," n_lat "," bl "," dl "," n_thr "," bt "," dt
}
' "$BASELINE_CSV" "$MEDIAN_CSV" >> "$COMPARE_CSV"
}
main() {
parse_args "$@"
validate_args
require_cmd rg
require_cmd awk
require_cmd sort
if [[ "$TOOL" == "warp" ]]; then
require_cmd "$WARP_BIN"
else
require_cmd "$S3BENCH_BIN"
fi
setup_output
echo "Output dir: $OUT_DIR"
echo "Tool: $TOOL"
echo "Sizes: $SIZES"
echo "Concurrency: $CONCURRENCY"
echo "Rounds: $ROUNDS"
echo "Retry per round: $RETRY_PER_ROUND"
echo "Cooldown secs: $COOLDOWN_SECS"
echo "Server image digest: ${SERVER_IMAGE_DIGEST:-unknown}"
echo "Server revision: ${SERVER_REVISION:-unknown}"
if ((${#RUN_LABELS[@]} > 0)); then
echo "Run labels: ${RUN_LABELS[*]}"
fi
if ((${#NODE_METRICS_URLS[@]} > 0)); then
echo "Node metric targets: ${#NODE_METRICS_URLS[@]}"
fi
if ((${#NODE_DOCKER_CONTAINERS[@]} > 0)); then
echo "Node Docker targets: ${#NODE_DOCKER_CONTAINERS[@]}"
fi
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "Service metrics filter: $SERVICE_METRICS_FILTER_REGEX"
if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "Service Prometheus query URL: $SERVICE_PROMETHEUS_QUERY_URL"
echo "Service Prometheus query: $SERVICE_PROMETHEUS_QUERY"
if [[ -n "$SERVICE_METRICS_SERVICE_NAME" ]]; then
echo "Service metrics service name: $SERVICE_METRICS_SERVICE_NAME"
fi
fi
fi
IFS=',' read -r -a size_arr <<< "$SIZES"
local total_sizes="${#size_arr[@]}"
local size_index=0
for raw in "${size_arr[@]}"; do
size="$(trim "$raw")"
[[ -z "$size" ]] && continue
size_index=$(( size_index + 1 ))
run_size "$size"
if (( COOLDOWN_SECS > 0 && size_index < total_sizes )); then
echo "Cooldown after size=$size: ${COOLDOWN_SECS}s"
cooldown_sleep "$COOLDOWN_SECS"
fi
done
build_median_summary
compare_baseline
echo
echo "=== Median Summary ==="
cat "$MEDIAN_CSV"
if [[ -n "$BASELINE_CSV" ]]; then
echo
echo "=== Baseline Compare ==="
cat "$COMPARE_CSV"
fi
if [[ "$DRY_RUN" != "true" && "$FAILED_FINAL_ROUNDS" -gt 0 ]]; then
echo "ERROR: ${FAILED_FINAL_ROUNDS} benchmark rounds failed after retries." >&2
exit 1
fi
}
main "$@"
Reference in New Issue View Git Blame Copy Permalink