gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): harden internode data-path controls (#4224)

Browse Source 
* fix(rio): propagate http writer shutdown errors

* fix(ecstore): unify remote lock rpc deadlines

* fix(storage): reject corrupt read multiple payloads

* feat(rio): add internode http tuning profiles

* feat(metrics): add internode baseline signals

* feat(ecstore): observe shard locality topology

* feat(ecstore): gate shard locality scheduling

* feat(ecstore): gate batch read version rpc

* feat(ecstore): observe batch processor adaptation

* feat(ecstore): gate batch processor observation

* docs: add get benchmark regression analysis

* docs: add issue 797 execution plan status

* fix(ecstore): require explicit batch rpc support

* fix(ecstore): honor documented batch read gate

* fix(ecstore): keep batch read gate stable per call

* chore: update workspace dependencies

* feat(ecstore): log batch read gate decisions

* feat(ecstore): count batch read gate decisions

* test(issue-797): add local internode A/B runner

* test(rio): fix tuning profile spelling fixture

* fix(protocols): adapt sftp channel open callbacks

* fix(metrics): wrap batch processor observation args

* chore(docs): keep issue notes local only

* fix(storage): address internode review feedback

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

* fix(storage): address internode data-path review findings

- Run the BatchReadVersion auto-mode unary fallback outside the batch
  RPC deadline so each read_version keeps its own per-op timeout and
  health accounting instead of racing the whole batch against one
  drive timeout.
- Cap adaptive batch-processor concurrency growth at a hard multiple
  of the configured baseline so sustained fast batches cannot ratchet
  past the configured limit.
- Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE,
  and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading
  the environment on hot paths.
- Skip shard read-cost collection in observe mode when stage metrics
  are disabled, and cache the local endpoint host list instead of
  rebuilding it on every read.
- Allow --warp-extra-args values starting with -- and drop the unused
  warp_hosts_csv helper in the issue-797 A/B runner.

Co-Authored-By: heihutu<heihutu@gmail.com>

* fix(storage): align buffer clamp test with media cap

* fix(ecstore): release optimized read locks before streaming

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
houseme
2026-07-03 17:08:15 +08:00
committed by GitHub
parent a9ac5f578d
commit 25d80d7c60
26 changed files with 3627 additions and 483 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_issue797_local_4node_16disk_ab.sh
Executable
+582
View File
@@ -0,0 +1,582 @@
#!/usr/bin/env bash
set -euo pipefail
# Local 4-node / 16-disk RustFS runner for rustfs/backlog#797.
# It starts four local RustFS processes, runs warp workloads, and captures
# health, logs, benchmark summaries, and optional signed admin metrics.
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUSTFS_BIN="${RUSTFS_BIN:-${PROJECT_ROOT}/target/debug/rustfs}"
BUILD_BIN="${BUILD_BIN:-true}"
BASE_PORT="${BASE_PORT:-19100}"
NODE_COUNT="${NODE_COUNT:-4}"
DISKS_PER_NODE="${DISKS_PER_NODE:-4}"
ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
REGION="${REGION:-us-east-1}"
SIZES="${SIZES:-4KiB,1MiB}"
CONCURRENCY="${CONCURRENCY:-8}"
DURATION="${DURATION:-60s}"
WARP_BIN="${WARP_BIN:-warp}"
WARP_MODE="${WARP_MODE:-mixed}"
WARP_EXTRA_ARGS="${WARP_EXTRA_ARGS:---noclear}"
PROFILES="${PROFILES:-baseline,metrics_logging}"
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/issue-797-local-4node-16disk-ab-$(date -u +%Y%m%dT%H%M%SZ)}"
DATA_ROOT="${DATA_ROOT:-/tmp/issue797-local-4node-16disk-ab-$(date -u +%Y%m%dT%H%M%SZ)}"
KEEP_DATA="${KEEP_DATA:-false}"
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
HEALTH_POLL_SECS="${HEALTH_POLL_SECS:-2}"
CAPTURE_ADMIN_METRICS="${CAPTURE_ADMIN_METRICS:-true}"
AWSCURL_BIN="${AWSCURL_BIN:-awscurl}"
CURL_BIN="${CURL_BIN:-curl}"
RG_BIN="${RG_BIN:-rg}"
DRY_RUN=false
PIDS=()
usage() {
cat <<'USAGE'
Usage:
scripts/run_issue797_local_4node_16disk_ab.sh [options]
Options:
--rustfs-bin <path> RustFS binary (default: target/debug/rustfs)
--skip-build Do not build rustfs before running
--base-port <port> First node port; uses port..port+3
--sizes <csv> Object sizes for warp (default: 4KiB,1MiB)
--concurrency <n> Warp concurrency (default: 8)
--duration <dur> Warp duration per size/profile (default: 60s)
--profiles <csv> baseline,metrics_logging,locality_on
--out-dir <dir> Output directory
--data-root <dir> Temporary disk root
--keep-data Keep data root after exit
--warp-extra-args <args> Extra args appended to warp (default: --noclear)
--skip-admin-metrics Do not attempt signed admin metrics capture
--dry-run Print planned layout without running
-h, --help Show help
Profiles:
baseline Metrics exports, file logging, shard locality, and batch
processor observation disabled.
metrics_logging Enables metrics export gate, bounded warn-level file
logging, shard locality observe mode, and batch processor
observe mode.
locality_on Same as metrics_logging, but uses shard locality on mode.
Notes:
Admin metrics are captured from /rustfs/admin/v3/metrics when awscurl or
curl --aws-sigv4 is available. If neither is available, benchmark still runs
and records a skipped metrics status file.
USAGE
}
log() {
printf '[INFO] %s\n' "$*"
}
warn() {
printf '[WARN] %s\n' "$*" >&2
}
die() {
printf '[ERROR] %s\n' "$*" >&2
exit 1
}
arg_value() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" || "$value" == --* ]]; then
die "missing value for ${flag}"
fi
printf '%s\n' "$value"
}
# Like arg_value, but accepts values that themselves start with `--`
# (e.g. `--warp-extra-args --noclear`).
arg_value_allow_dashes() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" ]]; then
die "missing value for ${flag}"
fi
printf '%s\n' "$value"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
--rustfs-bin) RUSTFS_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--skip-build) BUILD_BIN=false; shift ;;
--base-port) BASE_PORT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--sizes) SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--concurrency) CONCURRENCY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--duration) DURATION="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--profiles) PROFILES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--data-root) DATA_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--keep-data) KEEP_DATA=true; shift ;;
--warp-extra-args) WARP_EXTRA_ARGS="$(arg_value_allow_dashes "$1" "${2:-}")"; shift 2 ;;
--skip-admin-metrics) CAPTURE_ADMIN_METRICS=false; shift ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage; exit 0 ;;
*) die "unknown arg: $1" ;;
esac
done
}
is_positive_integer() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
validate_args() {
is_positive_integer "$BASE_PORT" || die "--base-port must be a positive integer"
is_positive_integer "$CONCURRENCY" || die "--concurrency must be a positive integer"
[[ "$NODE_COUNT" == "4" ]] || die "NODE_COUNT is fixed to 4 for this runner"
[[ "$DISKS_PER_NODE" == "4" ]] || die "DISKS_PER_NODE is fixed to 4 for this runner"
}
require_cmd() {
if ! command -v "$1" >/dev/null 2>&1; then
die "command not found: $1"
fi
}
print_redacted_command() {
local redact_next=false
local arg
printf 'Command:'
for arg in "$@"; do
if [[ "$redact_next" == "true" ]]; then
printf ' %q' "REDACTED"
redact_next=false
continue
fi
case "$arg" in
--access-key|--secret-key)
printf ' %q' "$arg"
redact_next=true
;;
-accessKey=*|-secretKey=*)
printf ' %q' "${arg%%=*}=REDACTED"
;;
*)
printf ' %q' "$arg"
;;
esac
done
printf '\n'
}
endpoint_for_node() {
local endpoint_node_id="$1"
printf 'http://127.0.0.1:%s' "$((BASE_PORT + endpoint_node_id - 1))"
}
endpoint_label() {
local endpoint="$1"
local label
label="${endpoint#*://}"
label="${label%%/*}"
printf '%s' "$label" | tr -c 'A-Za-z0-9._-' '_'
}
all_endpoints_csv() {
local endpoint_loop_id
local endpoints=()
for ((endpoint_loop_id = 1; endpoint_loop_id <= NODE_COUNT; endpoint_loop_id++)); do
endpoints+=("$(endpoint_for_node "$endpoint_loop_id")")
done
local IFS=','
printf '%s' "${endpoints[*]}"
}
metrics_url() {
# SCANNER(1) + NET(32) + RPC(256): enough for readiness context plus
# internode aggregate traffic without collecting the heavier all-metrics set.
printf '%s/rustfs/admin/v3/metrics?types=289&by-host=true&n=1\n' "${1%/}"
}
curl_supports_aws_sigv4() {
"$CURL_BIN" --help all 2>/dev/null | grep -q -- '--aws-sigv4'
}
profile_is_supported() {
case "$1" in
baseline|metrics_logging|locality_on) return 0 ;;
*) return 1 ;;
esac
}
profile_env_file() {
local profile="$1"
local env_file="$2"
case "$profile" in
baseline)
cat >"$env_file" <<'EOF'
RUSTFS_OBS_LOGGER_LEVEL=off
RUSTFS_OBS_TRACES_EXPORT_ENABLED=false
RUSTFS_OBS_METRICS_EXPORT_ENABLED=false
RUSTFS_OBS_LOGS_EXPORT_ENABLED=false
RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false
RUSTFS_OBS_USE_STDOUT=false
RUSTFS_OBS_LOG_STDOUT_ENABLED=false
RUSTFS_BATCH_PROCESSOR_ADAPTIVE=off
RUSTFS_SHARD_LOCALITY_SCHEDULING=off
EOF
;;
metrics_logging)
cat >"$env_file" <<'EOF'
RUSTFS_OBS_LOGGER_LEVEL=warn
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318
RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=500
RUSTFS_OBS_TRACES_EXPORT_ENABLED=false
RUSTFS_OBS_METRICS_EXPORT_ENABLED=true
RUSTFS_OBS_LOGS_EXPORT_ENABLED=false
RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false
RUSTFS_OBS_USE_STDOUT=false
RUSTFS_OBS_LOG_STDOUT_ENABLED=false
RUSTFS_OBS_METER_INTERVAL=5
RUSTFS_OBS_LOG_KEEP_FILES=2
RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES=268435456
RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES=134217728
RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS=0
RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=30
RUSTFS_BATCH_PROCESSOR_ADAPTIVE=observe
RUSTFS_SHARD_LOCALITY_SCHEDULING=observe
EOF
;;
locality_on)
cat >"$env_file" <<'EOF'
RUSTFS_OBS_LOGGER_LEVEL=warn
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318
RUSTFS_OBS_ENDPOINT_TIMEOUT_MILLIS=500
RUSTFS_OBS_TRACES_EXPORT_ENABLED=false
RUSTFS_OBS_METRICS_EXPORT_ENABLED=true
RUSTFS_OBS_LOGS_EXPORT_ENABLED=false
RUSTFS_OBS_PROFILING_EXPORT_ENABLED=false
RUSTFS_OBS_USE_STDOUT=false
RUSTFS_OBS_LOG_STDOUT_ENABLED=false
RUSTFS_OBS_METER_INTERVAL=5
RUSTFS_OBS_LOG_KEEP_FILES=2
RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES=268435456
RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES=134217728
RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS=0
RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=30
RUSTFS_BATCH_PROCESSOR_ADAPTIVE=observe
RUSTFS_SHARD_LOCALITY_SCHEDULING=on
EOF
;;
esac
}
load_profile_env() {
local env_file="$1"
set -a
# shellcheck disable=SC1090
source "$env_file"
set +a
}
prepare_profile_layout() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
mkdir -p "$profile_dir"/{bench,health,logs,metrics,pids}
profile_env_file "$profile" "$profile_dir/profile.env"
}
build_volumes() {
local profile="$1"
local node_index disk_index
local volumes=()
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
for ((disk_index = 1; disk_index <= DISKS_PER_NODE; disk_index++)); do
volumes+=("http://127.0.0.1:$((BASE_PORT + node_index - 1))${DATA_ROOT}/${profile}/node${node_index}/disk${disk_index}")
done
done
printf '%s ' "${volumes[@]}"
}
prepare_data_dirs() {
local profile="$1"
local node_index disk_index
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
for ((disk_index = 1; disk_index <= DISKS_PER_NODE; disk_index++)); do
mkdir -p "${DATA_ROOT}/${profile}/node${node_index}/disk${disk_index}"
done
done
}
stop_nodes() {
local pid
if [[ ${#PIDS[@]} -eq 0 ]]; then
return
fi
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" >/dev/null 2>&1; then
kill "$pid" >/dev/null 2>&1 || true
fi
done
for ((_ = 1; _ <= 5; _++)); do
local remaining=0
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" >/dev/null 2>&1; then
remaining=$((remaining + 1))
fi
done
[[ "$remaining" -eq 0 ]] && break
sleep 1
done
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" >/dev/null 2>&1; then
kill -9 "$pid" >/dev/null 2>&1 || true
fi
wait "$pid" >/dev/null 2>&1 || true
done
PIDS=()
}
cleanup() {
stop_nodes
if [[ "$KEEP_DATA" != "true" && "$DRY_RUN" != "true" && -n "$DATA_ROOT" && -d "$DATA_ROOT" ]]; then
rm -rf "$DATA_ROOT"
fi
}
trap cleanup EXIT
start_nodes() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local volumes
volumes="$(build_volumes "$profile")"
prepare_data_dirs "$profile"
load_profile_env "$profile_dir/profile.env"
local node_index endpoint log_file pid_file
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
endpoint="$(endpoint_for_node "$node_index")"
log_file="$profile_dir/logs/node${node_index}.log"
pid_file="$profile_dir/pids/node${node_index}.pid"
(
export RUSTFS_ACCESS_KEY="$ACCESS_KEY"
export RUSTFS_SECRET_KEY="$SECRET_KEY"
export RUSTFS_ADDRESS="127.0.0.1:$((BASE_PORT + node_index - 1))"
export RUSTFS_CONSOLE_ENABLE=false
export RUSTFS_SCANNER_ENABLED=false
export RUSTFS_SCANNER_START_DELAY_SECS=3600
export RUSTFS_SCANNER_CYCLE=3600
export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
export RUSTFS_VOLUMES="$volumes"
export RUSTFS_OBS_SERVICE_NAME="rustfs-issue797-${profile}-node${node_index}"
export RUSTFS_OBS_LOG_DIRECTORY="$profile_dir/logs/node${node_index}"
mkdir -p "$RUSTFS_OBS_LOG_DIRECTORY"
# Keep both env and positional volumes: current CLI still requires the
# positional server volumes, while env makes the layout visible in logs.
# shellcheck disable=SC2086
exec "$RUSTFS_BIN" server $volumes
) >"$log_file" 2>&1 &
PIDS+=("$!")
printf '%s\n' "$!" >"$pid_file"
log "started ${profile} node${node_index} ${endpoint} pid=$!"
done
}
wait_for_health() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local deadline node_index endpoint status_file
deadline=$((SECONDS + WAIT_TIMEOUT_SECS))
while (( SECONDS < deadline )); do
local ready=0
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
endpoint="$(endpoint_for_node "$node_index")"
status_file="$profile_dir/health/node${node_index}.live"
if "$CURL_BIN" -fsS --max-time 2 "${endpoint}/health/live" >"$status_file" 2>"$status_file.err"; then
ready=$((ready + 1))
fi
done
if [[ "$ready" -eq "$NODE_COUNT" ]]; then
log "${profile}: all nodes are live"
return 0
fi
sleep "$HEALTH_POLL_SECS"
done
for ((node_index = 1; node_index <= NODE_COUNT; node_index++)); do
warn "${profile} node${node_index} log tail:"
tail -n 40 "$profile_dir/logs/node${node_index}.log" >&2 || true
done
die "${profile}: timed out waiting for health"
}
capture_admin_metrics() {
local profile="$1"
local phase="$2"
local profile_dir="$OUT_DIR/$profile"
local status_file="$profile_dir/metrics/${phase}.status"
local endpoint label metrics_file endpoints
if [[ "$CAPTURE_ADMIN_METRICS" != "true" ]]; then
echo "skipped: disabled" >"$status_file"
return 0
fi
local capture_method=""
if command -v "$AWSCURL_BIN" >/dev/null 2>&1; then
capture_method="awscurl"
elif curl_supports_aws_sigv4; then
capture_method="curl-sigv4"
else
echo "skipped: awscurl not found and curl lacks --aws-sigv4" >"$status_file"
return 0
fi
echo "capturing: ${capture_method}" >"$status_file"
IFS=',' read -r -a endpoints <<<"$(all_endpoints_csv)"
for endpoint in "${endpoints[@]}"; do
label="$(endpoint_label "$endpoint")"
metrics_file="$profile_dir/metrics/${phase}.${label}.ndjson"
if [[ "$capture_method" == "awscurl" ]]; then
AWS_ACCESS_KEY_ID="$ACCESS_KEY" \
AWS_SECRET_ACCESS_KEY="$SECRET_KEY" \
AWS_DEFAULT_REGION="$REGION" \
"$AWSCURL_BIN" \
--service s3 \
--region "$REGION" \
--request GET \
"$(metrics_url "$endpoint")" \
>"$metrics_file" 2>"$metrics_file.err" || true
else
local curl_config="$profile_dir/metrics/.curl-sigv4-${phase}.${label}.conf"
{
printf 'aws-sigv4 = "aws:amz:%s:s3"\n' "$REGION"
printf 'user = "%s:%s"\n' "$ACCESS_KEY" "$SECRET_KEY"
printf 'request = "GET"\n'
printf 'max-time = 10\n'
printf 'fail\n'
printf 'silent\n'
printf 'show-error\n'
} >"$curl_config"
chmod 600 "$curl_config"
"$CURL_BIN" --config "$curl_config" "$(metrics_url "$endpoint")" >"$metrics_file" 2>"$metrics_file.err" || true
rm -f "$curl_config"
fi
done
echo "done: ${capture_method}" >"$status_file"
}
run_bench() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local bucket_profile="${profile//_/-}"
local -a cmd=(
"${PROJECT_ROOT}/scripts/run_object_batch_bench.sh"
--tool warp
--endpoint "$(all_endpoints_csv)"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--region "$REGION"
--auto-new-bucket
--bucket-prefix "issue797-${bucket_profile}"
--sizes "$SIZES"
--concurrency "$CONCURRENCY"
--duration "$DURATION"
--warp-bin "$WARP_BIN"
--warp-mode "$WARP_MODE"
--out-dir "$profile_dir/bench"
)
if [[ -n "$WARP_EXTRA_ARGS" ]]; then
cmd+=(--extra-args "$WARP_EXTRA_ARGS")
fi
print_redacted_command "${cmd[@]}" >"$profile_dir/bench/command.txt"
log "${profile}: running warp benchmark"
"${cmd[@]}" 2>&1 | tee "$profile_dir/bench/run.log"
}
extract_tail_summary() {
local profile="$1"
local profile_dir="$OUT_DIR/$profile"
local output="$profile_dir/bench/tail_latency_summary.txt"
: >"$output"
if ! command -v "$RG_BIN" >/dev/null 2>&1; then
echo "rg not found; skipped" >"$output"
return 0
fi
"$RG_BIN" -n 'Average:|Median:|90th:|99th:|Fastest:|Slowest:|StdDev:|Total:|Throughput by host|warp: <ERROR>' \
"$profile_dir/bench"/*.log >"$output" || true
}
write_run_meta() {
local meta="$OUT_DIR/run-meta.txt"
mkdir -p "$OUT_DIR"
{
echo "created_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "rustfs_bin=$RUSTFS_BIN"
echo "base_port=$BASE_PORT"
echo "node_count=$NODE_COUNT"
echo "disks_per_node=$DISKS_PER_NODE"
echo "sizes=$SIZES"
echo "concurrency=$CONCURRENCY"
echo "duration=$DURATION"
echo "profiles=$PROFILES"
echo "warp_extra_args=$WARP_EXTRA_ARGS"
echo "endpoints=$(all_endpoints_csv)"
echo "data_root=$DATA_ROOT"
echo "keep_data=$KEEP_DATA"
echo "capture_admin_metrics=$CAPTURE_ADMIN_METRICS"
} >"$meta"
}
run_profile() {
local profile="$1"
profile_is_supported "$profile" || die "unsupported profile: $profile"
prepare_profile_layout "$profile"
start_nodes "$profile"
wait_for_health "$profile"
capture_admin_metrics "$profile" before
run_bench "$profile"
capture_admin_metrics "$profile" after
extract_tail_summary "$profile"
stop_nodes
}
main() {
parse_args "$@"
validate_args
write_run_meta
if [[ "$DRY_RUN" == "true" ]]; then
log "dry run only"
cat "$OUT_DIR/run-meta.txt"
return 0
fi
require_cmd "$CURL_BIN"
require_cmd "$WARP_BIN"
if [[ "$BUILD_BIN" == "true" ]]; then
log "building rustfs binary"
cargo build -p rustfs --bin rustfs
fi
[[ -x "$RUSTFS_BIN" ]] || die "rustfs binary is not executable: $RUSTFS_BIN"
local profile
IFS=',' read -r -a profile_arr <<<"$PROFILES"
for profile in "${profile_arr[@]}"; do
profile="${profile//[[:space:]]/}"
[[ -z "$profile" ]] && continue
run_profile "$profile"
done
log "done. Output dir: $OUT_DIR"
}
main "$@"