From 960c13a34b65d8688a34d8d501c05b1b19a5cc23 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 21 Apr 2026 15:20:57 +0800 Subject: [PATCH] feat(storage): wire capacity/object perf tuning and add batch benchmark runners (#2628) --- _typos.toml | 1 + crates/config/src/constants/capacity.rs | 9 + crates/ecstore/src/set_disk.rs | 5 +- .../object-capacity/src/capacity_manager.rs | 113 +++- rustfs/src/capacity/mod.rs | 1 + scripts/run_object_batch_bench.sh | 258 ++++++++++ scripts/run_object_batch_bench_abc.sh | 380 ++++++++++++++ scripts/run_object_batch_bench_enhanced.sh | 487 ++++++++++++++++++ 8 files changed, 1239 insertions(+), 15 deletions(-) create mode 100755 scripts/run_object_batch_bench.sh create mode 100755 scripts/run_object_batch_bench_abc.sh create mode 100755 scripts/run_object_batch_bench_enhanced.sh diff --git a/_typos.toml b/_typos.toml index 2d1aa7e51..06c56774f 100644 --- a/_typos.toml +++ b/_typos.toml @@ -39,6 +39,7 @@ abd = "abd" mak = "mak" gae = "gae" GAE = "GAE" +thr = "thr" # s3-tests original test names (cannot be changed) nonexisted = "nonexisted" consts = "consts" diff --git a/crates/config/src/constants/capacity.rs b/crates/config/src/constants/capacity.rs index 7afb50590..179326cc5 100644 --- a/crates/config/src/constants/capacity.rs +++ b/crates/config/src/constants/capacity.rs @@ -39,6 +39,9 @@ pub const ENV_CAPACITY_STAT_TIMEOUT: &str = "RUSTFS_CAPACITY_STAT_TIMEOUT"; /// Environment variable for sample rate pub const ENV_CAPACITY_SAMPLE_RATE: &str = "RUSTFS_CAPACITY_SAMPLE_RATE"; +/// Environment variable for metrics logging interval +pub const ENV_CAPACITY_METRICS_INTERVAL: &str = "RUSTFS_CAPACITY_METRICS_INTERVAL"; + /// Environment variable for following symbolic links during capacity calculation pub const ENV_CAPACITY_FOLLOW_SYMLINKS: &str = "RUSTFS_CAPACITY_FOLLOW_SYMLINKS"; @@ -89,6 +92,10 @@ pub const DEFAULT_STAT_TIMEOUT_SECS: u64 = 3; /// Default: 200 pub const DEFAULT_SAMPLE_RATE: usize = 200; +/// Capacity metrics logging interval in seconds +/// Default: 600 seconds (10 minutes) +pub const DEFAULT_CAPACITY_METRICS_INTERVAL_SECS: u64 = 600; + /// Follow symbolic links during capacity calculation /// Default: false (disabled for safety) pub const DEFAULT_CAPACITY_FOLLOW_SYMLINKS: bool = false; @@ -130,6 +137,7 @@ mod tests { assert_eq!(ENV_CAPACITY_MAX_FILES_THRESHOLD, "RUSTFS_CAPACITY_MAX_FILES_THRESHOLD"); assert_eq!(ENV_CAPACITY_STAT_TIMEOUT, "RUSTFS_CAPACITY_STAT_TIMEOUT"); assert_eq!(ENV_CAPACITY_SAMPLE_RATE, "RUSTFS_CAPACITY_SAMPLE_RATE"); + assert_eq!(ENV_CAPACITY_METRICS_INTERVAL, "RUSTFS_CAPACITY_METRICS_INTERVAL"); assert_eq!(ENV_CAPACITY_FOLLOW_SYMLINKS, "RUSTFS_CAPACITY_FOLLOW_SYMLINKS"); assert_eq!(ENV_CAPACITY_MAX_SYMLINK_DEPTH, "RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH"); assert_eq!(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, "RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT"); @@ -147,6 +155,7 @@ mod tests { assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000); assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3); assert_eq!(DEFAULT_SAMPLE_RATE, 200); + assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600); assert_eq!(DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, 3); assert_eq!(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 2); assert_eq!(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 15); diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 2ace75f3e..80c9893fe 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -707,6 +707,10 @@ impl ObjectIO for SetDisks { tokio::spawn(async move { let _guard = read_lock_guard; // keep guard alive until task ends (None if optimization enabled) let mut writer = wd; + // Do not wrap the entire read+write pipeline in `disk_read_timeout`. + // `get_object_with_fileinfo` also waits on `writer`, so an outer timeout + // would incorrectly treat downstream backpressure as disk-read latency. + // Disk read timeouts must be enforced at the actual disk I/O operations. if let Err(e) = Self::get_object_with_fileinfo( &bucket, &object, @@ -757,7 +761,6 @@ impl ObjectIO for SetDisks { user_defined.insert(key.clone(), value.clone()); } } - let sc_parity_drives = { if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { sc.get_parity_for_sc(user_defined.get(AMZ_STORAGE_CLASS).cloned().unwrap_or_default().as_str()) diff --git a/crates/object-capacity/src/capacity_manager.rs b/crates/object-capacity/src/capacity_manager.rs index 24a386537..92edb6ba1 100644 --- a/crates/object-capacity/src/capacity_manager.rs +++ b/crates/object-capacity/src/capacity_manager.rs @@ -20,13 +20,14 @@ use futures::FutureExt; use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope}; use rustfs_config::{ DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH, - DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, - DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE, DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, - DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_TRIGGER_DELAY_SECS, - ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_FOLLOW_SYMLINKS, - ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT, ENV_CAPACITY_MIN_TIMEOUT, - ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL, ENV_CAPACITY_STALL_TIMEOUT, ENV_CAPACITY_STAT_TIMEOUT, - ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY, + DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, + DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE, + DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD, + DEFAULT_WRITE_TRIGGER_DELAY_SECS, ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD, + ENV_CAPACITY_FOLLOW_SYMLINKS, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT, + ENV_CAPACITY_METRICS_INTERVAL, ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL, + ENV_CAPACITY_STALL_TIMEOUT, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, + ENV_CAPACITY_WRITE_TRIGGER_DELAY, }; use rustfs_io_metrics::capacity_metrics::{ record_capacity_current_bytes, record_capacity_dirty_disk_count, record_capacity_refresh_inflight, @@ -63,6 +64,8 @@ struct CachedCapacityConfig { stat_timeout: Duration, /// Sample rate sample_rate: usize, + /// Metrics logging interval + metrics_interval: Duration, /// Follow symlinks flag follow_symlinks: bool, /// Max symlink depth @@ -97,6 +100,10 @@ impl CachedCapacityConfig { max_files_threshold: get_env_usize(ENV_CAPACITY_MAX_FILES_THRESHOLD, DEFAULT_MAX_FILES_THRESHOLD), stat_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_STAT_TIMEOUT, DEFAULT_STAT_TIMEOUT_SECS)), sample_rate: get_env_usize(ENV_CAPACITY_SAMPLE_RATE, DEFAULT_SAMPLE_RATE), + metrics_interval: Duration::from_secs(get_env_u64( + ENV_CAPACITY_METRICS_INTERVAL, + DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, + )), follow_symlinks: get_env_bool(ENV_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_FOLLOW_SYMLINKS), max_symlink_depth: get_env_u64(ENV_CAPACITY_MAX_SYMLINK_DEPTH, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH as u64) as u8, enable_dynamic_timeout: get_env_bool(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT), @@ -204,6 +211,18 @@ pub fn get_sample_rate() -> usize { get_cached_config().sample_rate } +/// Get capacity metrics logging interval from environment or default +#[cfg(not(test))] +pub fn get_metrics_interval() -> Duration { + get_cached_config().metrics_interval +} + +/// Get capacity metrics logging interval from environment or default (test mode) +#[cfg(test)] +pub fn get_metrics_interval() -> Duration { + get_cached_config().metrics_interval +} + /// Get follow symlinks flag from environment or default #[cfg(not(test))] pub fn get_follow_symlinks() -> bool { @@ -462,6 +481,8 @@ pub struct HybridStrategyConfig { pub write_frequency_threshold: usize, /// Fast update threshold pub fast_update_threshold: Duration, + /// Metrics logging interval + pub metrics_interval: Duration, /// Enable smart update pub enable_smart_update: bool, /// Enable write trigger @@ -475,6 +496,7 @@ impl Default for HybridStrategyConfig { write_trigger_delay: get_write_trigger_delay(), write_frequency_threshold: get_write_frequency_threshold(), fast_update_threshold: get_fast_update_threshold(), + metrics_interval: get_metrics_interval(), enable_smart_update: true, enable_write_trigger: true, } @@ -877,6 +899,35 @@ impl HybridCapacityManager { pub async fn refresh_in_progress(&self) -> bool { self.refresh_state.lock().await.running } + + /// Log capacity runtime summary for observability. + async fn log_runtime_summary(&self) { + let cached = self.get_capacity().await; + let recent_write_frequency = self.get_write_frequency().await; + let dirty_disks = self.get_dirty_disks().await; + let refresh_running = self.refresh_in_progress().await; + + if let Some(cached) = cached { + info!( + total_used = cached.total_used, + file_count = cached.file_count, + estimated = cached.is_estimated, + source = ?cached.source, + cache_age_secs = cached.last_update.elapsed().as_secs(), + writes_per_minute = recent_write_frequency, + dirty_disk_count = dirty_disks.len(), + refresh_inflight = refresh_running, + "Capacity metrics summary" + ); + } else { + info!( + writes_per_minute = recent_write_frequency, + dirty_disk_count = dirty_disks.len(), + refresh_inflight = refresh_running, + "Capacity metrics summary (cache empty)" + ); + } + } } /// Global capacity manager instance @@ -909,23 +960,30 @@ pub fn create_isolated_manager(config: HybridStrategyConfig) -> Arc) { let manager = get_capacity_manager(); - let mut interval = manager.get_config().scheduled_update_interval; + let manager_for_refresh = manager.clone(); + let manager_for_metrics = manager.clone(); + let mut refresh_interval = manager.get_config().scheduled_update_interval; + let mut metrics_interval = manager.get_config().metrics_interval; // Prevent panic in tokio::time::interval when misconfigured to 0 - if interval.is_zero() { + if refresh_interval.is_zero() { warn!("RUSTFS_CAPACITY_SCHEDULED_INTERVAL is configured as 0; clamping to 1s to avoid panic"); - interval = Duration::from_secs(1); + refresh_interval = Duration::from_secs(1); + } + if metrics_interval.is_zero() { + warn!("RUSTFS_CAPACITY_METRICS_INTERVAL is configured as 0; clamping to 1s to avoid panic"); + metrics_interval = Duration::from_secs(1); } tokio::spawn(async move { - let mut timer = tokio::time::interval(interval); + let mut timer = tokio::time::interval(refresh_interval); loop { timer.tick().await; info!("Starting scheduled capacity update"); let start = Instant::now(); - let manager = manager.clone(); + let manager = manager_for_refresh.clone(); let disks = disks.clone(); let started = manager .clone() @@ -942,6 +1000,14 @@ pub async fn start_background_task(disks: Vec) { } } }); + + tokio::spawn(async move { + let mut timer = tokio::time::interval(metrics_interval); + loop { + timer.tick().await; + manager_for_metrics.log_runtime_summary().await; + } + }); } // ============================================================================ @@ -953,8 +1019,9 @@ mod tests { use super::*; use rustfs_common::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope}; use rustfs_config::{ - ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_SAMPLE_RATE, - ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY, + ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_METRICS_INTERVAL, + ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, + ENV_CAPACITY_WRITE_TRIGGER_DELAY, }; use serial_test::serial; use std::sync::Arc; @@ -1009,6 +1076,13 @@ mod tests { assert_eq!(rate, 200); } + #[test] + #[serial] + fn test_get_metrics_interval() { + let interval = get_metrics_interval(); + assert_eq!(interval, Duration::from_secs(600)); + } + #[test] #[serial] fn test_env_var_override_scheduled_interval() { @@ -1045,6 +1119,15 @@ mod tests { }); } + #[test] + #[serial] + fn test_env_var_override_metrics_interval() { + temp_env::with_var(ENV_CAPACITY_METRICS_INTERVAL, Some("90"), || { + let interval = get_metrics_interval(); + assert_eq!(interval, Duration::from_secs(90)); + }); + } + #[test] #[serial] fn test_env_var_override_max_files_threshold() { @@ -1214,6 +1297,7 @@ mod tests { write_trigger_delay: Duration::from_millis(50), write_frequency_threshold: 1, fast_update_threshold: Duration::from_millis(10), + metrics_interval: Duration::from_secs(600), enable_smart_update: true, enable_write_trigger: true, }); @@ -1244,6 +1328,7 @@ mod tests { write_trigger_delay: Duration::from_secs(60), write_frequency_threshold: 1, fast_update_threshold: Duration::from_millis(10), + metrics_interval: Duration::from_secs(600), enable_smart_update: true, enable_write_trigger: false, }); diff --git a/rustfs/src/capacity/mod.rs b/rustfs/src/capacity/mod.rs index 1bb9ae2aa..cf8dc9a29 100644 --- a/rustfs/src/capacity/mod.rs +++ b/rustfs/src/capacity/mod.rs @@ -30,6 +30,7 @@ //! - `RUSTFS_CAPACITY_MAX_FILES_THRESHOLD` - Max files before sampling (default: 200,000) //! - `RUSTFS_CAPACITY_STAT_TIMEOUT` - Stat operation timeout (default: 3s) //! - `RUSTFS_CAPACITY_SAMPLE_RATE` - Sampling rate for metrics (default: 200) +//! - `RUSTFS_CAPACITY_METRICS_INTERVAL` - Metrics summary logging interval (default: 600s) //! - `RUSTFS_CAPACITY_FOLLOW_SYMLINKS` - Follow symlinks during traversal (default: false) //! - `RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH` - Max symlink depth (default: 3) //! - `RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT` - Enable dynamic timeout (default: true) diff --git a/scripts/run_object_batch_bench.sh b/scripts/run_object_batch_bench.sh new file mode 100755 index 000000000..7ff320a54 --- /dev/null +++ b/scripts/run_object_batch_bench.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Batch object benchmark runner for warp/s3bench. +# Runs a fixed size matrix under the same concurrency and exports per-size logs + summary CSV. + +DEFAULT_SIZES="1KiB,4KiB,8KiB,16KiB,32KiB,100KiB,512KiB,1MiB,2MiB,5MiB,10MiB" + +TOOL="warp" +ENDPOINT="" +ACCESS_KEY="" +SECRET_KEY="" +BUCKET="rustfs-bench" +REGION="us-east-1" +CONCURRENCY=128 +DURATION="60s" +SAMPLES=20000 +SIZES="$DEFAULT_SIZES" +OUT_DIR="" +WARP_BIN="warp" +WARP_MODE="mixed" +S3BENCH_BIN="s3bench" +INSECURE=false +DRY_RUN=false +EXTRA_ARGS=() + +usage() { + cat <<'USAGE' +Usage: + scripts/run_object_batch_bench.sh --tool --endpoint \ + --access-key --secret-key [options] + +Required: + --tool warp | s3bench + --endpoint S3 endpoint + --access-key S3 access key + --secret-key S3 secret key + +Optional: + --bucket Bucket name (default: rustfs-bench) + --region Region (default: us-east-1) + --concurrency Concurrency for all sizes (default: 128) + --duration warp duration, e.g. 60s/2m (default: 60s) + --samples s3bench numSamples (default: 20000) + --sizes Comma-separated sizes (default: 1KiB..10MiB matrix) + --out-dir Output directory (default: target/bench/object-batch-) + --warp-bin warp binary (default: warp) + --warp-mode warp mode: get|put|mixed (default: mixed) + --s3bench-bin s3bench binary (default: s3bench) + --extra-args Extra args appended to tool command, quoted as one string + --insecure For TLS endpoints with self-signed certs + --dry-run Print commands only + -h, --help Show help + +Examples: + # warp + scripts/run_object_batch_bench.sh \ + --tool warp --endpoint http://127.0.0.1:9000 \ + --access-key minioadmin --secret-key minioadmin \ + --bucket bench-obj --concurrency 128 --duration 90s --warp-mode get + + # s3bench + scripts/run_object_batch_bench.sh \ + --tool s3bench --endpoint http://127.0.0.1:9000 \ + --access-key minioadmin --secret-key minioadmin \ + --bucket bench-obj --concurrency 128 --samples 50000 +USAGE +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "ERROR: command not found: $1" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --tool) TOOL="$2"; shift 2 ;; + --endpoint) ENDPOINT="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --bucket) BUCKET="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --samples) SAMPLES="$2"; shift 2 ;; + --sizes) SIZES="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --warp-mode) WARP_MODE="$2"; shift 2 ;; + --s3bench-bin) S3BENCH_BIN="$2"; shift 2 ;; + --extra-args) + # shellcheck disable=SC2206 + EXTRA_ARGS=($2) + shift 2 + ;; + --insecure) INSECURE=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +validate_args() { + if [[ "$TOOL" != "warp" && "$TOOL" != "s3bench" ]]; then + echo "ERROR: --tool must be warp or s3bench" >&2 + exit 1 + fi + if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then + echo "ERROR: --endpoint/--access-key/--secret-key are required" >&2 + exit 1 + fi + if ! [[ "$CONCURRENCY" =~ ^[0-9]+$ ]] || [[ "$CONCURRENCY" -le 0 ]]; then + echo "ERROR: --concurrency must be a positive integer" >&2 + exit 1 + fi + if [[ "$TOOL" == "s3bench" ]]; then + if ! [[ "$SAMPLES" =~ ^[0-9]+$ ]] || [[ "$SAMPLES" -le 0 ]]; then + echo "ERROR: --samples must be a positive integer" >&2 + exit 1 + fi + fi +} + +setup_output() { + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="target/bench/object-batch-$(date +%Y%m%d-%H%M%S)" + fi + mkdir -p "$OUT_DIR" + SUMMARY_CSV="$OUT_DIR/summary.csv" + echo "size,tool,concurrency,status,throughput,requests_per_sec,avg_latency,log_file" > "$SUMMARY_CSV" +} + +extract_value() { + local pattern="$1" + local file="$2" + rg -o "$pattern" "$file" | head -n1 | sed -E "s/$pattern/\\1/" || true +} + +collect_metrics() { + local log_file="$1" + local throughput reqps latency + throughput="$(extract_value '([0-9]+(\\.[0-9]+)?\\s*(GiB/s|MiB/s|MB/s|KB/s))' "$log_file")" + reqps="$(extract_value '([0-9]+(\\.[0-9]+)?\\s*(req/s|ops/s|requests/s))' "$log_file")" + latency="$(extract_value '([0-9]+(\\.[0-9]+)?\\s*(ms|us|µs|s))(\\s*(avg|mean))?' "$log_file")" + echo "${throughput:-N/A},${reqps:-N/A},${latency:-N/A}" +} + +run_one() { + local size="$1" + local log_file="$OUT_DIR/${TOOL}_${size}.log" + local status="ok" + + echo "==== [$TOOL] size=$size concurrency=$CONCURRENCY ====" + + if [[ "$TOOL" == "warp" ]]; then + local cmd=( + "$WARP_BIN" "$WARP_MODE" + "--host" "$ENDPOINT" + "--access-key" "$ACCESS_KEY" + "--secret-key" "$SECRET_KEY" + "--bucket" "$BUCKET" + "--obj.size" "$size" + "--concurrent" "$CONCURRENCY" + "--duration" "$DURATION" + "--region" "$REGION" + ) + if [[ "$INSECURE" == "true" ]]; then + cmd+=("--insecure") + fi + cmd+=("${EXTRA_ARGS[@]}") + + if [[ "$DRY_RUN" == "true" ]]; then + printf '[DRY-RUN] %q ' "${cmd[@]}" + printf '\n' + echo "size=$size tool=$TOOL dry_run" > "$log_file" + else + if ! "${cmd[@]}" 2>&1 | tee "$log_file"; then + status="failed" + fi + fi + else + local cmd=( + "$S3BENCH_BIN" + "-accessKey=$ACCESS_KEY" + "-secretKey=$SECRET_KEY" + "-bucket=$BUCKET" + "-endpoint=$ENDPOINT" + "-region=$REGION" + "-numClients=$CONCURRENCY" + "-numSamples=$SAMPLES" + "-objectSize=$size" + ) + if [[ "$INSECURE" == "true" ]]; then + cmd+=("-insecure") + fi + cmd+=("${EXTRA_ARGS[@]}") + + if [[ "$DRY_RUN" == "true" ]]; then + printf '[DRY-RUN] %q ' "${cmd[@]}" + printf '\n' + echo "size=$size tool=$TOOL dry_run" > "$log_file" + else + if ! "${cmd[@]}" 2>&1 | tee "$log_file"; then + status="failed" + fi + fi + fi + + local metrics throughput reqps latency + metrics="$(collect_metrics "$log_file")" + throughput="$(echo "$metrics" | cut -d',' -f1)" + reqps="$(echo "$metrics" | cut -d',' -f2)" + latency="$(echo "$metrics" | cut -d',' -f3)" + + echo "$size,$TOOL,$CONCURRENCY,$status,$throughput,$reqps,$latency,$log_file" >> "$SUMMARY_CSV" +} + +main() { + parse_args "$@" + validate_args + require_cmd rg + if [[ "$TOOL" == "warp" ]]; then + require_cmd "$WARP_BIN" + else + require_cmd "$S3BENCH_BIN" + fi + + setup_output + + echo "Output dir: $OUT_DIR" + echo "Tool: $TOOL" + echo "Sizes: $SIZES" + echo "Concurrency: $CONCURRENCY" + + IFS=',' read -r -a size_arr <<< "$SIZES" + for raw_size in "${size_arr[@]}"; do + size="$(echo "$raw_size" | xargs)" + if [[ -z "$size" ]]; then + continue + fi + run_one "$size" + done + + echo + echo "Done. Summary:" + cat "$SUMMARY_CSV" +} + +main "$@" + diff --git a/scripts/run_object_batch_bench_abc.sh b/scripts/run_object_batch_bench_abc.sh new file mode 100755 index 000000000..c0707aa50 --- /dev/null +++ b/scripts/run_object_batch_bench_abc.sh @@ -0,0 +1,380 @@ +#!/usr/bin/env bash +set -euo pipefail + +# One-click controller: +# - Switches RUSTFS_CAPACITY_* and RUSTFS_OBJECT_* by profile A/B/C +# - Calls scripts/run_object_batch_bench_enhanced.sh for each profile +# - Supports optional "apply command" hook to reload/restart RustFS per profile + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENHANCED_SCRIPT="$SCRIPT_DIR/run_object_batch_bench_enhanced.sh" + +GROUP="all" # all|A|B|C +ENDPOINT="" +ACCESS_KEY="" +SECRET_KEY="" +BUCKET="rustfs-bench" +REGION="us-east-1" +TOOL="warp" +CONCURRENCY=128 +ROUNDS=3 +RETRY_PER_ROUND=2 +RETRY_SLEEP_SECS=2 +INSECURE=false +DRY_RUN=false +OUT_ROOT="" +BASELINE_ROOT="" + +# tool-specific +WARP_BIN="warp" +WARP_MODE="mixed" +DURATION="60s" +S3BENCH_BIN="s3bench" +SAMPLES=20000 + +# optional hooks +APPLY_CMD="" +APPLY_WAIT_SECS=20 + +EXTRA_ARGS=() + +usage() { + cat <<'USAGE' +Usage: + scripts/run_object_batch_bench_abc.sh \ + --tool --endpoint --access-key --secret-key [options] + +Required: + --tool warp | s3bench + --endpoint S3 endpoint + --access-key S3 access key + --secret-key S3 secret key + +Core options: + --group all|A|B|C (default: all) + --bucket Bucket name (default: rustfs-bench) + --region Region (default: us-east-1) + --concurrency Default 128 + --rounds Default 3 + --retry-per-round Default 2 + --retry-sleep-secs Default 2 + --out-root Default target/bench/object-batch-abc- + --baseline-root If set, use //median_summary.csv + --insecure Allow insecure TLS + --dry-run Print commands without execution + +Warp options: + --warp-bin Default: warp + --warp-mode get|put|mixed (default: mixed) + --duration Default: 60s + +s3bench options: + --s3bench-bin Default: s3bench + --samples Default: 20000 + +Hooks: + --apply-cmd Optional command to apply/restart RustFS after profile env switch. + Runs via: eval "$APPLY_CMD" + --apply-wait-secs Wait time after apply cmd (default: 20) + +Extra: + --extra-args Extra args passed to enhanced script, quoted as one string + -h, --help Show this help + +Examples: + scripts/run_object_batch_bench_abc.sh \ + --tool warp --endpoint http://127.0.0.1:9000 \ + --access-key minioadmin --secret-key minioadmin \ + --bucket bench-obj --group all --duration 90s + + scripts/run_object_batch_bench_abc.sh \ + --tool s3bench --endpoint http://127.0.0.1:9000 \ + --access-key minioadmin --secret-key minioadmin \ + --group B --samples 50000 --apply-cmd "bash scripts/run.capacity-object.lab.sh" +USAGE +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "ERROR: command not found: $1" >&2 + exit 1 + fi +} + +validate_positive_int() { + local v="$1" + local n="$2" + if ! [[ "$v" =~ ^[0-9]+$ ]] || [[ "$v" -le 0 ]]; then + echo "ERROR: $n must be a positive integer, got: $v" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --tool) TOOL="$2"; shift 2 ;; + --endpoint) ENDPOINT="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --group) GROUP="$2"; shift 2 ;; + --bucket) BUCKET="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --rounds) ROUNDS="$2"; shift 2 ;; + --retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;; + --retry-sleep-secs) RETRY_SLEEP_SECS="$2"; shift 2 ;; + --out-root) OUT_ROOT="$2"; shift 2 ;; + --baseline-root) BASELINE_ROOT="$2"; shift 2 ;; + --insecure) INSECURE=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --warp-mode) WARP_MODE="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --s3bench-bin) S3BENCH_BIN="$2"; shift 2 ;; + --samples) SAMPLES="$2"; shift 2 ;; + --apply-cmd) APPLY_CMD="$2"; shift 2 ;; + --apply-wait-secs) APPLY_WAIT_SECS="$2"; shift 2 ;; + --extra-args) + # shellcheck disable=SC2206 + EXTRA_ARGS=($2) + shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +validate_args() { + if [[ "$TOOL" != "warp" && "$TOOL" != "s3bench" ]]; then + echo "ERROR: --tool must be warp or s3bench" >&2 + exit 1 + fi + case "$GROUP" in + all|A|B|C) ;; + *) echo "ERROR: --group must be all|A|B|C" >&2; exit 1 ;; + esac + if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then + echo "ERROR: --endpoint/--access-key/--secret-key are required" >&2 + exit 1 + fi + validate_positive_int "$CONCURRENCY" "--concurrency" + validate_positive_int "$ROUNDS" "--rounds" + validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" + validate_positive_int "$RETRY_SLEEP_SECS" "--retry-sleep-secs" + validate_positive_int "$APPLY_WAIT_SECS" "--apply-wait-secs" + if [[ "$TOOL" == "s3bench" ]]; then + validate_positive_int "$SAMPLES" "--samples" + fi +} + +setup_out_root() { + if [[ -z "$OUT_ROOT" ]]; then + OUT_ROOT="target/bench/object-batch-abc-$(date +%Y%m%d-%H%M%S)" + fi + mkdir -p "$OUT_ROOT" +} + +apply_capacity_common() { + export RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300 + export RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY=8 + export RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=14 + export RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD=45 + export RUSTFS_CAPACITY_MAX_FILES_THRESHOLD=1000000 + export RUSTFS_CAPACITY_STAT_TIMEOUT=5 + export RUSTFS_CAPACITY_SAMPLE_RATE=100 + export RUSTFS_CAPACITY_METRICS_INTERVAL=120 +} + +apply_object_profile_A() { + export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=128 + export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=2097152 + export RUSTFS_OBJECT_GET_TIMEOUT=18 + export RUSTFS_OBJECT_DISK_READ_TIMEOUT=6 + export RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=4 + export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true + export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true + export RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12 + export RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6 +} + +apply_object_profile_B() { + export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=112 + export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=4194304 + export RUSTFS_OBJECT_GET_TIMEOUT=30 + export RUSTFS_OBJECT_DISK_READ_TIMEOUT=10 + export RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=5 + export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true + export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true + export RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12 + export RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6 +} + +apply_object_profile_C() { + export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=72 + export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 + export RUSTFS_OBJECT_GET_TIMEOUT=50 + export RUSTFS_OBJECT_DISK_READ_TIMEOUT=14 + export RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=6 + export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true + export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true + export RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12 + export RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6 +} + +sizes_for_group() { + case "$1" in + A) echo "1KiB,4KiB,8KiB,16KiB,32KiB,100KiB" ;; + B) echo "100KiB,512KiB,1MiB,2MiB" ;; + C) echo "2MiB,5MiB,10MiB" ;; + *) echo "" ;; + esac +} + +run_apply_hook_if_needed() { + local group="$1" + if [[ -z "$APPLY_CMD" ]]; then + return + fi + echo "[${group}] running apply command..." + if [[ "$DRY_RUN" == "true" ]]; then + echo "[DRY-RUN] eval \"$APPLY_CMD\"" + echo "[DRY-RUN] sleep $APPLY_WAIT_SECS" + else + eval "$APPLY_CMD" + echo "[${group}] waiting ${APPLY_WAIT_SECS}s for service readiness..." + sleep "$APPLY_WAIT_SECS" + fi +} + +write_env_snapshot() { + local out_file="$1" + cat > "$out_file" <&2; exit 1 ;; + esac + + sizes="$(sizes_for_group "$g")" + out_dir="$OUT_ROOT/$g" + mkdir -p "$out_dir" + write_env_snapshot "$out_dir/env_snapshot.env" + + run_apply_hook_if_needed "$g" + + baseline_csv="" + if [[ -n "$BASELINE_ROOT" && -f "$BASELINE_ROOT/$g/median_summary.csv" ]]; then + baseline_csv="$BASELINE_ROOT/$g/median_summary.csv" + fi + + local cmd=( + "$ENHANCED_SCRIPT" + "--tool" "$TOOL" + "--endpoint" "$ENDPOINT" + "--access-key" "$ACCESS_KEY" + "--secret-key" "$SECRET_KEY" + "--bucket" "$BUCKET" + "--region" "$REGION" + "--concurrency" "$CONCURRENCY" + "--sizes" "$sizes" + "--rounds" "$ROUNDS" + "--retry-per-round" "$RETRY_PER_ROUND" + "--retry-sleep-secs" "$RETRY_SLEEP_SECS" + "--out-dir" "$out_dir" + ) + + if [[ -n "$baseline_csv" ]]; then + cmd+=("--baseline-csv" "$baseline_csv") + fi + if [[ "$INSECURE" == "true" ]]; then + cmd+=("--insecure") + fi + if [[ "$DRY_RUN" == "true" ]]; then + cmd+=("--dry-run") + fi + + if [[ "$TOOL" == "warp" ]]; then + cmd+=("--warp-bin" "$WARP_BIN" "--warp-mode" "$WARP_MODE" "--duration" "$DURATION") + else + cmd+=("--s3bench-bin" "$S3BENCH_BIN" "--samples" "$SAMPLES") + fi + if [[ "${#EXTRA_ARGS[@]}" -gt 0 ]]; then + local joined + joined="$(printf '%s ' "${EXTRA_ARGS[@]}" | sed 's/[[:space:]]*$//')" + cmd+=("--extra-args" "$joined") + fi + + echo + echo "===== Running group ${g} =====" + echo "Sizes: $sizes" + echo "Output: $out_dir" + if [[ "$DRY_RUN" == "true" ]]; then + printf '[DRY-RUN] %q ' "${cmd[@]}" + printf '\n' + else + "${cmd[@]}" + fi +} + +main() { + parse_args "$@" + validate_args + require_cmd awk + require_cmd sed + if [[ ! -x "$ENHANCED_SCRIPT" ]]; then + echo "ERROR: enhanced script missing or not executable: $ENHANCED_SCRIPT" >&2 + exit 1 + fi + setup_out_root + + echo "Controller output root: $OUT_ROOT" + echo "Tool=$TOOL Group=$GROUP Concurrency=$CONCURRENCY Rounds=$ROUNDS" + + case "$GROUP" in + all) + run_group A + run_group B + run_group C + ;; + A|B|C) + run_group "$GROUP" + ;; + esac + + echo + echo "Done. Group outputs are under: $OUT_ROOT" +} + +main "$@" diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh new file mode 100755 index 000000000..e72b71ba7 --- /dev/null +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -0,0 +1,487 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Enhanced batch object benchmark runner for warp/s3bench: +# - Multi-round execution (default 3 rounds) +# - Retry failed round attempts automatically +# - Median aggregation per object size +# - Optional baseline CSV comparison + +DEFAULT_SIZES="1KiB,4KiB,8KiB,16KiB,32KiB,100KiB,512KiB,1MiB,2MiB,5MiB,10MiB" + +TOOL="warp" +ENDPOINT="" +ACCESS_KEY="" +SECRET_KEY="" +BUCKET="rustfs-bench" +REGION="us-east-1" +CONCURRENCY=128 +SIZES="$DEFAULT_SIZES" +OUT_DIR="" +INSECURE=false +DRY_RUN=false + +# warp options +WARP_BIN="warp" +WARP_MODE="mixed" +DURATION="60s" + +# s3bench options +S3BENCH_BIN="s3bench" +SAMPLES=20000 + +# enhancement options +ROUNDS=3 +RETRY_PER_ROUND=2 +RETRY_SLEEP_SECS=2 +BASELINE_CSV="" +EXTRA_ARGS=() + +usage() { + cat <<'USAGE' +Usage: + scripts/run_object_batch_bench_enhanced.sh --tool --endpoint \ + --access-key --secret-key [options] + +Required: + --tool warp | s3bench + --endpoint S3 endpoint + --access-key S3 access key + --secret-key S3 secret key + +Core options: + --bucket Bucket name (default: rustfs-bench) + --region Region (default: us-east-1) + --concurrency Concurrency for all sizes (default: 128) + --sizes Comma-separated sizes (default: 1KiB..10MiB matrix) + --out-dir Output directory (default: target/bench/object-batch-enhanced-) + --insecure Allow insecure TLS + --dry-run Print commands only, do not execute + +Warp options: + --warp-bin warp binary (default: warp) + --warp-mode get|put|mixed (default: mixed) + --duration e.g. 60s/2m (default: 60s) + +s3bench options: + --s3bench-bin s3bench binary (default: s3bench) + --samples numSamples (default: 20000) + +Enhanced options: + --rounds Benchmark rounds per size (default: 3) + --retry-per-round Retry count per failed round (default: 2) + --retry-sleep-secs Sleep seconds between retries (default: 2) + --baseline-csv Baseline median CSV to compare + --extra-args Extra args appended to tool command, quoted as one string + +Output files: + round_results.csv One row per round attempt (with retry trace) + median_summary.csv Median metrics per object size + baseline_compare.csv Delta vs baseline (if --baseline-csv is set) + +Example: + scripts/run_object_batch_bench_enhanced.sh \ + --tool warp --endpoint http://127.0.0.1:9000 \ + --access-key minioadmin --secret-key minioadmin \ + --bucket bench-obj --concurrency 128 --duration 90s \ + --rounds 3 --retry-per-round 2 --baseline-csv old/median_summary.csv +USAGE +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "ERROR: command not found: $1" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --tool) TOOL="$2"; shift 2 ;; + --endpoint) ENDPOINT="$2"; shift 2 ;; + --access-key) ACCESS_KEY="$2"; shift 2 ;; + --secret-key) SECRET_KEY="$2"; shift 2 ;; + --bucket) BUCKET="$2"; shift 2 ;; + --region) REGION="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --sizes) SIZES="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --insecure) INSECURE=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --warp-bin) WARP_BIN="$2"; shift 2 ;; + --warp-mode) WARP_MODE="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --s3bench-bin) S3BENCH_BIN="$2"; shift 2 ;; + --samples) SAMPLES="$2"; shift 2 ;; + --rounds) ROUNDS="$2"; shift 2 ;; + --retry-per-round) RETRY_PER_ROUND="$2"; shift 2 ;; + --retry-sleep-secs) RETRY_SLEEP_SECS="$2"; shift 2 ;; + --baseline-csv) BASELINE_CSV="$2"; shift 2 ;; + --extra-args) + # shellcheck disable=SC2206 + EXTRA_ARGS=($2) + shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +validate_positive_int() { + local v="$1" + local n="$2" + if ! [[ "$v" =~ ^[0-9]+$ ]] || [[ "$v" -le 0 ]]; then + echo "ERROR: $n must be a positive integer, got: $v" >&2 + exit 1 + fi +} + +validate_args() { + if [[ "$TOOL" != "warp" && "$TOOL" != "s3bench" ]]; then + echo "ERROR: --tool must be warp or s3bench" >&2 + exit 1 + fi + if [[ -z "$ENDPOINT" || -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then + echo "ERROR: --endpoint/--access-key/--secret-key are required" >&2 + exit 1 + fi + validate_positive_int "$CONCURRENCY" "--concurrency" + validate_positive_int "$ROUNDS" "--rounds" + validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round" + validate_positive_int "$RETRY_SLEEP_SECS" "--retry-sleep-secs" + if [[ "$TOOL" == "s3bench" ]]; then + validate_positive_int "$SAMPLES" "--samples" + fi + if [[ -n "$BASELINE_CSV" && ! -f "$BASELINE_CSV" ]]; then + echo "ERROR: --baseline-csv does not exist: $BASELINE_CSV" >&2 + exit 1 + fi +} + +setup_output() { + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="target/bench/object-batch-enhanced-$(date +%Y%m%d-%H%M%S)" + fi + mkdir -p "$OUT_DIR/logs" + + ROUND_CSV="$OUT_DIR/round_results.csv" + MEDIAN_CSV="$OUT_DIR/median_summary.csv" + COMPARE_CSV="$OUT_DIR/baseline_compare.csv" + + echo "size,tool,round,attempt,concurrency,status,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file" > "$ROUND_CSV" + echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms" > "$MEDIAN_CSV" +} + +trim() { + echo "$1" | awk '{$1=$1;print}' +} + +to_bps() { + local human="$1" + if [[ "$human" == "N/A" || -z "$human" ]]; then + echo "N/A" + return + fi + awk -v v="$human" ' + function abs(x){return x<0?-x:x} + BEGIN{ + if (match(v, /^([0-9]+(\.[0-9]+)?)\s*(GiB\/s|MiB\/s|KiB\/s|GB\/s|MB\/s|KB\/s|B\/s)$/, m)) { + n=m[1]; u=m[3]; + if (u=="GiB/s") f=1024*1024*1024; + else if (u=="MiB/s") f=1024*1024; + else if (u=="KiB/s") f=1024; + else if (u=="GB/s") f=1000*1000*1000; + else if (u=="MB/s") f=1000*1000; + else if (u=="KB/s") f=1000; + else f=1; + printf "%.6f\n", n*f; + } else { + print "N/A"; + } + }' +} + +to_ms() { + local human="$1" + if [[ "$human" == "N/A" || -z "$human" ]]; then + echo "N/A" + return + fi + awk -v v="$human" ' + BEGIN{ + if (match(v, /^([0-9]+(\.[0-9]+)?)\s*(ms|us|µs|s)$/, m)) { + n=m[1]; u=m[3]; + if (u=="s") f=1000; + else if (u=="ms") f=1; + else f=0.001; + printf "%.6f\n", n*f; + } else { + print "N/A"; + } + }' +} + +extract_first() { + local regex="$1" + local file="$2" + rg -o "$regex" "$file" | head -n1 || true +} + +extract_metrics() { + local log_file="$1" + + local throughput reqps latency + throughput="$(extract_first '[0-9]+(\.[0-9]+)?\s*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")" + reqps="$(extract_first '[0-9]+(\.[0-9]+)?\s*(req/s|ops/s|requests/s)' "$log_file")" + latency="$(extract_first '[0-9]+(\.[0-9]+)?\s*(ms|us|µs|s)\s*(avg|mean)' "$log_file")" + + if [[ -z "$latency" ]]; then + latency="$(extract_first '[0-9]+(\.[0-9]+)?\s*(ms|us|µs|s)' "$log_file")" + fi + + throughput="$(trim "${throughput:-N/A}")" + reqps="$(trim "${reqps:-N/A}")" + latency="$(trim "${latency:-N/A}")" + + # Keep only " " for latency if suffix avg/mean exists. + latency="$(echo "$latency" | awk '{print $1" "$2}')" + reqps_num="$(echo "$reqps" | awk '{print $1}')" + + echo "$throughput,${reqps_num:-N/A},$latency" +} + +median_from_numbers() { + local values="$1" + local count + count="$(printf '%s\n' "$values" | awk 'NF{c++} END{print c+0}')" + if [[ "$count" -eq 0 ]]; then + echo "N/A" + return + fi + + printf '%s\n' "$values" | awk 'NF' | sort -n | awk ' + {a[NR]=$1} + END{ + n=NR + if (n==0) { print "N/A"; exit } + if (n%2==1) { + printf "%.6f\n", a[(n+1)/2] + } else { + printf "%.6f\n", (a[n/2]+a[n/2+1])/2 + } + }' +} + +run_one_attempt() { + local size="$1" + local round="$2" + local attempt="$3" + local log_file="$OUT_DIR/logs/${TOOL}_${size}_r${round}_a${attempt}.log" + local status="ok" + + if [[ "$TOOL" == "warp" ]]; then + local cmd=( + "$WARP_BIN" "$WARP_MODE" + "--host" "$ENDPOINT" + "--access-key" "$ACCESS_KEY" + "--secret-key" "$SECRET_KEY" + "--bucket" "$BUCKET" + "--obj.size" "$size" + "--concurrent" "$CONCURRENCY" + "--duration" "$DURATION" + "--region" "$REGION" + ) + if [[ "$INSECURE" == "true" ]]; then + cmd+=("--insecure") + fi + cmd+=("${EXTRA_ARGS[@]}") + + if [[ "$DRY_RUN" == "true" ]]; then + printf '[DRY-RUN] %q ' "${cmd[@]}" + printf '\n' + echo "dry run" > "$log_file" + else + if ! "${cmd[@]}" 2>&1 | tee "$log_file"; then + status="failed" + fi + fi + else + local cmd=( + "$S3BENCH_BIN" + "-accessKey=$ACCESS_KEY" + "-secretKey=$SECRET_KEY" + "-bucket=$BUCKET" + "-endpoint=$ENDPOINT" + "-region=$REGION" + "-numClients=$CONCURRENCY" + "-numSamples=$SAMPLES" + "-objectSize=$size" + ) + if [[ "$INSECURE" == "true" ]]; then + cmd+=("-insecure") + fi + cmd+=("${EXTRA_ARGS[@]}") + + if [[ "$DRY_RUN" == "true" ]]; then + printf '[DRY-RUN] %q ' "${cmd[@]}" + printf '\n' + echo "dry run" > "$log_file" + else + if ! "${cmd[@]}" 2>&1 | tee "$log_file"; then + status="failed" + fi + fi + fi + + local metrics throughput_human reqps latency_human throughput_bps latency_ms + metrics="$(extract_metrics "$log_file")" + throughput_human="$(echo "$metrics" | cut -d',' -f1)" + reqps="$(echo "$metrics" | cut -d',' -f2)" + latency_human="$(echo "$metrics" | cut -d',' -f3)" + throughput_bps="$(to_bps "$throughput_human")" + latency_ms="$(to_ms "$latency_human")" + + if [[ "$DRY_RUN" != "true" && "$status" == "ok" ]]; then + if [[ "$throughput_bps" == "N/A" && "$reqps" == "N/A" ]]; then + status="failed" + fi + fi + + echo "$size,$TOOL,$round,$attempt,$CONCURRENCY,$status,$throughput_human,$throughput_bps,$reqps,$latency_human,$latency_ms,$log_file" >> "$ROUND_CSV" + echo "$status" +} + +run_size() { + local size="$1" + local round success attempt rc + + for ((round=1; round<=ROUNDS; round++)); do + success="no" + for ((attempt=1; attempt<=RETRY_PER_ROUND+1; attempt++)); do + echo "==== size=$size round=$round attempt=$attempt/${RETRY_PER_ROUND+1} ====" + rc="$(run_one_attempt "$size" "$round" "$attempt")" + if [[ "$rc" == "ok" || "$DRY_RUN" == "true" ]]; then + success="yes" + break + fi + if (( attempt < RETRY_PER_ROUND+1 )); then + echo "Round failed, retry in ${RETRY_SLEEP_SECS}s..." + sleep "$RETRY_SLEEP_SECS" + fi + done + + if [[ "$success" == "no" ]]; then + echo "WARN: size=$size round=$round failed after retries." + fi + done +} + +build_median_summary() { + local sizes_arr size + IFS=',' read -r -a sizes_arr <<< "$SIZES" + + for raw in "${sizes_arr[@]}"; do + size="$(trim "$raw")" + [[ -z "$size" ]] && continue + + local ok_rounds fail_rounds t_vals r_vals l_vals + ok_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" {c++} END{print c+0}' "$ROUND_CSV")" + fail_rounds="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6!="ok" {c++} END{print c+0}' "$ROUND_CSV")" + + t_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $8!="N/A" {print $8}' "$ROUND_CSV")" + r_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $9!="N/A" {print $9}' "$ROUND_CSV")" + l_vals="$(awk -F',' -v s="$size" 'NR>1 && $1==s && $6=="ok" && $11!="N/A" {print $11}' "$ROUND_CSV")" + + local m_t m_r m_l + m_t="$(median_from_numbers "$t_vals")" + m_r="$(median_from_numbers "$r_vals")" + m_l="$(median_from_numbers "$l_vals")" + + echo "$size,$TOOL,$CONCURRENCY,$ok_rounds,$fail_rounds,$m_t,$m_r,$m_l" >> "$MEDIAN_CSV" + done +} + +compare_baseline() { + if [[ -z "$BASELINE_CSV" ]]; then + return + fi + + echo "size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct" > "$COMPARE_CSV" + + awk -F',' ' + NR==FNR { + if (FNR==1) next + key=$1 + b_req[key]=$7 + b_lat[key]=$8 + b_thr[key]=$6 + next + } + FNR==1 {next} + { + key=$1 + n_thr=$6; n_req=$7; n_lat=$8 + br=(key in b_req)?b_req[key]:"N/A" + bl=(key in b_lat)?b_lat[key]:"N/A" + bt=(key in b_thr)?b_thr[key]:"N/A" + + dr="N/A"; dl="N/A"; dt="N/A" + if (br!="N/A" && n_req!="N/A" && br+0!=0) dr=sprintf("%.2f", ((n_req-br)/br)*100) + if (bl!="N/A" && n_lat!="N/A" && bl+0!=0) dl=sprintf("%.2f", ((n_lat-bl)/bl)*100) + if (bt!="N/A" && n_thr!="N/A" && bt+0!=0) dt=sprintf("%.2f", ((n_thr-bt)/bt)*100) + + print key "," $2 "," $3 "," n_req "," br "," dr "," n_lat "," bl "," dl "," n_thr "," bt "," dt + } + ' "$BASELINE_CSV" "$MEDIAN_CSV" >> "$COMPARE_CSV" +} + +main() { + parse_args "$@" + validate_args + require_cmd rg + require_cmd awk + require_cmd sort + if [[ "$TOOL" == "warp" ]]; then + require_cmd "$WARP_BIN" + else + require_cmd "$S3BENCH_BIN" + fi + + setup_output + + echo "Output dir: $OUT_DIR" + echo "Tool: $TOOL" + echo "Sizes: $SIZES" + echo "Concurrency: $CONCURRENCY" + echo "Rounds: $ROUNDS" + echo "Retry per round: $RETRY_PER_ROUND" + + IFS=',' read -r -a size_arr <<< "$SIZES" + for raw in "${size_arr[@]}"; do + size="$(trim "$raw")" + [[ -z "$size" ]] && continue + run_size "$size" + done + + build_median_summary + compare_baseline + + echo + echo "=== Median Summary ===" + cat "$MEDIAN_CSV" + + if [[ -n "$BASELINE_CSV" ]]; then + echo + echo "=== Baseline Compare ===" + cat "$COMPARE_CSV" + fi +} + +main "$@" +