mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-04 20:37:43 +00:00
feat(storage): extend PUT path tuning and observability (#3829)
* feat(storage): add multipart put stage metrics * feat(scripts): add multipart put focus runner * docs(operations): add multipart put server-path guides * chore(scripts): add local rustfs restart helper * docs(observability): add local metrics backend guide * docs(observability): add localized multipart guides * fix(ecstore): validate multipart batching path * feat(obs): add erasure encode overlap metrics * docs(ops): update overlap retest summary * docs(ops): add batchblocks retest matrix * docs(ops): extend overlap candidate summary * docs(ops): capture 8-run overlap summary * feat(storage): switch rename_data to msgpack map * test(storage): add rename_data payload checks * feat(object): add zero_copy_eager put path * docs(ops): add zero_copy_eager put guide * docs(ops): add deeper zero-copy next steps
This commit is contained in:
houseme
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PIDFILE=/private/tmp/issue708-single-node-multidisk/rustfs.pid
|
||||
LOGFILE=/private/tmp/issue708-single-node-multidisk/rustfs.log
|
||||
RUSTFS_CMD_PATTERN='target/debug/rustfs server /private/tmp/issue708-single-node-multidisk/d1 /private/tmp/issue708-single-node-multidisk/d2 /private/tmp/issue708-single-node-multidisk/d3 /private/tmp/issue708-single-node-multidisk/d4'
|
||||
|
||||
mkdir -p /private/tmp/issue708-single-node-multidisk/{d1,d2,d3,d4}
|
||||
|
||||
if [[ -f "$PIDFILE" ]]; then
|
||||
old_pid="$(cat "$PIDFILE")"
|
||||
if kill -0 "$old_pid" >/dev/null 2>&1; then
|
||||
kill "$old_pid" >/dev/null 2>&1 || true
|
||||
sleep 2
|
||||
fi
|
||||
fi
|
||||
|
||||
lingering_pids="$(pgrep -f "$RUSTFS_CMD_PATTERN" || true)"
|
||||
if [[ -n "$lingering_pids" ]]; then
|
||||
while IFS= read -r lingering_pid; do
|
||||
[[ -z "$lingering_pid" ]] && continue
|
||||
kill "$lingering_pid" >/dev/null 2>&1 || true
|
||||
done <<< "$lingering_pids"
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
if [[ -n "${RUSTFS_TUNING_ENV_FILE:-}" && -f "${RUSTFS_TUNING_ENV_FILE:-}" ]]; then
|
||||
set -a
|
||||
source "$RUSTFS_TUNING_ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
launch_rustfs() {
|
||||
if command -v setsid >/dev/null 2>&1; then
|
||||
setsid env \
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
RUSTFS_ADDRESS=127.0.0.1:9000 \
|
||||
RUSTFS_ACCESS_KEY=rustfsadmin \
|
||||
RUSTFS_SECRET_KEY=rustfsadmin \
|
||||
RUSTFS_RPC_SECRET=rustfs-rpc-secret \
|
||||
RUSTFS_REGION=us-east-1 \
|
||||
RUSTFS_CONSOLE_ENABLE=false \
|
||||
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 \
|
||||
target/debug/rustfs server \
|
||||
/private/tmp/issue708-single-node-multidisk/d1 \
|
||||
/private/tmp/issue708-single-node-multidisk/d2 \
|
||||
/private/tmp/issue708-single-node-multidisk/d3 \
|
||||
/private/tmp/issue708-single-node-multidisk/d4 \
|
||||
</dev/null >"$LOGFILE" 2>&1 &
|
||||
else
|
||||
nohup env \
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
RUSTFS_ADDRESS=127.0.0.1:9000 \
|
||||
RUSTFS_ACCESS_KEY=rustfsadmin \
|
||||
RUSTFS_SECRET_KEY=rustfsadmin \
|
||||
RUSTFS_RPC_SECRET=rustfs-rpc-secret \
|
||||
RUSTFS_REGION=us-east-1 \
|
||||
RUSTFS_CONSOLE_ENABLE=false \
|
||||
RUSTFS_OBS_ENDPOINT=http://127.0.0.1:4318 \
|
||||
target/debug/rustfs server \
|
||||
/private/tmp/issue708-single-node-multidisk/d1 \
|
||||
/private/tmp/issue708-single-node-multidisk/d2 \
|
||||
/private/tmp/issue708-single-node-multidisk/d3 \
|
||||
/private/tmp/issue708-single-node-multidisk/d4 \
|
||||
</dev/null >"$LOGFILE" 2>&1 &
|
||||
fi
|
||||
}
|
||||
|
||||
launch_rustfs
|
||||
|
||||
new_pid=$!
|
||||
sleep 5
|
||||
|
||||
actual_pid="$(pgrep -f "$RUSTFS_CMD_PATTERN" | head -n 1 || true)"
|
||||
if [[ -z "$actual_pid" ]]; then
|
||||
echo "failed to locate restarted rustfs process; launcher pid was $new_pid" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "$actual_pid" > "$PIDFILE"
|
||||
curl -fsS http://127.0.0.1:9000/health >/dev/null
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
HOST="${HOST:-127.0.0.1:9000}"
|
||||
ACCESS_KEY="${ACCESS_KEY:-}"
|
||||
SECRET_KEY="${SECRET_KEY:-}"
|
||||
BUCKET_PREFIX="${BUCKET_PREFIX:-rustfs-gt1g-multipart}"
|
||||
LOOKUP="${LOOKUP:-path}"
|
||||
DURATION="${DURATION:-10m}"
|
||||
WARP_BIN="${WARP_BIN:-warp}"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/gt1g-multipart-put-$(date +%Y%m%d-%H%M%S)}"
|
||||
DRY_RUN=false
|
||||
|
||||
DEFAULT_PROFILES="1g-64m-pc4,1g-128m-pc4,2g-128m-pc4,2g-256m-pc4"
|
||||
PROFILES="${PROFILES:-$DEFAULT_PROFILES}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/run_gt1g_multipart_put_matrix.sh --access-key <ak> --secret-key <sk> [options]
|
||||
|
||||
Required:
|
||||
--access-key <ak>
|
||||
--secret-key <sk>
|
||||
|
||||
Options:
|
||||
--host <host:port> Default: 127.0.0.1:9000
|
||||
--bucket-prefix <prefix> Default: rustfs-gt1g-multipart
|
||||
--lookup <path|dns> Default: path
|
||||
--duration <dur> Default: 10m
|
||||
--profiles <csv> Default: 1g-64m-pc4,1g-128m-pc4,2g-128m-pc4,2g-256m-pc4
|
||||
--warp-bin <path> Default: warp
|
||||
--out-dir <dir> Default: target/bench/gt1g-multipart-put-<timestamp>
|
||||
--dry-run
|
||||
-h, --help
|
||||
|
||||
Built-in profiles:
|
||||
1g-64m-pc4 concurrent=4 parts=16 part.size=64MiB part.concurrent=4
|
||||
1g-128m-pc4 concurrent=4 parts=8 part.size=128MiB part.concurrent=4
|
||||
2g-128m-pc4 concurrent=4 parts=16 part.size=128MiB part.concurrent=4
|
||||
2g-256m-pc4 concurrent=4 parts=8 part.size=256MiB part.concurrent=4
|
||||
1g-64m-pc8 concurrent=4 parts=16 part.size=64MiB part.concurrent=8
|
||||
2g-128m-pc8 concurrent=4 parts=16 part.size=128MiB part.concurrent=8
|
||||
2g-256m-pc8 concurrent=4 parts=8 part.size=256MiB part.concurrent=8
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "ERROR: command not found: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
trim() {
|
||||
echo "$1" | awk '{$1=$1;print}'
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--host) HOST="$2"; shift 2 ;;
|
||||
--access-key) ACCESS_KEY="$2"; shift 2 ;;
|
||||
--secret-key) SECRET_KEY="$2"; shift 2 ;;
|
||||
--bucket-prefix) BUCKET_PREFIX="$2"; shift 2 ;;
|
||||
--lookup) LOOKUP="$2"; shift 2 ;;
|
||||
--duration) DURATION="$2"; shift 2 ;;
|
||||
--profiles) PROFILES="$2"; shift 2 ;;
|
||||
--warp-bin) WARP_BIN="$2"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
validate_args() {
|
||||
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
|
||||
echo "ERROR: --access-key and --secret-key are required" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
setup_output() {
|
||||
mkdir -p "$OUT_DIR/logs" "$OUT_DIR/benchdata"
|
||||
SUMMARY_CSV="$OUT_DIR/summary.csv"
|
||||
COMMANDS_TXT="$OUT_DIR/commands.txt"
|
||||
MANIFEST_TXT="$OUT_DIR/run_manifest.txt"
|
||||
|
||||
echo "profile,host,bucket,duration,concurrent,parts,part_size,part_concurrent,throughput_human,reqps,latency_human,log_file,benchdata_file,status" > "$SUMMARY_CSV"
|
||||
: > "$COMMANDS_TXT"
|
||||
}
|
||||
|
||||
write_manifest() {
|
||||
{
|
||||
echo "created_at=$(date +%Y-%m-%dT%H:%M:%S%z)"
|
||||
echo "host=${HOST}"
|
||||
echo "bucket_prefix=${BUCKET_PREFIX}"
|
||||
echo "lookup=${LOOKUP}"
|
||||
echo "duration=${DURATION}"
|
||||
echo "profiles=${PROFILES}"
|
||||
echo "warp_bin=${WARP_BIN}"
|
||||
echo "dry_run=${DRY_RUN}"
|
||||
echo "access_key=REDACTED"
|
||||
echo "secret_key=REDACTED"
|
||||
} > "$MANIFEST_TXT"
|
||||
}
|
||||
|
||||
profile_spec() {
|
||||
case "$1" in
|
||||
1g-64m-pc4) echo "4|16|64MiB|4" ;;
|
||||
1g-128m-pc4) echo "4|8|128MiB|4" ;;
|
||||
2g-128m-pc4) echo "4|16|128MiB|4" ;;
|
||||
2g-256m-pc4) echo "4|8|256MiB|4" ;;
|
||||
1g-64m-pc8) echo "4|16|64MiB|8" ;;
|
||||
2g-128m-pc8) echo "4|16|128MiB|8" ;;
|
||||
2g-256m-pc8) echo "4|8|256MiB|8" ;;
|
||||
*)
|
||||
echo "ERROR: unknown profile: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
extract_first() {
|
||||
local regex="$1"
|
||||
local file="$2"
|
||||
rg -o "$regex" "$file" | head -n1 || true
|
||||
}
|
||||
|
||||
extract_metrics() {
|
||||
local log_file="$1"
|
||||
local throughput reqps latency
|
||||
|
||||
throughput="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")"
|
||||
reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")"
|
||||
latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')"
|
||||
|
||||
throughput="$(trim "${throughput:-N/A}")"
|
||||
reqps="$(trim "${reqps:-N/A}")"
|
||||
latency="$(trim "${latency:-N/A}")"
|
||||
reqps="$(echo "$reqps" | awk '{print $1}')"
|
||||
|
||||
echo "${throughput},${reqps:-N/A},${latency}"
|
||||
}
|
||||
|
||||
run_profile() {
|
||||
local profile="$1"
|
||||
local spec concurrent parts part_size part_concurrent
|
||||
local bucket log_file benchdata_file status metrics throughput reqps latency
|
||||
|
||||
spec="$(profile_spec "$profile")"
|
||||
IFS='|' read -r concurrent parts part_size part_concurrent <<< "$spec"
|
||||
|
||||
bucket="$(echo "${BUCKET_PREFIX}-${profile}-$(date +%Y%m%d%H%M%S)" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/-{2,}/-/g; s/^-+//; s/-+$//')"
|
||||
bucket="${bucket:0:63}"
|
||||
log_file="$OUT_DIR/logs/${profile}.log"
|
||||
benchdata_file="$OUT_DIR/benchdata/${profile}.csv.zst"
|
||||
|
||||
local -a cmd=(
|
||||
"$WARP_BIN" multipart-put
|
||||
--host "$HOST"
|
||||
--access-key "$ACCESS_KEY"
|
||||
--secret-key "$SECRET_KEY"
|
||||
--bucket "$bucket"
|
||||
--lookup "$LOOKUP"
|
||||
--duration "$DURATION"
|
||||
--concurrent "$concurrent"
|
||||
--parts "$parts"
|
||||
--part.size "$part_size"
|
||||
--part.concurrent "$part_concurrent"
|
||||
--benchdata "$benchdata_file"
|
||||
--analyze.v
|
||||
--no-color
|
||||
)
|
||||
|
||||
printf '%q ' "${cmd[@]}" >> "$COMMANDS_TXT"
|
||||
printf '\n' >> "$COMMANDS_TXT"
|
||||
|
||||
status="ok"
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
printf '\n'
|
||||
: > "$log_file"
|
||||
else
|
||||
if ! "${cmd[@]}" >"$log_file" 2>&1; then
|
||||
status="failed"
|
||||
fi
|
||||
fi
|
||||
|
||||
metrics="$(extract_metrics "$log_file")"
|
||||
throughput="$(echo "$metrics" | cut -d',' -f1)"
|
||||
reqps="$(echo "$metrics" | cut -d',' -f2)"
|
||||
latency="$(echo "$metrics" | cut -d',' -f3)"
|
||||
|
||||
echo "${profile},${HOST},${bucket},${DURATION},${concurrent},${parts},${part_size},${part_concurrent},${throughput},${reqps},${latency},${log_file},${benchdata_file},${status}" >> "$SUMMARY_CSV"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
validate_args
|
||||
require_cmd "$WARP_BIN"
|
||||
require_cmd rg
|
||||
setup_output
|
||||
write_manifest
|
||||
|
||||
echo "Output dir: $OUT_DIR"
|
||||
echo "Profiles: $PROFILES"
|
||||
|
||||
IFS=',' read -r -a profiles <<< "$PROFILES"
|
||||
for raw in "${profiles[@]}"; do
|
||||
profile="$(trim "$raw")"
|
||||
[[ -z "$profile" ]] && continue
|
||||
run_profile "$profile"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Summary:"
|
||||
cat "$SUMMARY_CSV"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
HOST="${HOST:-127.0.0.1:9000}"
|
||||
ACCESS_KEY="${ACCESS_KEY:-}"
|
||||
SECRET_KEY="${SECRET_KEY:-}"
|
||||
BUCKET_PREFIX="${BUCKET_PREFIX:-issue712-gt1g-multipart-focus}"
|
||||
LOOKUP="${LOOKUP:-path}"
|
||||
DURATION="${DURATION:-10m}"
|
||||
WARP_BIN="${WARP_BIN:-warp}"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/gt1g-multipart-server-path-$(date +%Y%m%d-%H%M%S)}"
|
||||
DRY_RUN=false
|
||||
|
||||
DEFAULT_PROFILES="1g-64m-pc4,2g-128m-pc4"
|
||||
PROFILES="${PROFILES:-$DEFAULT_PROFILES}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/run_gt1g_multipart_put_server_path_focus.sh --access-key <ak> --secret-key <sk> [options]
|
||||
|
||||
Required:
|
||||
--access-key <ak>
|
||||
--secret-key <sk>
|
||||
|
||||
Options:
|
||||
--host <host:port> Default: 127.0.0.1:9000
|
||||
--bucket-prefix <prefix> Default: issue712-gt1g-multipart-focus
|
||||
--lookup <path|dns> Default: path
|
||||
--duration <dur> Default: 10m
|
||||
--profiles <csv> Default: 1g-64m-pc4,2g-128m-pc4
|
||||
--warp-bin <path> Default: warp
|
||||
--out-dir <dir> Default: target/bench/gt1g-multipart-server-path-<timestamp>
|
||||
--dry-run
|
||||
-h, --help
|
||||
|
||||
Profiles:
|
||||
1g-64m-pc4 concurrent=4 parts=16 part.size=64MiB part.concurrent=4
|
||||
2g-128m-pc4 concurrent=4 parts=16 part.size=128MiB part.concurrent=4
|
||||
2g-256m-pc4 concurrent=4 parts=8 part.size=256MiB part.concurrent=4
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "ERROR: command not found: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
trim() {
|
||||
echo "$1" | awk '{$1=$1;print}'
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--host) HOST="$2"; shift 2 ;;
|
||||
--access-key) ACCESS_KEY="$2"; shift 2 ;;
|
||||
--secret-key) SECRET_KEY="$2"; shift 2 ;;
|
||||
--bucket-prefix) BUCKET_PREFIX="$2"; shift 2 ;;
|
||||
--lookup) LOOKUP="$2"; shift 2 ;;
|
||||
--duration) DURATION="$2"; shift 2 ;;
|
||||
--profiles) PROFILES="$2"; shift 2 ;;
|
||||
--warp-bin) WARP_BIN="$2"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
validate_args() {
|
||||
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" ]]; then
|
||||
echo "ERROR: --access-key and --secret-key are required" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
setup_output() {
|
||||
mkdir -p "$OUT_DIR/logs" "$OUT_DIR/benchdata"
|
||||
SUMMARY_CSV="$OUT_DIR/summary.csv"
|
||||
COMMANDS_TXT="$OUT_DIR/commands.txt"
|
||||
MANIFEST_TXT="$OUT_DIR/run_manifest.txt"
|
||||
|
||||
echo "profile,host,bucket,duration,concurrent,parts,part_size,part_concurrent,throughput_human,reqps,latency_human,log_file,benchdata_file,status" > "$SUMMARY_CSV"
|
||||
: > "$COMMANDS_TXT"
|
||||
}
|
||||
|
||||
write_manifest() {
|
||||
{
|
||||
echo "created_at=$(date +%Y-%m-%dT%H:%M:%S%z)"
|
||||
echo "host=${HOST}"
|
||||
echo "bucket_prefix=${BUCKET_PREFIX}"
|
||||
echo "lookup=${LOOKUP}"
|
||||
echo "duration=${DURATION}"
|
||||
echo "profiles=${PROFILES}"
|
||||
echo "warp_bin=${WARP_BIN}"
|
||||
echo "dry_run=${DRY_RUN}"
|
||||
echo "access_key=REDACTED"
|
||||
echo "secret_key=REDACTED"
|
||||
} > "$MANIFEST_TXT"
|
||||
}
|
||||
|
||||
profile_spec() {
|
||||
case "$1" in
|
||||
1g-64m-pc4) echo "4|16|64MiB|4" ;;
|
||||
2g-128m-pc4) echo "4|16|128MiB|4" ;;
|
||||
2g-256m-pc4) echo "4|8|256MiB|4" ;;
|
||||
*)
|
||||
echo "ERROR: unknown profile: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
extract_first() {
|
||||
local regex="$1"
|
||||
local file="$2"
|
||||
rg -o "$regex" "$file" | head -n1 || true
|
||||
}
|
||||
|
||||
extract_metrics() {
|
||||
local log_file="$1"
|
||||
local throughput reqps latency
|
||||
|
||||
throughput="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(GiB/s|MiB/s|KiB/s|GB/s|MB/s|KB/s|B/s)' "$log_file")"
|
||||
reqps="$(extract_first '[0-9]+(\.[0-9]+)?[[:space:]]*(obj/s|req/s|ops/s|requests/s)' "$log_file")"
|
||||
latency="$(rg -o 'Reqs:[[:space:]]+Avg:[[:space:]]+[0-9]+(\.[0-9]+)?(ms|us|µs|s)' "$log_file" | head -n1 | sed -E 's/^Reqs:[[:space:]]+Avg:[[:space:]]+//')"
|
||||
|
||||
throughput="$(trim "${throughput:-N/A}")"
|
||||
reqps="$(trim "${reqps:-N/A}")"
|
||||
latency="$(trim "${latency:-N/A}")"
|
||||
reqps="$(echo "$reqps" | awk '{print $1}')"
|
||||
|
||||
echo "${throughput},${reqps:-N/A},${latency}"
|
||||
}
|
||||
|
||||
run_profile() {
|
||||
local profile="$1"
|
||||
local spec concurrent parts part_size part_concurrent
|
||||
local bucket log_file benchdata_file status metrics throughput reqps latency
|
||||
|
||||
spec="$(profile_spec "$profile")"
|
||||
IFS='|' read -r concurrent parts part_size part_concurrent <<< "$spec"
|
||||
|
||||
bucket="$(echo "${BUCKET_PREFIX}-${profile}-$(date +%Y%m%d%H%M%S)" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/-{2,}/-/g; s/^-+//; s/-+$//')"
|
||||
bucket="${bucket:0:63}"
|
||||
log_file="$OUT_DIR/logs/${profile}.log"
|
||||
benchdata_file="$OUT_DIR/benchdata/${profile}.csv.zst"
|
||||
|
||||
local -a cmd=(
|
||||
"$WARP_BIN" multipart-put
|
||||
--host "$HOST"
|
||||
--access-key "$ACCESS_KEY"
|
||||
--secret-key "$SECRET_KEY"
|
||||
--bucket "$bucket"
|
||||
--lookup "$LOOKUP"
|
||||
--duration "$DURATION"
|
||||
--concurrent "$concurrent"
|
||||
--parts "$parts"
|
||||
--part.size "$part_size"
|
||||
--part.concurrent "$part_concurrent"
|
||||
--benchdata "$benchdata_file"
|
||||
--analyze.v
|
||||
--no-color
|
||||
)
|
||||
|
||||
printf '%q ' "${cmd[@]}" >> "$COMMANDS_TXT"
|
||||
printf '\n' >> "$COMMANDS_TXT"
|
||||
|
||||
status="ok"
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
printf '\n'
|
||||
: > "$log_file"
|
||||
else
|
||||
if ! "${cmd[@]}" >"$log_file" 2>&1; then
|
||||
status="failed"
|
||||
fi
|
||||
fi
|
||||
|
||||
metrics="$(extract_metrics "$log_file")"
|
||||
throughput="$(echo "$metrics" | cut -d',' -f1)"
|
||||
reqps="$(echo "$metrics" | cut -d',' -f2)"
|
||||
latency="$(echo "$metrics" | cut -d',' -f3)"
|
||||
|
||||
echo "${profile},${HOST},${bucket},${DURATION},${concurrent},${parts},${part_size},${part_concurrent},${throughput},${reqps},${latency},${log_file},${benchdata_file},${status}" >> "$SUMMARY_CSV"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
validate_args
|
||||
require_cmd "$WARP_BIN"
|
||||
require_cmd rg
|
||||
setup_output
|
||||
write_manifest
|
||||
|
||||
echo "Output dir: $OUT_DIR"
|
||||
echo "Profiles: $PROFILES"
|
||||
|
||||
IFS=',' read -r -a profiles <<< "$PROFILES"
|
||||
for raw in "${profiles[@]}"; do
|
||||
profile="$(trim "$raw")"
|
||||
[[ -z "$profile" ]] && continue
|
||||
run_profile "$profile"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Summary:"
|
||||
cat "$SUMMARY_CSV"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user