gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: avoid eager parity reader setup (#4133)

Browse Source
This commit is contained in:
houseme
2026-07-02 08:49:08 +08:00
committed by GitHub
parent 18d89d1e31
commit 7a075c91da
13 changed files with 3159 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/analyze_put_service_metrics_deltas.py
+322
View File
@@ -0,0 +1,322 @@
#!/usr/bin/env python3
"""Summarize per-round PUT service metric deltas from before/after snapshots."""
from __future__ import annotations
import argparse
import csv
import re
from collections import defaultdict
from pathlib import Path
PROM_LINE_RE = re.compile(
r"^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{(.*)\})?\s+([-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)"
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--capture-csv", required=True, help="aggregate_service_metrics_captures.csv path")
parser.add_argument("--delta-csv", required=True, help="Output generic metric delta CSV")
parser.add_argument("--path-summary-csv", required=True, help="Output PUT path delta summary CSV")
parser.add_argument("--stage-summary-csv", required=True, help="Output PUT stage duration summary CSV")
return parser.parse_args()
def split_label_items(labels: str) -> list[str]:
items: list[str] = []
start = 0
in_quotes = False
escaped = False
for index, char in enumerate(labels):
if escaped:
escaped = False
continue
if char == "\\":
escaped = True
continue
if char == '"':
in_quotes = not in_quotes
continue
if char == "," and not in_quotes:
items.append(labels[start:index])
start = index + 1
items.append(labels[start:])
return [item.strip() for item in items if item.strip()]
def parse_labels(labels: str) -> dict[str, str]:
parsed: dict[str, str] = {}
for item in split_label_items(labels):
key, sep, value = item.partition("=")
if not sep:
continue
value = value.strip()
if value.startswith('"') and value.endswith('"') and len(value) >= 2:
value = value[1:-1]
value = value.replace(r"\"", '"').replace(r"\\", "\\")
parsed[key.strip()] = value
return parsed
def canonical_labels(labels: dict[str, str]) -> str:
return ",".join(f'{key}="{escape_label_value(labels[key])}"' for key in sorted(labels))
def escape_label_value(value: str) -> str:
return value.replace("\\", r"\\").replace('"', r"\"").replace("\n", r"\n")
def parse_prom(path: Path) -> dict[tuple[str, str], float]:
metrics: dict[tuple[str, str], float] = {}
with path.open("r", encoding="utf-8") as file:
for line in file:
line = line.strip()
if not line or line.startswith("#"):
continue
match = PROM_LINE_RE.match(line)
if not match:
continue
metric, raw_labels, raw_value = match.groups()
labels = canonical_labels(parse_labels(raw_labels or ""))
try:
value = float(raw_value)
except ValueError:
continue
metrics[(metric, labels)] = value
return metrics
def classify_metric(metric: str) -> str:
if metric == "rustfs_s3_put_object_path_total":
return "path_total"
if metric == "rustfs_s3_put_object_diagnostics_total":
return "diagnostics_total"
if metric.endswith("_bucket"):
return "histogram_bucket"
if metric.endswith("_count"):
return "histogram_count"
if metric.endswith("_sum"):
return "histogram_sum"
if metric.endswith("_total"):
return "counter_total"
return "gauge_delta"
def is_monotonic_metric(metric: str) -> bool:
return (
metric.endswith("_total")
or metric.endswith("_bucket")
or metric.endswith("_count")
or metric.endswith("_sum")
)
def group_capture_rows(capture_csv: Path) -> dict[tuple[str, str, str, str, str, str], dict[str, dict[str, str]]]:
groups: dict[tuple[str, str, str, str, str, str], dict[str, dict[str, str]]] = defaultdict(dict)
with capture_csv.open("r", encoding="utf-8", newline="") as file:
for row in csv.DictReader(file):
phase = row.get("phase", "")
if phase not in {"before", "after"}:
continue
key = (
row.get("concurrency", ""),
row.get("run_dir", ""),
row.get("size", ""),
row.get("tool", ""),
row.get("round", ""),
row.get("attempt", ""),
)
groups[key][phase] = row
return groups
def label_value(labels: str, name: str) -> str:
return parse_labels(labels).get(name, "")
def write_outputs(args: argparse.Namespace) -> None:
capture_csv = Path(args.capture_csv)
delta_csv = Path(args.delta_csv)
path_summary_csv = Path(args.path_summary_csv)
stage_summary_csv = Path(args.stage_summary_csv)
for output in (delta_csv, path_summary_csv, stage_summary_csv):
output.parent.mkdir(parents=True, exist_ok=True)
groups = group_capture_rows(capture_csv)
delta_fields = [
"concurrency",
"run_dir",
"size",
"tool",
"round",
"attempt",
"source",
"before_status",
"after_status",
"metric",
"labels",
"before",
"after",
"delta",
"classification",
]
path_fields = [
"concurrency",
"run_dir",
"size",
"tool",
"round",
"attempt",
"path",
"delta",
"before_status",
"after_status",
]
stage_fields = [
"concurrency",
"run_dir",
"size",
"tool",
"round",
"attempt",
"stage",
"count_delta",
"sum_delta",
"avg_ms",
"before_status",
"after_status",
]
with (
delta_csv.open("w", encoding="utf-8", newline="") as delta_file,
path_summary_csv.open("w", encoding="utf-8", newline="") as path_file,
stage_summary_csv.open("w", encoding="utf-8", newline="") as stage_file,
):
delta_writer = csv.DictWriter(delta_file, fieldnames=delta_fields)
path_writer = csv.DictWriter(path_file, fieldnames=path_fields)
stage_writer = csv.DictWriter(stage_file, fieldnames=stage_fields)
delta_writer.writeheader()
path_writer.writeheader()
stage_writer.writeheader()
for key in sorted(groups):
phases = groups[key]
before_row = phases.get("before")
after_row = phases.get("after")
if not before_row or not after_row:
continue
before_path = Path(before_row.get("snapshot_file", ""))
after_path = Path(after_row.get("snapshot_file", ""))
if before_row.get("status") != "ok" or after_row.get("status") != "ok":
continue
if not before_path.is_file() or not after_path.is_file():
continue
before = parse_prom(before_path)
after = parse_prom(after_path)
concurrency, run_dir, size, tool, round_no, attempt = key
source = after_row.get("source", before_row.get("source", ""))
before_status = before_row.get("status", "")
after_status = after_row.get("status", "")
stage_counts: dict[str, float] = defaultdict(float)
stage_sums: dict[str, float] = defaultdict(float)
for metric_key in sorted(after):
metric, labels = metric_key
before_value = before.get(metric_key, 0.0)
after_value = after.get(metric_key, 0.0)
delta = after_value - before_value
if is_monotonic_metric(metric) and delta < 0:
continue
if delta == 0:
continue
classification = classify_metric(metric)
delta_writer.writerow(
{
"concurrency": concurrency,
"run_dir": run_dir,
"size": size,
"tool": tool,
"round": round_no,
"attempt": attempt,
"source": source,
"before_status": before_status,
"after_status": after_status,
"metric": metric,
"labels": labels,
"before": f"{before_value:.12g}",
"after": f"{after_value:.12g}",
"delta": f"{delta:.12g}",
"classification": classification,
}
)
if metric == "rustfs_s3_put_object_path_total":
path_writer.writerow(
{
"concurrency": concurrency,
"run_dir": run_dir,
"size": size,
"tool": tool,
"round": round_no,
"attempt": attempt,
"path": label_value(labels, "path"),
"delta": f"{delta:.12g}",
"before_status": before_status,
"after_status": after_status,
}
)
if metric.endswith("_stage_duration_ms_count"):
stage = label_value(labels, "stage")
if stage:
stage_counts[stage] += delta
elif metric.endswith("_stage_duration_ms_sum"):
stage = label_value(labels, "stage")
if stage:
stage_sums[stage] += delta
elif metric.endswith("_stage_duration_seconds_count"):
stage = label_value(labels, "stage")
if stage:
stage_counts[stage] += delta
elif metric.endswith("_stage_duration_seconds_sum"):
stage = label_value(labels, "stage")
if stage:
stage_sums[stage] += delta * 1000.0
for stage in sorted(set(stage_counts) | set(stage_sums)):
count_delta = stage_counts.get(stage, 0.0)
if count_delta <= 0:
continue
sum_delta = stage_sums.get(stage, 0.0)
avg_ms = sum_delta / count_delta if count_delta else 0.0
stage_writer.writerow(
{
"concurrency": concurrency,
"run_dir": run_dir,
"size": size,
"tool": tool,
"round": round_no,
"attempt": attempt,
"stage": stage,
"count_delta": f"{count_delta:.12g}",
"sum_delta": f"{sum_delta:.12g}",
"avg_ms": f"{avg_ms:.6f}",
"before_status": before_status,
"after_status": after_status,
}
)
def main() -> None:
write_outputs(parse_args())
if __name__ == "__main__":
main()
scripts/run_get_codec_streaming_smoke.sh
+143 -49
View File
@@ -35,6 +35,8 @@ PROFILE_ORDER="normal"
CODEC_ENGINES="legacy"
CODEC_MAX_INFLIGHT=1
CODEC_READER_SETUP="all_shards"
CODEC_DATA_BLOCKS_FIRST_GATE="off"
CODEC_DATA_BLOCKS_FIRST_MAX_SIZE=""
CODEC_MULTIPART="off"
CODEC_MULTIPART_MAX_PARTS=256
METADATA_EARLY_STOP="off"
@@ -62,7 +64,9 @@ OUT_DIR=""
RUSTFS_BIN="${PROJECT_ROOT}/target/release/rustfs"
WARP_BIN="warp"
PYTHON_BIN="python3"
CODEC_MIN_SIZE=1
CODEC_MIN_SIZE=""
DEFAULT_CODEC_MIN_SIZE=1048576
DEFAULT_RUSTFS_CODEC_MIN_SIZE=1048576
RUST_LOG="warn"
HEALTH_TIMEOUT_SECS=60
COMPAT_OBJECT_KEY="__rustfs_get_v2_pr24_compat/object.bin"
@@ -102,6 +106,12 @@ Core options:
--codec-reader-setup <all_shards|data_blocks_first>
RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP
for codec verify reader setup (default: all_shards)
--codec-data-blocks-first-gate <on|off>
Enable size-gated data-blocks-first reader setup for
codec-rustfs plain single-part GETs (default: off)
--codec-data-blocks-first-max-size <bytes>
RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE
for the gated path (default: server default)
--codec-multipart <on|off> Enable multipart codec streaming opt-in for codec profiles
(default: off)
--codec-multipart-max-parts <n>
@@ -172,7 +182,8 @@ Binary/options:
--rustfs-bin <path> RustFS binary (default: target/release/rustfs)
--warp-bin <path> warp binary (default: warp)
--python-bin <path> Python binary for SigV4 compatibility probe (default: python3)
--codec-min-size <bytes> RUSTFS_GET_CODEC_STREAMING_MIN_SIZE (default: 1)
--codec-min-size <bytes> Override RUSTFS_GET_CODEC_STREAMING_MIN_SIZE
(default: unset, use server size-aware default)
--compat-object-key <key> Object key used by the compatibility probe
--compat-object-size <bytes> Object size used by the compatibility probe (default: 65536)
--skip-compat-probe skip post-benchmark compatibility/fallback probes
@@ -282,6 +293,8 @@ parse_args() {
--shard-locality-preference) SHARD_LOCALITY_PREFERENCE="$2"; shift 2 ;;
--codec-max-inflight) CODEC_MAX_INFLIGHT="$2"; shift 2 ;;
--codec-reader-setup) CODEC_READER_SETUP="$2"; shift 2 ;;
--codec-data-blocks-first-gate) CODEC_DATA_BLOCKS_FIRST_GATE="$2"; shift 2 ;;
--codec-data-blocks-first-max-size) CODEC_DATA_BLOCKS_FIRST_MAX_SIZE="$2"; shift 2 ;;
--codec-multipart) CODEC_MULTIPART="$2"; shift 2 ;;
--codec-multipart-max-parts) CODEC_MULTIPART_MAX_PARTS="$2"; shift 2 ;;
--handoff-attribution) OUTPUT_HANDOFF_ATTRIBUTION=true; shift ;;
@@ -388,6 +401,13 @@ validate_args() {
all_shards|data_blocks_first) ;;
*) die "--codec-reader-setup must be all_shards or data_blocks_first" ;;
esac
case "$CODEC_DATA_BLOCKS_FIRST_GATE" in
on|off) ;;
*) die "--codec-data-blocks-first-gate must be on or off" ;;
esac
if [[ -n "$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE" ]]; then
validate_positive_int "$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE" "--codec-data-blocks-first-max-size"
fi
validate_positive_int "$CODEC_MULTIPART_MAX_PARTS" "--codec-multipart-max-parts"
validate_positive_int "$ROUNDS" "--rounds"
validate_positive_int "$RETRY_PER_ROUND" "--retry-per-round"
@@ -399,11 +419,6 @@ validate_args() {
per-round|prepare-once|existing-only) ;;
*) die "--warp-object-lifecycle must be per-round, prepare-once, or existing-only" ;;
esac
if [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then
local size_count
size_count="$(printf '%s\n' "$SIZES" | awk -F',' '{ count=0; for (i=1; i<=NF; i++) { gsub(/^[ \t]+|[ \t]+$/, "", $i); if ($i != "") count++ } print count }')"
[[ "$size_count" -eq 1 ]] || die "--warp-object-lifecycle=${WARP_OBJECT_LIFECYCLE} currently requires a single --sizes value to avoid mixing object sizes in one bucket"
fi
if [[ -n "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" ]]; then
validate_positive_int "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" "--metadata-cache-max-entries"
fi
@@ -422,7 +437,9 @@ validate_args() {
*) die "--local-read-copy-method must be mmap_copy or direct_read_copy" ;;
esac
fi
validate_positive_int "$CODEC_MIN_SIZE" "--codec-min-size"
if [[ -n "$CODEC_MIN_SIZE" ]]; then
validate_positive_int "$CODEC_MIN_SIZE" "--codec-min-size"
fi
validate_positive_int "$COMPAT_OBJECT_SIZE" "--compat-object-size"
validate_positive_int "$RESOURCE_SAMPLE_INTERVAL_SECS" "--resource-sample-interval-secs"
validate_positive_int "$HEALTH_TIMEOUT_SECS" "--health-timeout-secs"
@@ -705,6 +722,8 @@ metadata_early_stop=${METADATA_EARLY_STOP}
shard_locality_preference=${SHARD_LOCALITY_PREFERENCE}
codec_max_inflight=${CODEC_MAX_INFLIGHT}
codec_reader_setup=${CODEC_READER_SETUP}
codec_data_blocks_first_gate=${CODEC_DATA_BLOCKS_FIRST_GATE}
codec_data_blocks_first_max_size=${CODEC_DATA_BLOCKS_FIRST_MAX_SIZE:-server-default}
codec_rollout_codec_profile=benchmark
codec_body_compat_confirmed_codec_profile=true
codec_header_compat_confirmed_codec_profile=true
@@ -748,7 +767,7 @@ dry_run=${DRY_RUN}
rustfs_bin=${RUSTFS_BIN}
warp_bin=${WARP_BIN}
python_bin=${PYTHON_BIN}
codec_min_size=${CODEC_MIN_SIZE}
codec_min_size=${CODEC_MIN_SIZE:-server-default}
compat_object_key=${COMPAT_OBJECT_KEY}
compat_object_size=${COMPAT_OBJECT_SIZE}
command_line=${command_line% }
@@ -819,6 +838,20 @@ profile_metrics_path() {
esac
}
compat_probe_codec_min_size() {
local profile="$1"
if [[ -n "$CODEC_MIN_SIZE" ]]; then
echo "$CODEC_MIN_SIZE"
return
fi
case "$profile" in
codec-rustfs) echo "${RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE:-$DEFAULT_RUSTFS_CODEC_MIN_SIZE}" ;;
codec-legacy) echo "$DEFAULT_CODEC_MIN_SIZE" ;;
*) echo "0" ;;
esac
}
profile_data_root() {
local profile="$1"
echo "${OUT_DIR}/${profile}/data"
@@ -845,6 +878,36 @@ single_benchmark_size() {
}'
}
benchmark_sizes() {
printf '%s\n' "$SIZES" | awk -F',' '{
for (i=1; i<=NF; i++) {
gsub(/^[ \t]+|[ \t]+$/, "", $i)
if ($i != "") {
print $i
}
}
}'
}
benchmark_size_count() {
benchmark_sizes | awk 'END { print NR + 0 }'
}
sanitize_bucket_suffix() {
printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-'
}
bucket_for_size() {
local size="$1"
local size_count
size_count="$(benchmark_size_count)"
if [[ "$WARP_OBJECT_LIFECYCLE" == "per-round" || "$size_count" -le 1 ]]; then
echo "$BUCKET"
return
fi
echo "${BUCKET}-$(sanitize_bucket_suffix "$size")"
}
write_manifest() {
local profile="$1"
local profile_dir="$2"
@@ -895,10 +958,12 @@ RUSTFS_GET_METADATA_EARLY_STOP_ENABLE=$(bool_from_on_off "$METADATA_EARLY_STOP")
RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE=$(bool_from_on_off "$SHARD_LOCALITY_PREFERENCE")
RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT=${CODEC_MAX_INFLIGHT}
RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP=$(codec_reader_setup_data_blocks_first)
RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE=$(bool_from_on_off "$CODEC_DATA_BLOCKS_FIRST_GATE")
RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE=${CODEC_DATA_BLOCKS_FIRST_MAX_SIZE:-server-default}
RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE=$(bool_from_on_off "$CODEC_MULTIPART")
RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS=${CODEC_MULTIPART_MAX_PARTS}
RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE=${OUTPUT_HANDOFF_ATTRIBUTION}
RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE}
RUSTFS_GET_CODEC_STREAMING_MIN_SIZE=${CODEC_MIN_SIZE:-server-default}
RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES=${GET_OBJECT_METADATA_CACHE_MAX_ENTRIES}
RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY=${GET_SMALL_OBJECT_DIRECT_MEMORY}
RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD=${GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD}
@@ -1027,11 +1092,22 @@ start_server() {
export RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT="$CODEC_MAX_INFLIGHT"
RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP="$(codec_reader_setup_data_blocks_first)"
export RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP
RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE="$(bool_from_on_off "$CODEC_DATA_BLOCKS_FIRST_GATE")"
export RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE
if [[ -n "$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE" ]]; then
export RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE="$CODEC_DATA_BLOCKS_FIRST_MAX_SIZE"
else
unset RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE
fi
RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE="$(bool_from_on_off "$CODEC_MULTIPART")"
export RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE
export RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS="$CODEC_MULTIPART_MAX_PARTS"
export RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE="$OUTPUT_HANDOFF_ATTRIBUTION"
export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE"
if [[ -n "$CODEC_MIN_SIZE" ]]; then
export RUSTFS_GET_CODEC_STREAMING_MIN_SIZE="$CODEC_MIN_SIZE"
else
unset RUSTFS_GET_CODEC_STREAMING_MIN_SIZE
fi
if [[ -n "$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES" ]]; then
export RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="$GET_OBJECT_METADATA_CACHE_MAX_ENTRIES"
fi
@@ -1078,8 +1154,6 @@ prepare_warp_existing_objects() {
local profile="$1"
local profile_dir="${OUT_DIR}/${profile}"
local prepare_dir="${profile_dir}/warp_prepare"
local size
size="$(single_benchmark_size)"
if [[ "$WARP_OBJECT_LIFECYCLE" != "prepare-once" ]]; then
return 0
@@ -1087,43 +1161,50 @@ prepare_warp_existing_objects() {
mkdir -p "$prepare_dir"
local cmd=(
"$WARP_BIN" get
--host "$ADDRESS"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--bucket "$BUCKET"
--obj.size "$size"
--concurrent "$CONCURRENCY"
--duration "$WARP_PREPARE_DURATION"
--region "$REGION"
--noclear
)
if [[ -n "$WARP_OBJECTS" ]]; then
cmd+=(--objects "$WARP_OBJECTS")
fi
local size bucket size_slug
while IFS= read -r size; do
[[ -n "$size" ]] || continue
bucket="$(bucket_for_size "$size")"
size_slug="$(sanitize_bucket_suffix "$size")"
{
printf 'profile=%s\n' "$profile"
printf 'mode=prepare-once\n'
printf 'size=%s\n' "$size"
printf 'duration=%s\n' "$WARP_PREPARE_DURATION"
printf 'bucket=%s\n' "$BUCKET"
printf 'command='
printf '%q ' "${cmd[@]}"
printf '\n'
} >"${prepare_dir}/manifest.env"
local cmd=(
"$WARP_BIN" get
--host "$ADDRESS"
--access-key "$ACCESS_KEY"
--secret-key "$SECRET_KEY"
--bucket "$bucket"
--obj.size "$size"
--concurrent "$CONCURRENCY"
--duration "$WARP_PREPARE_DURATION"
--region "$REGION"
--noclear
)
if [[ -n "$WARP_OBJECTS" ]]; then
cmd+=(--objects "$WARP_OBJECTS")
fi
if [[ "$DRY_RUN" == "true" ]]; then
log "[DRY-RUN] prepare existing warp objects profile=${profile} size=${size}"
printf '[DRY-RUN] ' >"${prepare_dir}/prepare.log"
printf '%q ' "${cmd[@]}" >>"${prepare_dir}/prepare.log"
printf '\n' >>"${prepare_dir}/prepare.log"
return 0
fi
{
printf 'profile=%s\n' "$profile"
printf 'mode=prepare-once\n'
printf 'size=%s\n' "$size"
printf 'duration=%s\n' "$WARP_PREPARE_DURATION"
printf 'bucket=%s\n' "$bucket"
printf 'command='
printf '%q ' "${cmd[@]}"
printf '\n'
} >"${prepare_dir}/manifest-${size_slug}.env"
log "Preparing existing warp objects profile=${profile} size=${size} lifecycle=${WARP_OBJECT_LIFECYCLE}..."
"${cmd[@]}" >"${prepare_dir}/prepare.log" 2>&1
if [[ "$DRY_RUN" == "true" ]]; then
log "[DRY-RUN] prepare existing warp objects profile=${profile} size=${size} bucket=${bucket}"
printf '[DRY-RUN] ' >"${prepare_dir}/prepare-${size_slug}.log"
printf '%q ' "${cmd[@]}" >>"${prepare_dir}/prepare-${size_slug}.log"
printf '\n' >>"${prepare_dir}/prepare-${size_slug}.log"
continue
fi
log "Preparing existing warp objects profile=${profile} size=${size} bucket=${bucket} lifecycle=${WARP_OBJECT_LIFECYCLE}..."
"${cmd[@]}" >"${prepare_dir}/prepare-${size_slug}.log" 2>&1
done < <(benchmark_sizes)
}
run_bench() {
@@ -1162,6 +1243,9 @@ run_bench() {
lifecycle_args="${lifecycle_args} --objects ${WARP_OBJECTS}"
fi
cmd+=(--extra-args "$lifecycle_args")
if [[ "$(benchmark_size_count)" -gt 1 ]]; then
cmd+=(--bucket-size-suffix)
fi
fi
if [[ "$DIAGNOSTIC_METRICS" == "true" && -z "$DIAGNOSTIC_PROMETHEUS_QUERY_URL" ]]; then
cmd+=(
@@ -1268,7 +1352,9 @@ if payload.get("status") != "success":
samples = [
sample
for sample in payload.get("data", {}).get("result", [])
if not service_name or sample.get("metric", {}).get("service.name") == service_name
if not service_name
or sample.get("metric", {}).get("service.name") == service_name
or sample.get("metric", {}).get("service_name") == service_name
]
lines = [line for sample in samples if (line := sample_to_line(sample))]
if not lines:
@@ -2508,7 +2594,7 @@ EOF
return
fi
"$PYTHON_BIN" - "$(endpoint_url)" "$ACCESS_KEY" "$SECRET_KEY" "$REGION" "$BUCKET" \
"$COMPAT_OBJECT_KEY" "$COMPAT_OBJECT_SIZE" "$CODEC_MIN_SIZE" "$compat_dir" "$profile" \
"$COMPAT_OBJECT_KEY" "$COMPAT_OBJECT_SIZE" "$(compat_probe_codec_min_size "$profile")" "$compat_dir" "$profile" \
"$COMPRESSED_FALLBACK_PROBE" "$COMPRESSED_PROBE_EXTENSION" "$COMPRESSED_PROBE_MIME_TYPE" \
"$CODEC_MULTIPART" "$CODEC_MULTIPART_MAX_PARTS" <<'PY'
import base64
@@ -3338,6 +3424,8 @@ for profile_dir in profile_dirs:
"shard_locality_preference": manifest.get("RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE", ""),
"codec_max_inflight": manifest.get("RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT", ""),
"codec_reader_setup": manifest.get("RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_READER_SETUP", ""),
"codec_data_blocks_first_gate": manifest.get("RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE", ""),
"codec_data_blocks_first_max_size": manifest.get("RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE", ""),
"local_read_copy_method": manifest.get("RUSTFS_OBJECT_MMAP_READ_METHOD", ""),
"output_handoff_attribution": manifest.get("RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE", ""),
"round_cooldown_secs": manifest.get("round_cooldown_secs", ""),
@@ -3387,6 +3475,8 @@ fieldnames = [
"shard_locality_preference",
"codec_max_inflight",
"codec_reader_setup",
"codec_data_blocks_first_gate",
"codec_data_blocks_first_max_size",
"local_read_copy_method",
"output_handoff_attribution",
"round_cooldown_secs",
@@ -3423,6 +3513,8 @@ baseline_fields = [
"shard_locality_preference",
"codec_max_inflight",
"codec_reader_setup",
"codec_data_blocks_first_gate",
"codec_data_blocks_first_max_size",
"local_read_copy_method",
"output_handoff_attribution",
"round_cooldown_secs",
@@ -3467,6 +3559,8 @@ for row in rows:
"shard_locality_preference": row["shard_locality_preference"],
"codec_max_inflight": row["codec_max_inflight"],
"codec_reader_setup": row["codec_reader_setup"],
"codec_data_blocks_first_gate": row["codec_data_blocks_first_gate"],
"codec_data_blocks_first_max_size": row["codec_data_blocks_first_max_size"],
"local_read_copy_method": row["local_read_copy_method"],
"output_handoff_attribution": row["output_handoff_attribution"],
"round_cooldown_secs": row["round_cooldown_secs"],
scripts/run_object_batch_bench_enhanced.sh
+359 -13
View File
@@ -14,12 +14,14 @@ ENDPOINT=""
ACCESS_KEY=""
SECRET_KEY=""
BUCKET="rustfs-bench"
BUCKET_SIZE_SUFFIX=false
REGION="us-east-1"
CONCURRENCY=128
SIZES="$DEFAULT_SIZES"
OUT_DIR=""
INSECURE=false
DRY_RUN=false
PYTHON_BIN="python3"
# warp options
WARP_BIN="warp"
@@ -34,7 +36,7 @@ SAMPLES=20000
ROUNDS=3
RETRY_PER_ROUND=2
RETRY_SLEEP_SECS=2
COOLDOWN_SECS=0
COOLDOWN_SECS=20
BASELINE_CSV=""
EXTRA_ARGS=()
FAILED_FINAL_ROUNDS=0
@@ -44,7 +46,13 @@ SERVICE_METRICS_CAPTURE_ATTEMPTS=3
SERVICE_METRICS_CAPTURE_RETRY_SECS=1
SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2
SERVICE_METRICS_MAX_TIME_SECS=15
SERVICE_METRICS_FILTER_REGEX="rustfs_io_get_object_"
SERVICE_METRICS_SETTLE_SECS=0
SERVICE_METRICS_FILTER_REGEX=""
SERVICE_METRICS_FILTER_REGEX_EXPLICIT=false
SERVICE_PROMETHEUS_QUERY_URL=""
SERVICE_PROMETHEUS_QUERY=""
SERVICE_PROMETHEUS_QUERY_EXPLICIT=false
SERVICE_METRICS_SERVICE_NAME=""
usage() {
cat <<'USAGE'
@@ -60,12 +68,15 @@ Required:
Core options:
--bucket Bucket name (default: rustfs-bench)
--bucket-size-suffix Append a sanitized object size suffix to the
bucket for each size
--region Region (default: us-east-1)
--concurrency Concurrency for all sizes (default: 128)
--sizes Comma-separated sizes (default: 1KiB..10MiB matrix)
--out-dir Output directory (default: target/bench/object-batch-enhanced-<timestamp>)
--insecure Allow insecure TLS
--dry-run Print commands only, do not execute
--python-bin Python binary for Prometheus query capture (default: python3)
Warp options:
--warp-bin warp binary (default: warp)
@@ -80,11 +91,17 @@ Enhanced options:
--rounds Benchmark rounds per size (default: 3)
--retry-per-round Retry count per failed round (default: 2)
--retry-sleep-secs Sleep seconds between retries (default: 2)
--cooldown-secs Sleep seconds between rounds/sizes (default: 0)
--cooldown-secs Sleep seconds between rounds/sizes (default: 20)
--round-cooldown-secs Compatibility alias for --cooldown-secs
--baseline-csv Baseline median CSV to compare
--extra-args Extra args appended to tool command, quoted as one string
--service-metrics-url Optional Prometheus scrape URL captured before/after each round attempt
--service-prometheus-query-url
Optional Prometheus HTTP API /api/v1/query URL for OTLP-exported metrics
--service-prometheus-query PromQL used with --service-prometheus-query-url
(default: auto by tool/mode)
--service-metrics-service-name
Optional service.name/service_name label filter for Prometheus query results
--service-metrics-dir Output directory for per-round service metric snapshots
--service-metrics-attempts Capture attempts for each snapshot (default: 3)
--service-metrics-retry-secs Sleep seconds between failed capture attempts (default: 1)
@@ -92,9 +109,11 @@ Enhanced options:
Curl connect timeout for each metrics capture (default: 2)
--service-metrics-max-time-secs
Curl max time for each metrics capture (default: 15)
--service-metrics-settle-secs
Sleep seconds before after-snapshot capture (default: 0)
--service-metrics-filter-regex
Regex for retained metrics lines after scrape
(default: rustfs_io_get_object_)
(default: auto by tool/mode)
Output files:
round_results.csv One row per round attempt (with retry trace)
@@ -136,12 +155,14 @@ parse_args() {
--access-key) ACCESS_KEY="$2"; shift 2 ;;
--secret-key) SECRET_KEY="$2"; shift 2 ;;
--bucket) BUCKET="$2"; shift 2 ;;
--bucket-size-suffix) BUCKET_SIZE_SUFFIX=true; shift ;;
--region) REGION="$2"; shift 2 ;;
--concurrency) CONCURRENCY="$2"; shift 2 ;;
--sizes) SIZES="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--insecure) INSECURE=true; shift ;;
--dry-run) DRY_RUN=true; shift ;;
--python-bin) PYTHON_BIN="$2"; shift 2 ;;
--warp-bin) WARP_BIN="$2"; shift 2 ;;
--warp-mode) WARP_MODE="$2"; shift 2 ;;
--duration) DURATION="$2"; shift 2 ;;
@@ -154,12 +175,16 @@ parse_args() {
--round-cooldown-secs) COOLDOWN_SECS="$2"; shift 2 ;;
--baseline-csv) BASELINE_CSV="$2"; shift 2 ;;
--service-metrics-url) SERVICE_METRICS_URL="$2"; shift 2 ;;
--service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$2"; shift 2 ;;
--service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$2"; SERVICE_PROMETHEUS_QUERY_EXPLICIT=true; shift 2 ;;
--service-metrics-service-name) SERVICE_METRICS_SERVICE_NAME="$2"; shift 2 ;;
--service-metrics-dir) SERVICE_METRICS_DIR="$2"; shift 2 ;;
--service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$2"; shift 2 ;;
--service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$2"; shift 2 ;;
--service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$2"; shift 2 ;;
--service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$2"; shift 2 ;;
--service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; shift 2 ;;
--service-metrics-settle-secs) SERVICE_METRICS_SETTLE_SECS="$2"; shift 2 ;;
--service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$2"; SERVICE_METRICS_FILTER_REGEX_EXPLICIT=true; shift 2 ;;
--extra-args)
# shellcheck disable=SC2206
EXTRA_ARGS=($2)
@@ -205,6 +230,48 @@ cooldown_sleep() {
fi
}
default_service_metrics_filter_regex() {
if [[ "$TOOL" == "warp" ]]; then
case "$WARP_MODE" in
get)
echo "rustfs_io_get_object_|rustfs_s3_get_object_|rustfs_zero_copy_read|rustfs_mmap_|rustfs_get_object_"
return
;;
put)
echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_write|rustfs_buffer_|rustfs_ec_|rustfs_io_bytespool_"
return
;;
mixed)
echo "rustfs_s3_get_object_|rustfs_io_get_object_|rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_zero_copy_|rustfs_buffer_|rustfs_ec_"
return
;;
esac
fi
echo "rustfs_s3_put_object_|rustfs_io_put_object_|rustfs_s3_get_object_|rustfs_io_get_object_"
}
default_service_prometheus_query() {
if [[ "$TOOL" == "warp" ]]; then
case "$WARP_MODE" in
get)
echo '{__name__=~"rustfs_(io_get_object|s3_get_object|zero_copy_read|mmap|get_object)_.*"}'
return
;;
put)
echo '{__name__=~"rustfs_(s3_put_object|io_put_object|zero_copy_write|buffer|ec|io_bytespool)_.*"}'
return
;;
mixed)
echo '{__name__=~"rustfs_(s3_get_object|io_get_object|s3_put_object|io_put_object|zero_copy|buffer|ec)_.*"}'
return
;;
esac
fi
echo '{__name__=~"rustfs_(s3_put_object|io_put_object|s3_get_object|io_get_object)_.*"}'
}
validate_args() {
if [[ "$TOOL" != "warp" && "$TOOL" != "s3bench" ]]; then
echo "ERROR: --tool must be warp or s3bench" >&2
@@ -223,13 +290,21 @@ validate_args() {
validate_nonnegative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" "--service-metrics-retry-secs"
validate_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" "--service-metrics-connect-timeout-secs"
validate_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" "--service-metrics-max-time-secs"
if [[ -n "$SERVICE_METRICS_URL" && -z "$SERVICE_METRICS_DIR" ]]; then
echo "ERROR: --service-metrics-dir is required when --service-metrics-url is set" >&2
validate_nonnegative_int "$SERVICE_METRICS_SETTLE_SECS" "--service-metrics-settle-secs"
if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2
exit 1
fi
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]] && [[ -z "$SERVICE_METRICS_DIR" ]]; then
echo "ERROR: --service-metrics-dir is required when service metrics capture is enabled" >&2
exit 1
fi
if [[ -n "$SERVICE_METRICS_URL" ]]; then
require_cmd curl
fi
if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
require_cmd "$PYTHON_BIN"
fi
if [[ "$TOOL" == "s3bench" ]]; then
validate_positive_int "$SAMPLES" "--samples"
fi
@@ -238,6 +313,10 @@ validate_args() {
exit 1
fi
if [[ "$TOOL" == "warp" ]]; then
if [[ "$WARP_MODE" != "get" && "$WARP_MODE" != "put" && "$WARP_MODE" != "mixed" ]]; then
echo "ERROR: --warp-mode must be get, put, or mixed" >&2
exit 1
fi
local warp_host
warp_host="$(normalize_warp_host "$ENDPOINT")"
if [[ -z "$warp_host" ]]; then
@@ -245,6 +324,89 @@ validate_args() {
exit 1
fi
fi
if [[ "$SERVICE_METRICS_FILTER_REGEX_EXPLICIT" != "true" && -z "$SERVICE_METRICS_FILTER_REGEX" ]]; then
SERVICE_METRICS_FILTER_REGEX="$(default_service_metrics_filter_regex)"
fi
if [[ "$SERVICE_PROMETHEUS_QUERY_EXPLICIT" != "true" && -z "$SERVICE_PROMETHEUS_QUERY" ]]; then
SERVICE_PROMETHEUS_QUERY="$(default_service_prometheus_query)"
fi
}
git_value() {
local default_value="$1"
shift
git "$@" 2>/dev/null || echo "$default_value"
}
git_dirty_state() {
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "unknown"
return
fi
if ! git diff --quiet --ignore-submodules --; then
echo "dirty"
return
fi
if ! git diff --cached --quiet --ignore-submodules --; then
echo "dirty"
return
fi
echo "clean"
}
write_run_manifest() {
local manifest_file="$OUT_DIR/run_manifest.env"
local started_at_utc git_commit git_branch git_dirty rustc_version uname_s service_metrics_csv
started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
git_commit="$(git_value unknown rev-parse --short HEAD)"
git_branch="$(git_value unknown branch --show-current)"
git_dirty="$(git_dirty_state)"
rustc_version="$(rustc --version 2>/dev/null || echo unknown)"
uname_s="$(uname -a 2>/dev/null || echo unknown)"
service_metrics_csv="${SERVICE_METRICS_CSV:-}"
cat >"$manifest_file" <<EOF
started_at_utc=${started_at_utc}
git_commit=${git_commit}
git_branch=${git_branch}
git_dirty=${git_dirty}
rustc_version=${rustc_version}
uname=${uname_s}
tool=${TOOL}
endpoint=${ENDPOINT}
access_key=REDACTED
secret_key=REDACTED
bucket=${BUCKET}
bucket_size_suffix=${BUCKET_SIZE_SUFFIX}
region=${REGION}
concurrency=${CONCURRENCY}
sizes=${SIZES}
rounds=${ROUNDS}
retry_per_round=${RETRY_PER_ROUND}
retry_sleep_secs=${RETRY_SLEEP_SECS}
cooldown_secs=${COOLDOWN_SECS}
warp_bin=${WARP_BIN}
warp_mode=${WARP_MODE}
duration=${DURATION}
s3bench_bin=${S3BENCH_BIN}
samples=${SAMPLES}
baseline_csv=${BASELINE_CSV}
service_metrics_url=${SERVICE_METRICS_URL}
service_prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL}
service_prometheus_query=${SERVICE_PROMETHEUS_QUERY}
service_prometheus_query_explicit=${SERVICE_PROMETHEUS_QUERY_EXPLICIT}
service_metrics_service_name=${SERVICE_METRICS_SERVICE_NAME}
service_metrics_dir=${SERVICE_METRICS_DIR}
service_metrics_csv=${service_metrics_csv}
service_metrics_filter_regex=${SERVICE_METRICS_FILTER_REGEX}
service_metrics_filter_regex_explicit=${SERVICE_METRICS_FILTER_REGEX_EXPLICIT}
service_metrics_capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}
service_metrics_retry_secs=${SERVICE_METRICS_CAPTURE_RETRY_SECS}
service_metrics_connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}
service_metrics_max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
service_metrics_settle_secs=${SERVICE_METRICS_SETTLE_SECS}
dry_run=${DRY_RUN}
EOF
}
setup_output() {
@@ -252,22 +414,40 @@ setup_output() {
OUT_DIR="target/bench/object-batch-enhanced-$(date +%Y%m%d-%H%M%S)"
fi
mkdir -p "$OUT_DIR/logs"
if [[ -n "$SERVICE_METRICS_URL" ]]; then
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
mkdir -p "$SERVICE_METRICS_DIR"
fi
ROUND_CSV="$OUT_DIR/round_results.csv"
MEDIAN_CSV="$OUT_DIR/median_summary.csv"
COMPARE_CSV="$OUT_DIR/baseline_compare.csv"
SERVICE_METRICS_CSV="$OUT_DIR/service_metrics_captures.csv"
echo "size,tool,round,attempt,concurrency,status,exit_code,round_started_at_utc,round_finished_at_utc,throughput_human,throughput_bps,reqps,latency_human,latency_ms,log_file,req_p90_human,req_p90_ms,req_p99_human,req_p99_ms" > "$ROUND_CSV"
echo "size,tool,concurrency,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,median_req_p90_ms,median_req_p99_ms" > "$MEDIAN_CSV"
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "size,tool,round,attempt,phase,source,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex,prometheus_query" > "$SERVICE_METRICS_CSV"
fi
write_run_manifest
}
trim() {
echo "$1" | awk '{$1=$1;print}'
}
sanitize_bucket_suffix() {
printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-'
}
bucket_for_size() {
local size="$1"
if [[ "$BUCKET_SIZE_SUFFIX" != "true" ]]; then
echo "$BUCKET"
return
fi
echo "${BUCKET}-$(sanitize_bucket_suffix "$size")"
}
to_bps() {
local human="$1"
local number unit factor
@@ -400,17 +580,119 @@ filter_service_metrics_snapshot() {
fi
}
settle_before_after_metrics_capture() {
if (( SERVICE_METRICS_SETTLE_SECS <= 0 )); then
return 0
fi
if [[ "$DRY_RUN" == "true" ]]; then
echo "[DRY-RUN] sleep ${SERVICE_METRICS_SETTLE_SECS}"
else
sleep "$SERVICE_METRICS_SETTLE_SECS"
fi
}
capture_prometheus_query_snapshot() {
local phase="$1"
local snapshot_file="$2"
local status_file="$3"
local capture_attempt="$4"
"$PYTHON_BIN" - "$SERVICE_PROMETHEUS_QUERY_URL" "$SERVICE_PROMETHEUS_QUERY" "$phase" "$snapshot_file" "$status_file" "$SERVICE_METRICS_SERVICE_NAME" "$SERVICE_METRICS_SETTLE_SECS" "$SERVICE_METRICS_MAX_TIME_SECS" "$SERVICE_METRICS_FILTER_REGEX" "$capture_attempt" <<'PY'
import json
import pathlib
import sys
import urllib.parse
import urllib.request
query_url, query, phase, snapshot_raw, status_raw, service_name, settle_secs, timeout_secs_raw, filter_regex, capture_attempt = sys.argv[1:]
snapshot_path = pathlib.Path(snapshot_raw)
status_path = pathlib.Path(status_raw)
timeout_secs = float(timeout_secs_raw)
def write_status(status):
status_path.write_text(
"\n".join(
[
f"phase={phase}",
f"status={status}",
f"url={query_url}",
f"query={query}",
f"service_name={service_name}",
f"settle_secs={settle_secs}",
f"max_time_secs={timeout_secs_raw}",
f"filter_regex={filter_regex}",
f"capture_attempt={capture_attempt}",
"",
]
),
encoding="utf-8",
)
def escape_label(value):
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
def sample_to_line(sample):
metric = dict(sample.get("metric", {}))
name = metric.pop("__name__", "")
if not name:
return None
labels = ",".join(f'{key}="{escape_label(value)}"' for key, value in sorted(metric.items()))
value = sample.get("value", [None, None])[1]
if value is None:
return None
if labels:
return f"{name}{{{labels}}} {value}"
return f"{name} {value}"
try:
separator = "&" if "?" in query_url else "?"
url = f"{query_url}{separator}{urllib.parse.urlencode({'query': query})}"
request = urllib.request.Request(url, headers={"Accept": "application/json"})
with urllib.request.urlopen(request, timeout=timeout_secs) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception as err: # noqa: BLE001 - shell harness reports failures in status files.
snapshot_path.write_text(f"# prometheus_query_failed error={err}\n", encoding="utf-8")
write_status("query_failed")
raise SystemExit(0)
if payload.get("status") != "success":
snapshot_path.write_text(f"# prometheus_query_failed payload_status={payload.get('status')}\n", encoding="utf-8")
write_status("query_failed")
raise SystemExit(0)
samples = [
sample
for sample in payload.get("data", {}).get("result", [])
if not service_name
or sample.get("metric", {}).get("service.name") == service_name
or sample.get("metric", {}).get("service_name") == service_name
]
lines = [line for sample in samples if (line := sample_to_line(sample))]
if not lines:
snapshot_path.write_text(f"# no_matching_metrics service_name={service_name}\n", encoding="utf-8")
write_status("no_matching_metrics")
raise SystemExit(0)
snapshot_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
write_status("ok")
PY
}
capture_round_service_metrics() {
local size="$1"
local round="$2"
local attempt="$3"
local phase="$4"
if [[ -z "$SERVICE_METRICS_URL" ]]; then
if [[ -z "$SERVICE_METRICS_URL" && -z "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
return 0
fi
local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes
local token snapshot_file status_file tmp_file filtered_file capture_attempt raw_bytes snapshot_bytes capture_status
token="$(metric_snapshot_token "$size" "$round" "$attempt")"
snapshot_file="${SERVICE_METRICS_DIR}/${token}_${phase}.prom"
status_file="${SERVICE_METRICS_DIR}/${token}_${phase}.status"
@@ -426,8 +708,54 @@ attempt=${attempt}
phase=${phase}
status=not_run_dry_run
url=${SERVICE_METRICS_URL}
prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
prometheus_query=${SERVICE_PROMETHEUS_QUERY}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
echo "$size,$TOOL,$round,$attempt,$phase,dry_run,not_run_dry_run,0,0,0,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
return 0
fi
if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
for ((capture_attempt=1; capture_attempt<=SERVICE_METRICS_CAPTURE_ATTEMPTS; capture_attempt++)); do
capture_prometheus_query_snapshot "$phase" "$tmp_file" "$status_file" "$capture_attempt"
capture_status="$(awk -F= '$1=="status" {print $2; exit}' "$status_file")"
if [[ "${capture_status:-unknown}" == "ok" && -s "$tmp_file" ]]; then
raw_bytes="$(wc -c <"$tmp_file" | tr -d '[:space:]')"
filter_service_metrics_snapshot "$tmp_file" "$filtered_file"
mv "$filtered_file" "$snapshot_file"
snapshot_bytes="$(wc -c <"$snapshot_file" | tr -d '[:space:]')"
{
echo "raw_bytes=${raw_bytes}"
echo "snapshot_bytes=${snapshot_bytes}"
} >>"$status_file"
echo "$size,$TOOL,$round,$attempt,$phase,prometheus_query,ok,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
return 0
fi
rm -f "$tmp_file" "$filtered_file"
if (( capture_attempt < SERVICE_METRICS_CAPTURE_ATTEMPTS && SERVICE_METRICS_CAPTURE_RETRY_SECS > 0 )); then
sleep "$SERVICE_METRICS_CAPTURE_RETRY_SECS"
fi
done
: >"$snapshot_file"
cat >"$status_file" <<EOF
size=${size}
round=${round}
attempt=${attempt}
phase=${phase}
status=capture_failed
capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}
url=${SERVICE_PROMETHEUS_QUERY_URL}
query=${SERVICE_PROMETHEUS_QUERY}
max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
echo "$size,$TOOL,$round,$attempt,$phase,prometheus_query,capture_failed,$SERVICE_METRICS_CAPTURE_ATTEMPTS,N/A,N/A,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
echo "WARN: failed to capture service metrics size=${size} round=${round} attempt=${attempt} phase=${phase}" >&2
return 0
fi
@@ -454,7 +782,9 @@ max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
raw_bytes=${raw_bytes}
snapshot_bytes=${snapshot_bytes}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
echo "$size,$TOOL,$round,$attempt,$phase,prometheus_text,ok,$capture_attempt,$raw_bytes,$snapshot_bytes,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
return 0
fi
fi
@@ -477,7 +807,9 @@ url=${SERVICE_METRICS_URL}
connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}
max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}
filter_regex=${SERVICE_METRICS_FILTER_REGEX}
settle_secs=${SERVICE_METRICS_SETTLE_SECS}
EOF
echo "$size,$TOOL,$round,$attempt,$phase,prometheus_text,capture_failed,$SERVICE_METRICS_CAPTURE_ATTEMPTS,N/A,N/A,$status_file,$snapshot_file,$SERVICE_METRICS_FILTER_REGEX,$SERVICE_PROMETHEUS_QUERY" >> "$SERVICE_METRICS_CSV"
echo "WARN: failed to capture service metrics size=${size} round=${round} attempt=${attempt} phase=${phase}" >&2
}
@@ -515,14 +847,15 @@ run_one_attempt() {
started_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
if [[ "$TOOL" == "warp" ]]; then
local warp_host
local warp_host bucket
warp_host="$(normalize_warp_host "$ENDPOINT")"
bucket="$(bucket_for_size "$size")"
local cmd=(
"$WARP_BIN" "$WARP_MODE"
"--host" "$warp_host"
"--access-key" "$ACCESS_KEY"
"--secret-key" "$SECRET_KEY"
"--bucket" "$BUCKET"
"--bucket" "$bucket"
"--obj.size" "$size"
"--concurrent" "$CONCURRENCY"
"--duration" "$DURATION"
@@ -548,11 +881,13 @@ run_one_attempt() {
fi
fi
else
local bucket
bucket="$(bucket_for_size "$size")"
local cmd=(
"$S3BENCH_BIN"
"-accessKey=$ACCESS_KEY"
"-secretKey=$SECRET_KEY"
"-bucket=$BUCKET"
"-bucket=$bucket"
"-endpoint=$ENDPOINT"
"-region=$REGION"
"-numClients=$CONCURRENCY"
@@ -580,6 +915,7 @@ run_one_attempt() {
fi
fi
finished_at_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
settle_before_after_metrics_capture
capture_round_service_metrics "$size" "$round" "$attempt" after
local metrics throughput_human reqps latency_human throughput_bps latency_ms req_p90_human req_p90_ms req_p99_human req_p99_ms
@@ -733,6 +1069,16 @@ main() {
echo "Rounds: $ROUNDS"
echo "Retry per round: $RETRY_PER_ROUND"
echo "Cooldown secs: $COOLDOWN_SECS"
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "Service metrics filter: $SERVICE_METRICS_FILTER_REGEX"
if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "Service Prometheus query URL: $SERVICE_PROMETHEUS_QUERY_URL"
echo "Service Prometheus query: $SERVICE_PROMETHEUS_QUERY"
if [[ -n "$SERVICE_METRICS_SERVICE_NAME" ]]; then
echo "Service metrics service name: $SERVICE_METRICS_SERVICE_NAME"
fi
fi
fi
IFS=',' read -r -a size_arr <<< "$SIZES"
local total_sizes="${#size_arr[@]}"
scripts/run_put_large_stage_breakdown.sh
+145 -4
View File
@@ -11,6 +11,7 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
ENHANCED_BENCH_SCRIPT="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh"
SERVICE_METRICS_DELTA_SCRIPT="${PROJECT_ROOT}/scripts/analyze_put_service_metrics_deltas.py"
TOOL="warp"
ENDPOINT=""
@@ -24,13 +25,24 @@ DURATION="120s"
ROUNDS=3
RETRY_PER_ROUND=1
RETRY_SLEEP_SECS=2
COOLDOWN_SECS=0
COOLDOWN_SECS=20
WARP_BIN="${WARP_BIN:-warp}"
OUT_DIR=""
BASELINE_ROOT=""
EXTRA_ARGS=""
INSECURE=false
DRY_RUN=false
PYTHON_BIN="python3"
SERVICE_METRICS_URL=""
SERVICE_PROMETHEUS_QUERY_URL=""
SERVICE_PROMETHEUS_QUERY=""
SERVICE_METRICS_SERVICE_NAME=""
SERVICE_METRICS_FILTER_REGEX=""
SERVICE_METRICS_CAPTURE_ATTEMPTS=3
SERVICE_METRICS_CAPTURE_RETRY_SECS=1
SERVICE_METRICS_CONNECT_TIMEOUT_SECS=2
SERVICE_METRICS_MAX_TIME_SECS=15
SERVICE_METRICS_SETTLE_SECS=0
TOPOLOGY_NODES=""
TOPOLOGY_DISKS_PER_NODE=""
@@ -63,13 +75,29 @@ Core options:
--rounds <n> Rounds per size (default: 3)
--retry-per-round <n> Retries per failed round (default: 1)
--retry-sleep-secs <n> Sleep between retries (default: 2)
--cooldown-secs <n> Sleep between rounds/sizes (default: 0)
--cooldown-secs <n> Sleep between rounds/sizes/concurrency runs (default: 20)
--round-cooldown-secs <n> Compatibility alias for --cooldown-secs
--out-dir <dir> Output root (default: target/bench/put-large-stage-breakdown-<timestamp>)
--baseline-root <dir> Existing root from a previous run of this script
--extra-args "<args>" Extra args passed to run_object_batch_bench_enhanced.sh
--warp-bin <path> warp binary (default: warp)
--insecure Pass --insecure to warp
--dry-run Print commands only
--python-bin <path> Python binary for Prometheus query capture (default: python3)
--service-metrics-url <url> Plain Prometheus text scrape URL
--service-prometheus-query-url <url>
Prometheus HTTP API /api/v1/query URL for OTLP-exported metrics
--service-prometheus-query <promql> PromQL for --service-prometheus-query-url
--service-metrics-service-name <name>
Optional service.name/service_name filter for query results
--service-metrics-filter-regex <regex>
Regex for retained plain text metrics lines
--service-metrics-attempts <n> Direct scrape attempts per snapshot (default: 3)
--service-metrics-retry-secs <n> Sleep between direct scrape attempts (default: 1)
--service-metrics-connect-timeout-secs <n>
Curl connect timeout for direct scrape (default: 2)
--service-metrics-max-time-secs <n> Curl max time for direct scrape (default: 15)
--service-metrics-settle-secs <n> Sleep before after-snapshot capture (default: 0)
Topology metadata (optional but recommended):
--nodes <n>
@@ -117,6 +145,16 @@ arg_value() {
printf '%s\n' "$value"
}
arg_value_allow_option() {
local flag="$1"
local value="${2:-}"
if [[ -z "$value" ]]; then
echo "ERROR: missing value for $flag" >&2
exit 1
fi
printf '%s\n' "$value"
}
parse_args() {
while [[ $# -gt 0 ]]; do
case "$1" in
@@ -131,11 +169,22 @@ parse_args() {
--rounds) ROUNDS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--retry-per-round) RETRY_PER_ROUND="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--retry-sleep-secs) RETRY_SLEEP_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--cooldown-secs|--round-cooldown-secs) COOLDOWN_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--baseline-root) BASELINE_ROOT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--extra-args) EXTRA_ARGS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--extra-args) EXTRA_ARGS="$(arg_value_allow_option "$1" "${2:-}")"; shift 2 ;;
--warp-bin) WARP_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--python-bin) PYTHON_BIN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-url) SERVICE_METRICS_URL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-prometheus-query-url) SERVICE_PROMETHEUS_QUERY_URL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-prometheus-query) SERVICE_PROMETHEUS_QUERY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-service-name) SERVICE_METRICS_SERVICE_NAME="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-filter-regex) SERVICE_METRICS_FILTER_REGEX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-attempts) SERVICE_METRICS_CAPTURE_ATTEMPTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-retry-secs) SERVICE_METRICS_CAPTURE_RETRY_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-connect-timeout-secs) SERVICE_METRICS_CONNECT_TIMEOUT_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-max-time-secs) SERVICE_METRICS_MAX_TIME_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--service-metrics-settle-secs) SERVICE_METRICS_SETTLE_SECS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--nodes) TOPOLOGY_NODES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--disks-per-node) TOPOLOGY_DISKS_PER_NODE="$(arg_value "$1" "${2:-}")"; shift 2 ;;
--total-disks) TOPOLOGY_TOTAL_DISKS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
@@ -195,6 +244,14 @@ validate_args() {
echo "ERROR: --baseline-root does not exist: $BASELINE_ROOT" >&2
exit 1
fi
if [[ -n "$SERVICE_METRICS_URL" && -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "ERROR: --service-metrics-url and --service-prometheus-query-url are mutually exclusive" >&2
exit 1
fi
if ! is_positive_int "$SERVICE_METRICS_CAPTURE_ATTEMPTS" || ! is_nonnegative_int "$SERVICE_METRICS_CAPTURE_RETRY_SECS" || ! is_positive_int "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS" || ! is_positive_int "$SERVICE_METRICS_MAX_TIME_SECS" || ! is_nonnegative_int "$SERVICE_METRICS_SETTLE_SECS"; then
echo "ERROR: service metrics attempt/timeout/settle options must be valid integers" >&2
exit 1
fi
}
setup_output() {
@@ -204,11 +261,16 @@ setup_output() {
RUNS_DIR="${OUT_DIR}/runs"
AGG_MEDIAN_CSV="${OUT_DIR}/aggregate_median_summary.csv"
AGG_COMPARE_CSV="${OUT_DIR}/aggregate_baseline_compare.csv"
AGG_SERVICE_METRICS_CSV="${OUT_DIR}/aggregate_service_metrics_captures.csv"
AGG_SERVICE_METRICS_DELTAS_CSV="${OUT_DIR}/aggregate_service_metrics_deltas.csv"
AGG_SERVICE_METRICS_PATH_SUMMARY_CSV="${OUT_DIR}/aggregate_service_metrics_path_summary.csv"
AGG_SERVICE_METRICS_STAGE_SUMMARY_CSV="${OUT_DIR}/aggregate_service_metrics_stage_summary.csv"
RUN_MATRIX_CSV="${OUT_DIR}/run_matrix.csv"
mkdir -p "$RUNS_DIR"
echo "concurrency,size,tool,successful_rounds,failed_rounds,median_throughput_bps,median_reqps,median_latency_ms,bucket,run_dir" > "$AGG_MEDIAN_CSV"
echo "concurrency,size,tool,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct,run_dir" > "$AGG_COMPARE_CSV"
echo "concurrency,run_dir,size,tool,round,attempt,phase,source,status,capture_attempt,raw_bytes,snapshot_bytes,status_file,snapshot_file,filter_regex,prometheus_query" > "$AGG_SERVICE_METRICS_CSV"
echo "concurrency,bucket,run_dir,baseline_csv,status" > "$RUN_MATRIX_CSV"
}
@@ -253,6 +315,17 @@ write_manifest() {
echo "dry_run=$(join_bool "$DRY_RUN")"
echo "baseline_root=${BASELINE_ROOT:-N/A}"
echo "extra_args_present=$([[ -n "$EXTRA_ARGS" ]] && echo true || echo false)"
echo "python_bin=${PYTHON_BIN}"
echo "service_metrics_url=${SERVICE_METRICS_URL:-N/A}"
echo "service_prometheus_query_url=${SERVICE_PROMETHEUS_QUERY_URL:-N/A}"
echo "service_prometheus_query=${SERVICE_PROMETHEUS_QUERY:-N/A}"
echo "service_metrics_service_name=${SERVICE_METRICS_SERVICE_NAME:-N/A}"
echo "service_metrics_filter_regex=${SERVICE_METRICS_FILTER_REGEX:-N/A}"
echo "service_metrics_capture_attempts=${SERVICE_METRICS_CAPTURE_ATTEMPTS}"
echo "service_metrics_retry_secs=${SERVICE_METRICS_CAPTURE_RETRY_SECS}"
echo "service_metrics_connect_timeout_secs=${SERVICE_METRICS_CONNECT_TIMEOUT_SECS}"
echo "service_metrics_max_time_secs=${SERVICE_METRICS_MAX_TIME_SECS}"
echo "service_metrics_settle_secs=${SERVICE_METRICS_SETTLE_SECS}"
echo "workload_label=${WORKLOAD_LABEL}"
echo "nodes=${TOPOLOGY_NODES:-N/A}"
echo "disks_per_node=${TOPOLOGY_DISKS_PER_NODE:-N/A}"
@@ -275,11 +348,16 @@ Top-level artifacts:
- run_matrix.csv: one row per concurrency run, including bucket and baseline linkage
- aggregate_median_summary.csv: merged median_summary rows from every concurrency run
- aggregate_baseline_compare.csv: merged baseline_compare rows when a matching baseline exists
- aggregate_service_metrics_captures.csv: merged service metrics capture status from every concurrency run
- aggregate_service_metrics_deltas.csv: per-round before/after service metric deltas
- aggregate_service_metrics_path_summary.csv: per-round PUT path deltas
- aggregate_service_metrics_stage_summary.csv: per-round PUT stage duration averages from deltas
- runs/cXX/: direct output directory of scripts/run_object_batch_bench_enhanced.sh
Per-concurrency directory:
- round_results.csv
- median_summary.csv
- service_metrics_captures.csv (when service metrics capture is enabled)
- baseline_compare.csv (only when a baseline CSV is supplied)
- logs/
EOF
@@ -338,12 +416,35 @@ append_aggregate_rows() {
local run_dir="$3"
local median_csv="$4"
local compare_csv="$5"
local service_metrics_csv="${run_dir}/service_metrics_captures.csv"
awk -F',' -v c="$concurrency" -v b="$bucket" -v rd="$run_dir" 'NR>1 {print c "," $1 "," $2 "," $4 "," $5 "," $6 "," $7 "," $8 "," b "," rd}' "$median_csv" >> "$AGG_MEDIAN_CSV"
if [[ -f "$compare_csv" ]]; then
awk -F',' -v c="$concurrency" -v rd="$run_dir" 'NR>1 {print c "," $1 "," $2 "," $4 "," $5 "," $6 "," $7 "," $8 "," $9 "," $10 "," $11 "," $12 "," rd}' "$compare_csv" >> "$AGG_COMPARE_CSV"
fi
if [[ -f "$service_metrics_csv" ]]; then
awk -F',' -v c="$concurrency" -v rd="$run_dir" 'NR>1 {print c "," rd "," $0}' "$service_metrics_csv" >> "$AGG_SERVICE_METRICS_CSV"
fi
}
analyze_service_metric_deltas() {
if [[ ! -s "$AGG_SERVICE_METRICS_CSV" ]]; then
return 0
fi
if [[ "$DRY_RUN" == "true" ]]; then
return 0
fi
if [[ ! -f "$SERVICE_METRICS_DELTA_SCRIPT" ]]; then
echo "ERROR: missing service metrics delta script: $SERVICE_METRICS_DELTA_SCRIPT" >&2
exit 1
fi
"$PYTHON_BIN" "$SERVICE_METRICS_DELTA_SCRIPT" \
--capture-csv "$AGG_SERVICE_METRICS_CSV" \
--delta-csv "$AGG_SERVICE_METRICS_DELTAS_CSV" \
--path-summary-csv "$AGG_SERVICE_METRICS_PATH_SUMMARY_CSV" \
--stage-summary-csv "$AGG_SERVICE_METRICS_STAGE_SUMMARY_CSV"
}
run_concurrency() {
@@ -383,6 +484,32 @@ run_concurrency() {
if [[ -n "$EXTRA_ARGS" ]]; then
cmd+=(--extra-args "$EXTRA_ARGS")
fi
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
cmd+=(--service-metrics-dir "${run_dir}/service-metrics")
fi
if [[ -n "$SERVICE_METRICS_URL" ]]; then
cmd+=(--service-metrics-url "$SERVICE_METRICS_URL")
fi
if [[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
cmd+=(--service-prometheus-query-url "$SERVICE_PROMETHEUS_QUERY_URL")
fi
if [[ -n "$SERVICE_PROMETHEUS_QUERY" ]]; then
cmd+=(--service-prometheus-query "$SERVICE_PROMETHEUS_QUERY")
fi
if [[ -n "$SERVICE_METRICS_SERVICE_NAME" ]]; then
cmd+=(--service-metrics-service-name "$SERVICE_METRICS_SERVICE_NAME")
fi
if [[ -n "$SERVICE_METRICS_FILTER_REGEX" ]]; then
cmd+=(--service-metrics-filter-regex "$SERVICE_METRICS_FILTER_REGEX")
fi
cmd+=(
--python-bin "$PYTHON_BIN"
--service-metrics-attempts "$SERVICE_METRICS_CAPTURE_ATTEMPTS"
--service-metrics-retry-secs "$SERVICE_METRICS_CAPTURE_RETRY_SECS"
--service-metrics-connect-timeout-secs "$SERVICE_METRICS_CONNECT_TIMEOUT_SECS"
--service-metrics-max-time-secs "$SERVICE_METRICS_MAX_TIME_SECS"
--service-metrics-settle-secs "$SERVICE_METRICS_SETTLE_SECS"
)
if [[ "$DRY_RUN" == "true" ]]; then
cmd+=(--dry-run)
fi
@@ -433,6 +560,10 @@ main() {
echo "Duration: $DURATION"
echo "Rounds: $ROUNDS"
echo "Cooldown secs: $COOLDOWN_SECS"
if [[ -n "$SERVICE_METRICS_URL" || -n "$SERVICE_PROMETHEUS_QUERY_URL" ]]; then
echo "Service metrics capture: enabled"
echo "Service metrics source: $([[ -n "$SERVICE_PROMETHEUS_QUERY_URL" ]] && echo prometheus_query || echo prometheus_text)"
fi
local conc_count conc_index
conc_count="$(csv_to_lines "$CONCURRENCIES" | awk 'END{print NR+0}')"
@@ -446,12 +577,22 @@ main() {
fi
done < <(csv_to_lines "$CONCURRENCIES")
analyze_service_metric_deltas
echo
echo "Stage-breakdown run finished."
echo "Artifacts written to: $OUT_DIR"
echo "Top-level summaries:"
echo " - $RUN_MATRIX_CSV"
echo " - $AGG_MEDIAN_CSV"
if [[ -s "$AGG_SERVICE_METRICS_CSV" ]]; then
echo " - $AGG_SERVICE_METRICS_CSV"
fi
if [[ -s "$AGG_SERVICE_METRICS_DELTAS_CSV" ]]; then
echo " - $AGG_SERVICE_METRICS_DELTAS_CSV"
echo " - $AGG_SERVICE_METRICS_PATH_SUMMARY_CSV"
echo " - $AGG_SERVICE_METRICS_STAGE_SUMMARY_CSV"
fi
if [[ -s "$AGG_COMPARE_CSV" ]]; then
echo " - $AGG_COMPARE_CSV"
fi