gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-02 19:39:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: harden HotPath closure evidence (#5573)

Browse Source 
* test(ecstore): cover raw shard write errors

Co-Authored-By: heihutu <heihutu@gmail.com>

* perf(ecstore): track encode payload stage peaks

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(perf): harden formal warp ABBA evidence

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-08-01 21:16:31 +08:00
committed by GitHub
parent eb4f2d61ad
commit 749cb2c1a1
11 changed files with 959 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/hotpath_warp_ab_gate.sh
+62 -2
View File
@@ -26,6 +26,7 @@ set -euo pipefail
FAIL_PCT=10
WARN_PCT=5
ALLOW_REGRESSION="false"
REQUIRE_TAIL_ERROR="false"
MARKDOWN_OUT=""
EXEMPTION_REASON="deliberate correctness tradeoff"
declare -a COMPARE_CSVS=()
@@ -39,6 +40,8 @@ Usage: hotpath_warp_ab_gate.sh --compare-csv <file> [--compare-csv <file> ...] [
--warn-pct <n> Regression budget that warns (default 5).
--allow-regression Downgrade every FAIL to an exempted WARN (deliberate
correctness tradeoff); the gate then always exits 0.
--require-tail-error Require the extended p90/p99/error-rate schema and
reject missing tail/error or zero-success evidence.
--exemption-reason <s> Reason recorded when --allow-regression is set.
--markdown <file> Also write the result table as Markdown to <file>.
-h, --help Show this help.
@@ -51,6 +54,7 @@ while [[ $# -gt 0 ]]; do
--fail-pct) FAIL_PCT="$2"; shift 2 ;;
--warn-pct) WARN_PCT="$2"; shift 2 ;;
--allow-regression) ALLOW_REGRESSION="true"; shift ;;
--require-tail-error) REQUIRE_TAIL_ERROR="true"; shift ;;
--exemption-reason) EXEMPTION_REASON="$2"; shift 2 ;;
--markdown) MARKDOWN_OUT="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
@@ -67,10 +71,24 @@ for csv in "${COMPARE_CSVS[@]}"; do
[[ -f "$csv" ]] || { echo "error: compare CSV not found: $csv" >&2; exit 2; }
done
if [[ "$REQUIRE_TAIL_ERROR" == "true" ]]; then
strict_header='size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct,new_median_p90_latency_ms,baseline_median_p90_latency_ms,delta_p90_latency_pct,new_median_p99_latency_ms,baseline_median_p99_latency_ms,delta_p99_latency_pct,new_ok_rounds,baseline_ok_rounds,new_failed_rounds,baseline_failed_rounds,new_error_rate_pct,baseline_error_rate_pct,delta_error_rate_pct'
for csv in "${COMPARE_CSVS[@]}"; do
[[ "$(head -n1 "$csv")" == "$strict_header" ]] || {
echo "error: strict mode requires the current tail/error compare schema: $csv" >&2
exit 2
}
awk 'NR == 2 { found = 1 } END { exit found ? 0 : 1 }' "$csv" || {
echo "error: strict mode requires at least one comparison row: $csv" >&2
exit 2
}
done
fi
# One awk pass over all CSVs. Emits TSV rows "verdict\tworkload\tmetric\tdelta"
# on stdout and a final "OVERALL\t<verdict>" line. Verdict is PASS/WARN/FAIL.
gate_output="$(
awk -v fail_pct="$FAIL_PCT" -v warn_pct="$WARN_PCT" '
awk -v fail_pct="$FAIL_PCT" -v warn_pct="$WARN_PCT" -v strict="$REQUIRE_TAIL_ERROR" '
function classify(delta, higher_is_better, regress) {
if (delta == "" || delta == "N/A") return "SKIP"
# Signed regression magnitude: positive means "worse than baseline".
@@ -85,15 +103,57 @@ gate_output="$(
rank = (v == "FAIL") ? 3 : (v == "WARN") ? 2 : 1
if (rank > worst) worst = rank
}
function contract_failure(workload, metric) {
print "FAIL\t" workload "\t" metric "\tinvalid evidence"
if (3 > worst) worst = 3
}
function decimal(value) {
return value ~ /^-?[0-9]+([.][0-9]+)?$/
}
function count(value) {
return value ~ /^[0-9]+$/
}
function error_rate(value) {
return decimal(value) && value + 0 >= 0 && value + 0 <= 100
}
function error_delta(value) {
return decimal(value) && value + 0 >= -100 && value + 0 <= 100
}
function abs(value) {
return value < 0 ? -value : value
}
BEGIN { worst = 1 }
FNR == 1 { next } # skip each file header
{
n = split($0, f, ",")
if (n < 12) next
if (n < 12) {
if (strict == "true") contract_failure(FILENAME, "compare-schema")
next
}
workload = f[1] "/" f[2] "@" f[3] # size/tool@concurrency
emit(classify(f[6], 1), workload, "reqps", f[6])
emit(classify(f[9], 0), workload, "latency", f[9])
emit(classify(f[12], 1), workload, "throughput", f[12])
if (strict == "true") {
valid = n == 25
for (i = 4; i <= 18; i++) {
if (!decimal(f[i])) valid = 0
}
for (i = 19; i <= 22; i++) {
if (!count(f[i])) valid = 0
}
if (f[19] + 0 == 0 || f[20] + 0 == 0 || !error_rate(f[23]) || !error_rate(f[24]) || !error_delta(f[25])) valid = 0
new_error_rate = f[21] / (f[19] + f[21]) * 100
baseline_error_rate = f[22] / (f[20] + f[22]) * 100
if (abs(f[23] - new_error_rate) > 0.005 || abs(f[24] - baseline_error_rate) > 0.005 || abs(f[25] - (new_error_rate - baseline_error_rate)) > 0.005) valid = 0
if (!valid) {
contract_failure(workload, "tail-error-evidence")
next
}
emit(classify(f[15], 0), workload, "p90-latency", f[15])
emit(classify(f[18], 0), workload, "p99-latency", f[18])
emit(classify(f[25], 0), workload, "error-rate", f[25])
}
}
END {
print "OVERALL\t" (worst == 3 ? "FAIL" : worst == 2 ? "WARN" : "PASS")
scripts/run_hotpath_warp_abba.sh
+141 -24
View File
@@ -18,6 +18,8 @@ GATE="${PROJECT_ROOT}/scripts/hotpath_warp_ab_gate.sh"
BASELINE_BIN=""
CANDIDATE_BIN=""
BASELINE_REVISION="unknown"
CANDIDATE_REVISION="unknown"
ENDPOINT=""
DEPLOY_HOOK=""
HEALTH_PATH="/health"
@@ -32,6 +34,7 @@ CONCURRENCY=8
DURATION="60s"
ROUNDS=3
COOLDOWN_SECS=20
DATASET_SETUP_DURATION="10s"
HEALTH_TIMEOUT_SECS=180
FAIL_PCT=10
WARN_PCT=5
@@ -39,6 +42,7 @@ ALLOW_REGRESSION=false
EXEMPTION_REASON="deliberate correctness tradeoff"
OUT_DIR="${PROJECT_ROOT}/target/hotpath-abba/$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)"
DRY_RUN=false
ALLOW_UNMANAGED_EXTERNAL=false
WORKLOADS=(
"put-4kib|put|4KiB"
@@ -61,11 +65,15 @@ and drive-sync cell.
Required:
--baseline-bin <path> Baseline RustFS binary.
--candidate-bin <path> Candidate RustFS binary.
--baseline-revision <id> Source revision for the baseline binary.
--candidate-revision <id> Source revision for the candidate binary.
Local Linux runner mode:
--address <host:port> Local RustFS address (default 127.0.0.1:9000).
--disks <n> Throwaway local disks per node (default 4).
--data-root <path> Local disk root (default /tmp/rustfs-hotpath-abba).
A reused run namespace is rejected; this runner
never deletes existing benchmark disks.
Production / cluster mode:
--endpoint <host:port> Existing cluster endpoint. Enables external mode.
@@ -74,12 +82,27 @@ Production / cluster mode:
HOTPATH_ABBA_PHASE=baseline|candidate
HOTPATH_ABBA_BINARY=<baseline/candidate binary>
HOTPATH_ABBA_DRIVE_SYNC=true|false
HOTPATH_ABBA_WORKLOAD=<workload name>
HOTPATH_ABBA_MODE=put|get|mixed
HOTPATH_ABBA_SIZE=<object size>
HOTPATH_ABBA_CELL_ID=<sync/workload/leg>
HOTPATH_ABBA_DATASET_NAMESPACE=<run namespace>
HOTPATH_ABBA_BUCKET=<per-leg benchmark bucket>
HOTPATH_ABBA_DEPLOY_EVIDENCE_FILE=<path>
The hook must deploy the selected binary, reset or
provision HOTPATH_ABBA_BUCKET for the requested
mode, then write a non-empty evidence file.
--allow-unmanaged-external Preserve legacy external mode without a deploy
hook. Its output is not formal ABBA evidence.
--health-path <path> Readiness path (default /health).
Benchmark:
--duration <dur> warp duration per cell (default 60s).
--rounds <n> rounds per cell; must be >= 3 (default 3).
--cooldown <n> cooldown seconds between rounds/sizes (default 20).
--dataset-setup-duration <dur>
isolated Warp PUT warm-up for get/mixed legs
(default 10s; not included in the measurement).
--concurrency <n> warp concurrency (default 8).
--warp-bin <path> warp binary (default warp).
@@ -134,6 +157,8 @@ while [[ $# -gt 0 ]]; do
case "$1" in
--baseline-bin) BASELINE_BIN="$2"; shift 2 ;;
--candidate-bin) CANDIDATE_BIN="$2"; shift 2 ;;
--baseline-revision) BASELINE_REVISION="$2"; shift 2 ;;
--candidate-revision) CANDIDATE_REVISION="$2"; shift 2 ;;
--endpoint) ENDPOINT="$2"; shift 2 ;;
--deploy-hook) DEPLOY_HOOK="$2"; shift 2 ;;
--health-path) HEALTH_PATH="$2"; shift 2 ;;
@@ -148,6 +173,7 @@ while [[ $# -gt 0 ]]; do
--duration) DURATION="$2"; shift 2 ;;
--rounds) ROUNDS="$2"; shift 2 ;;
--cooldown) COOLDOWN_SECS="$2"; shift 2 ;;
--dataset-setup-duration) DATASET_SETUP_DURATION="$2"; shift 2 ;;
--health-timeout) HEALTH_TIMEOUT_SECS="$2"; shift 2 ;;
--fail-pct) FAIL_PCT="$2"; shift 2 ;;
--warn-pct) WARN_PCT="$2"; shift 2 ;;
@@ -155,6 +181,7 @@ while [[ $# -gt 0 ]]; do
--exemption-reason) EXEMPTION_REASON="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
--allow-unmanaged-external) ALLOW_UNMANAGED_EXTERNAL=true; shift ;;
-h|--help) usage; exit 0 ;;
*) die "unknown argument: $1" ;;
esac
@@ -170,6 +197,9 @@ validate_positive_int "$WARN_PCT" "--warn-pct"
[[ "$ROUNDS" -ge 3 ]] || die "--rounds must be >= 3 for formal ABBA evidence"
[[ -n "$BASELINE_BIN" ]] || die "--baseline-bin is required"
[[ -n "$CANDIDATE_BIN" ]] || die "--candidate-bin is required"
if [[ "$DRY_RUN" != "true" ]] && { [[ -z "$BASELINE_REVISION" || "$BASELINE_REVISION" == "unknown" || -z "$CANDIDATE_REVISION" || "$CANDIDATE_REVISION" == "unknown" ]]; }; then
die "formal runs require non-unknown --baseline-revision and --candidate-revision"
fi
[[ "$DRY_RUN" == "true" || -x "$BASELINE_BIN" ]] || die "baseline binary is not executable: $BASELINE_BIN"
[[ "$DRY_RUN" == "true" || -x "$CANDIDATE_BIN" ]] || die "candidate binary is not executable: $CANDIDATE_BIN"
[[ -x "$ENHANCED_BENCH" ]] || die "missing load driver: $ENHANCED_BENCH"
@@ -182,12 +212,24 @@ EXTERNAL=false
if [[ -n "$ENDPOINT" ]]; then
EXTERNAL=true
ADDRESS="$ENDPOINT"
[[ -n "$DEPLOY_HOOK" ]] || log "warning: external mode without --deploy-hook; binaries must be swapped out of band"
if [[ -z "$DEPLOY_HOOK" && "$ALLOW_UNMANAGED_EXTERNAL" != "true" ]]; then
die "external mode requires --deploy-hook to establish each leg's binary and fresh dataset namespace; use --allow-unmanaged-external only for non-formal legacy runs"
fi
[[ -n "$DEPLOY_HOOK" ]] || log "warning: unmanaged external mode does not establish binary or data isolation and is not formal ABBA evidence"
fi
mkdir -p "$OUT_DIR"
SERVER_LOG_DIR="$OUT_DIR/server-logs"
DEPLOY_EVIDENCE_DIR="$OUT_DIR/deploy-evidence"
DATASET_NAMESPACE="$(basename "$OUT_DIR" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-' | sed -E 's/^-+//; s/-+$//' | cut -c1-24)"
[[ -n "$DATASET_NAMESPACE" ]] || DATASET_NAMESPACE="run"
RUN_DATA_ROOT="$DATA_ROOT/$DATASET_NAMESPACE"
if [[ "$EXTERNAL" != "true" && "$DRY_RUN" != "true" ]]; then
mkdir -p "$DATA_ROOT"
mkdir "$RUN_DATA_ROOT" || die "local run namespace already exists: $RUN_DATA_ROOT; choose a new --out-dir or --data-root instead of reusing benchmark disks"
fi
run mkdir -p "$SERVER_LOG_DIR"
run mkdir -p "$DEPLOY_EVIDENCE_DIR"
SERVER_PID=""
SERVER_LOG=""
@@ -241,29 +283,47 @@ phase_for_leg() {
esac
}
bucket_for_leg() {
local sync_label="$1" workload="$2" leg="$3" bucket_leg
bucket_leg="$(printf '%s' "$leg" | tr '[:upper:]' '[:lower:]')"
printf 'rustfs-abba-%s-%s-%s-%s' "$DATASET_NAMESPACE" "$sync_label" "$workload" "$bucket_leg"
}
bring_up() {
local leg="$1" drive_sync="$2"
local leg="$1" drive_sync="$2" workload="$3" mode="$4" size="$5" sync_label="$6" bucket="$7"
local phase bin
phase="$(phase_for_leg "$leg")"
bin="$(binary_for_leg "$leg")"
if [[ "$EXTERNAL" == "true" ]]; then
if [[ -n "$DEPLOY_HOOK" ]]; then
log "deploy hook: leg=$leg phase=$phase drive_sync=$drive_sync"
HOTPATH_ABBA_LEG="$leg" HOTPATH_ABBA_PHASE="$phase" HOTPATH_ABBA_BINARY="$bin" HOTPATH_ABBA_DRIVE_SYNC="$drive_sync" \
log "deploy hook: leg=$leg phase=$phase drive_sync=$drive_sync workload=$workload"
local evidence_file="$DEPLOY_EVIDENCE_DIR/$sync_label-$workload-$leg.env"
local binary_sha
binary_sha="$(sha256_file "$bin")"
[[ "$binary_sha" != "unknown" ]] || die "cannot attest external binary without a SHA-256 tool"
HOTPATH_ABBA_LEG="$leg" HOTPATH_ABBA_PHASE="$phase" HOTPATH_ABBA_BINARY="$bin" HOTPATH_ABBA_BINARY_SHA256="$binary_sha" HOTPATH_ABBA_DRIVE_SYNC="$drive_sync" HOTPATH_ABBA_WORKLOAD="$workload" HOTPATH_ABBA_MODE="$mode" HOTPATH_ABBA_SIZE="$size" HOTPATH_ABBA_CELL_ID="$sync_label/$workload/$leg" HOTPATH_ABBA_DATASET_NAMESPACE="$DATASET_NAMESPACE" HOTPATH_ABBA_BUCKET="$bucket" HOTPATH_ABBA_DEPLOY_EVIDENCE_FILE="$evidence_file" \
run bash -c "$DEPLOY_HOOK"
if [[ "$DRY_RUN" != "true" ]]; then
[[ -s "$evidence_file" ]] || die "deploy hook did not write evidence for $sync_label/$workload/$leg: $evidence_file"
[[ "$(awk -F= '$1=="leg" {print substr($0, length($1) + 2)}' "$evidence_file")" == "$leg" ]] || die "deploy evidence leg does not match $sync_label/$workload/$leg"
[[ "$(awk -F= '$1=="phase" {print substr($0, length($1) + 2)}' "$evidence_file")" == "$phase" ]] || die "deploy evidence phase does not match $sync_label/$workload/$leg"
[[ "$(awk -F= '$1=="bucket" {print substr($0, length($1) + 2)}' "$evidence_file")" == "$bucket" ]] || die "deploy evidence bucket does not match $sync_label/$workload/$leg"
[[ "$(awk -F= '$1=="binary_sha256" {print substr($0, length($1) + 2)}' "$evidence_file")" == "$binary_sha" ]] || die "deploy evidence binary SHA-256 does not match $sync_label/$workload/$leg"
[[ "$(awk -F= '$1=="dataset_reset" {print substr($0, length($1) + 2)}' "$evidence_file")" == "true" ]] || die "deploy evidence must attest dataset_reset=true for $sync_label/$workload/$leg"
fi
fi
wait_health
return 0
fi
local node_dir="$DATA_ROOT/$leg-sync-$drive_sync"
local node_dir="$RUN_DATA_ROOT/$workload-$leg-sync-$drive_sync"
local disks=() d
for ((d = 1; d <= DISKS; d++)); do
disks+=("$node_dir/d$d")
done
run mkdir -p "${disks[@]}"
SERVER_LOG="$SERVER_LOG_DIR/$leg-sync-$drive_sync.log"
SERVER_LOG="$SERVER_LOG_DIR/$workload-$leg-sync-$drive_sync.log"
if [[ "$DRY_RUN" == "true" ]]; then
{ printf 'DRY-RUN: RUSTFS_DRIVE_SYNC_ENABLE=%s %q server' "$drive_sync" "$bin"
@@ -272,7 +332,7 @@ bring_up() {
return 0
fi
cat >"$SERVER_LOG_DIR/$leg-sync-$drive_sync.env" <<EOF
cat >"$SERVER_LOG_DIR/$workload-$leg-sync-$drive_sync.env" <<EOF
leg=$leg
phase=$phase
drive_sync=$drive_sync
@@ -297,33 +357,78 @@ EOF
}
measure() {
local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" baseline_csv="${6:-}"
local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" bucket="$6" baseline_csv="${7:-}"
local cell="$OUT_DIR/$workload/$sync_label/$leg"
local args=(
--tool warp --warp-bin "$WARP_BIN" --warp-mode "$mode"
--endpoint "$ADDRESS" --access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY"
--region "$REGION" --sizes "$size" --concurrency "$CONCURRENCY"
--region "$REGION" --bucket "$bucket" --sizes "$size" --concurrency "$CONCURRENCY"
--duration "$DURATION" --rounds "$ROUNDS" --cooldown-secs "$COOLDOWN_SECS"
--out-dir "$cell"
)
[[ "$mode" == "put" ]] || args+=(--extra-args "--noclear")
[[ -n "$baseline_csv" ]] && args+=(--baseline-csv "$baseline_csv")
run "$ENHANCED_BENCH" "${args[@]}" >&2
echo "$cell"
}
prepare_dataset() {
local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" bucket="$6"
[[ "$mode" != "put" ]] || return 0
local setup_cell="$OUT_DIR/$workload/$sync_label/$leg/dataset-setup"
local args=(
--tool warp --warp-bin "$WARP_BIN" --warp-mode put
--endpoint "$ADDRESS" --access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY"
--region "$REGION" --bucket "$bucket" --sizes "$size" --concurrency "$CONCURRENCY"
--duration "$DATASET_SETUP_DURATION" --rounds 1 --cooldown-secs 0
--extra-args "--noclear"
--out-dir "$setup_cell"
)
log "preparing isolated dataset: $sync_label/$workload/$leg bucket=$bucket"
run "$ENHANCED_BENCH" "${args[@]}" >&2
}
write_schedule_header() {
echo "sync_label,drive_sync,workload,mode,size,leg,phase,binary,out_dir" >"$OUT_DIR/abba_schedule.csv"
echo "sync_label,drive_sync,workload,mode,size,leg,phase,binary,out_dir,bucket,dataset_setup" >"$OUT_DIR/abba_schedule.csv"
}
append_schedule() {
local sync_label="$1" drive_sync="$2" workload="$3" mode="$4" size="$5" leg="$6"
local sync_label="$1" drive_sync="$2" workload="$3" mode="$4" size="$5" leg="$6" bucket="$7"
local phase bin
phase="$(phase_for_leg "$leg")"
bin="$(binary_for_leg "$leg")"
echo "$sync_label,$drive_sync,$workload,$mode,$size,$leg,$phase,$bin,$OUT_DIR/$workload/$sync_label/$leg" >>"$OUT_DIR/abba_schedule.csv"
local dataset_setup="none"
[[ "$mode" == "put" ]] || dataset_setup="warp-put"
echo "$sync_label,$drive_sync,$workload,$mode,$size,$leg,$phase,$bin,$OUT_DIR/$workload/$sync_label/$leg,$bucket,$dataset_setup" >>"$OUT_DIR/abba_schedule.csv"
}
sha256_file() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1" | awk '{print $1}'
elif command -v shasum >/dev/null 2>&1; then
shasum -a 256 "$1" | awk '{print $1}'
else
echo unknown
fi
}
isolation_mode() {
if [[ "$EXTERNAL" != "true" ]]; then
echo local-per-cell-disks
elif [[ -n "$DEPLOY_HOOK" ]]; then
echo deploy-hook-attested
else
echo unmanaged
fi
}
write_manifest() {
local baseline_sha candidate_sha workloads matrix
baseline_sha="$(sha256_file "$BASELINE_BIN" 2>/dev/null || echo unknown)"
candidate_sha="$(sha256_file "$CANDIDATE_BIN" 2>/dev/null || echo unknown)"
workloads="$(IFS=';'; echo "${WORKLOADS[*]}")"
matrix="$(IFS=';'; echo "${DRIVE_SYNC_MATRIX[*]}")"
cat >"$OUT_DIR/manifest.env" <<EOF
generated_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)
runner=$(uname -srm 2>/dev/null || echo unknown)
@@ -334,7 +439,19 @@ cooldown_secs=$COOLDOWN_SECS
concurrency=$CONCURRENCY
baseline_bin=$BASELINE_BIN
candidate_bin=$CANDIDATE_BIN
baseline_revision=$BASELINE_REVISION
candidate_revision=$CANDIDATE_REVISION
baseline_binary_sha256=$baseline_sha
candidate_binary_sha256=$candidate_sha
workloads=$workloads
drive_sync_matrix=$matrix
external=$EXTERNAL
external_isolation=$(isolation_mode)
dataset_namespace=$DATASET_NAMESPACE
local_run_data_root=$RUN_DATA_ROOT
bucket_isolation=per-leg
bucket_prefix=rustfs-abba-$DATASET_NAMESPACE
dataset_setup=get-and-mixed-via-warp-put
endpoint=$ADDRESS
warp_version=$("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown)
EOF
@@ -349,37 +466,37 @@ write_schedule_header
for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do
IFS='|' read -r sync_label drive_sync <<<"$ds_spec"
for leg in A1 B1 B2 A2; do
log "=== $sync_label leg $leg ($(phase_for_leg "$leg")) ==="
bring_up "$leg" "$drive_sync"
for wl_spec in "${WORKLOADS[@]}"; do
IFS='|' read -r workload mode size <<<"$wl_spec"
append_schedule "$sync_label" "$drive_sync" "$workload" "$mode" "$size" "$leg"
for wl_spec in "${WORKLOADS[@]}"; do
IFS='|' read -r workload mode size <<<"$wl_spec"
for leg in A1 B1 B2 A2; do
log "=== $sync_label $workload leg $leg ($(phase_for_leg "$leg")) ==="
bucket="$(bucket_for_leg "$sync_label" "$workload" "$leg")"
bring_up "$leg" "$drive_sync" "$workload" "$mode" "$size" "$sync_label" "$bucket"
prepare_dataset "$leg" "$workload" "$mode" "$size" "$sync_label" "$bucket"
append_schedule "$sync_label" "$drive_sync" "$workload" "$mode" "$size" "$leg" "$bucket"
baseline_csv=""
if [[ "$leg" != "A1" ]]; then
baseline_csv="$OUT_DIR/$workload/$sync_label/A1/median_summary.csv"
fi
cell="$(measure "$leg" "$workload" "$mode" "$size" "$sync_label" "$baseline_csv")"
cell="$(measure "$leg" "$workload" "$mode" "$size" "$sync_label" "$bucket" "$baseline_csv")"
case "$leg" in
B1|B2) CANDIDATE_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;;
A2) DRIFT_COMPARE_CSVS+=("$cell/baseline_compare.csv") ;;
esac
tear_down
done
tear_down
done
done
gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/candidate_gate.md")
gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --require-tail-error --markdown "$OUT_DIR/candidate_gate.md")
for csv in "${CANDIDATE_COMPARE_CSVS[@]}"; do
gate_args+=(--compare-csv "$csv")
done
[[ "$ALLOW_REGRESSION" == "true" ]] && gate_args+=(--allow-regression --exemption-reason "$EXEMPTION_REASON")
drift_gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/baseline_drift_gate.md")
drift_gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --require-tail-error --markdown "$OUT_DIR/baseline_drift_gate.md")
for csv in "${DRIFT_COMPARE_CSVS[@]}"; do
drift_gate_args+=(--compare-csv "$csv")
done
scripts/run_object_batch_bench_enhanced.sh
+17 -4
View File
@@ -1289,7 +1289,7 @@ compare_baseline() {
return
fi
echo "size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct" > "$COMPARE_CSV"
echo "size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct,new_median_p90_latency_ms,baseline_median_p90_latency_ms,delta_p90_latency_pct,new_median_p99_latency_ms,baseline_median_p99_latency_ms,delta_p99_latency_pct,new_ok_rounds,baseline_ok_rounds,new_failed_rounds,baseline_failed_rounds,new_error_rate_pct,baseline_error_rate_pct,delta_error_rate_pct" > "$COMPARE_CSV"
awk -F',' '
NR==FNR {
@@ -1298,22 +1298,35 @@ compare_baseline() {
b_req[key]=$7
b_lat[key]=$8
b_thr[key]=$6
b_p90[key]=$9
b_p99[key]=$10
b_ok[key]=$4
b_fail[key]=$5
next
}
FNR==1 {next}
{
key=$1
n_thr=$6; n_req=$7; n_lat=$8
n_thr=$6; n_req=$7; n_lat=$8; n_p90=$9; n_p99=$10; n_ok=$4; n_fail=$5
br=(key in b_req)?b_req[key]:"N/A"
bl=(key in b_lat)?b_lat[key]:"N/A"
bt=(key in b_thr)?b_thr[key]:"N/A"
bp90=(key in b_p90)?b_p90[key]:"N/A"
bp99=(key in b_p99)?b_p99[key]:"N/A"
bok=(key in b_ok)?b_ok[key]:"N/A"
bfail=(key in b_fail)?b_fail[key]:"N/A"
dr="N/A"; dl="N/A"; dt="N/A"
dr="N/A"; dl="N/A"; dt="N/A"; dp90="N/A"; dp99="N/A"; ne="N/A"; be="N/A"; de="N/A"
if (br!="N/A" && n_req!="N/A" && br+0!=0) dr=sprintf("%.2f", ((n_req-br)/br)*100)
if (bl!="N/A" && n_lat!="N/A" && bl+0!=0) dl=sprintf("%.2f", ((n_lat-bl)/bl)*100)
if (bt!="N/A" && n_thr!="N/A" && bt+0!=0) dt=sprintf("%.2f", ((n_thr-bt)/bt)*100)
if (bp90!="N/A" && n_p90!="N/A" && bp90+0!=0) dp90=sprintf("%.2f", ((n_p90-bp90)/bp90)*100)
if (bp99!="N/A" && n_p99!="N/A" && bp99+0!=0) dp99=sprintf("%.2f", ((n_p99-bp99)/bp99)*100)
if (n_ok!="N/A" && n_fail!="N/A" && n_ok+n_fail>0) ne=sprintf("%.2f", (n_fail/(n_ok+n_fail))*100)
if (bok!="N/A" && bfail!="N/A" && bok+bfail>0) be=sprintf("%.2f", (bfail/(bok+bfail))*100)
if (ne!="N/A" && be!="N/A") de=sprintf("%.2f", ne-be)
print key "," $2 "," $3 "," n_req "," br "," dr "," n_lat "," bl "," dl "," n_thr "," bt "," dt
print key "," $2 "," $3 "," n_req "," br "," dr "," n_lat "," bl "," dl "," n_thr "," bt "," dt "," n_p90 "," bp90 "," dp90 "," n_p99 "," bp99 "," dp99 "," n_ok "," bok "," n_fail "," bfail "," ne "," be "," de
}
' "$BASELINE_CSV" "$MEDIAN_CSV" >> "$COMPARE_CSV"
}
scripts/test_hotpath_warp_ab_gate.sh
Executable
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GATE="${SCRIPT_DIR}/hotpath_warp_ab_gate.sh"
TMP_DIR="$(mktemp -d)"
cleanup() {
rm -rf "$TMP_DIR"
}
trap cleanup EXIT
header='size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct,new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct,new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct,new_median_p90_latency_ms,baseline_median_p90_latency_ms,delta_p90_latency_pct,new_median_p99_latency_ms,baseline_median_p99_latency_ms,delta_p99_latency_pct,new_ok_rounds,baseline_ok_rounds,new_failed_rounds,baseline_failed_rounds,new_error_rate_pct,baseline_error_rate_pct,delta_error_rate_pct'
printf '%s\n' "$header" >"$TMP_DIR/equal-budget.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,9,10,1,0,10.00,0.00,10.00' >>"$TMP_DIR/equal-budget.csv"
"$GATE" --require-tail-error --fail-pct 10 --warn-pct 5 --compare-csv "$TMP_DIR/equal-budget.csv" >/dev/null
printf '%s\n' "$header" >"$TMP_DIR/rounded-error.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,2,5,1,1,33.33,16.67,16.67' >>"$TMP_DIR/rounded-error.csv"
if "$GATE" --require-tail-error --fail-pct 20 --warn-pct 10 --compare-csv "$TMP_DIR/rounded-error.csv" >/dev/null; then
:
else
echo "expected strict gate to accept error-rate deltas rounded from raw fractions" >&2
exit 1
fi
printf '%s\n' "$header" >"$TMP_DIR/p99-regression.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,4,3,33.33,3,3,0,0,0.00,0.00,0.00' >>"$TMP_DIR/p99-regression.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/p99-regression.csv" >/dev/null 2>&1; then
echo "expected p99-only regression to fail the strict gate" >&2
exit 1
fi
printf '%s\n' "$header" >"$TMP_DIR/missing-evidence.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,N/A,N/A,N/A,N/A,N/A,N/A,0,3,3,0,N/A,0.00,N/A' >>"$TMP_DIR/missing-evidence.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/missing-evidence.csv" >/dev/null 2>&1; then
echo "expected missing tail/error evidence or zero successes to fail the strict gate" >&2
exit 1
fi
printf '%s\n' 'size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct' >"$TMP_DIR/old-schema.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00' >>"$TMP_DIR/old-schema.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/old-schema.csv" >/dev/null 2>&1; then
echo "expected strict gate to reject the old comparison schema" >&2
exit 1
fi
printf '%s\n' "$header" >"$TMP_DIR/header-only.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/header-only.csv" >/dev/null 2>&1; then
echo "expected strict gate to reject a comparison without evidence rows" >&2
exit 1
fi
printf '%s\n' "$header" >"$TMP_DIR/malformed-number.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,3,3,0,0,0x0,0.00,0.00' >>"$TMP_DIR/malformed-number.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/malformed-number.csv" >/dev/null 2>&1; then
echo "expected strict gate to reject malformed numeric evidence" >&2
exit 1
fi
printf '%s\n' "$header" >"$TMP_DIR/out-of-range-error.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,3,3,0,0,101.00,0.00,101.00' >>"$TMP_DIR/out-of-range-error.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/out-of-range-error.csv" >/dev/null 2>&1; then
echo "expected strict gate to reject out-of-range error evidence" >&2
exit 1
fi
printf '%s\n' "$header" >"$TMP_DIR/inconsistent-error.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,3,3,0,0,100.00,0.00,0.00' >>"$TMP_DIR/inconsistent-error.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/inconsistent-error.csv" >/dev/null 2>&1; then
echo "expected strict gate to reject error rates inconsistent with round counts" >&2
exit 1
fi
printf '%s\n' "$header,unexpected" >"$TMP_DIR/extra-column.csv"
printf '%s\n' '1MiB,warp,1,100,100,0.00,1,1,0.00,100,100,0.00,2,2,0.00,3,3,0.00,3,3,0,0,0.00,0.00,0.00,unexpected' >>"$TMP_DIR/extra-column.csv"
if "$GATE" --require-tail-error --compare-csv "$TMP_DIR/extra-column.csv" >/dev/null 2>&1; then
echo "expected strict gate to reject extra columns" >&2
exit 1
fi
echo "hotpath warp A/B gate tests passed"
scripts/test_hotpath_warp_abba.sh
Executable
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RUNNER="${SCRIPT_DIR}/run_hotpath_warp_abba.sh"
TMP_DIR="$(mktemp -d)"
OUT_DIR="${TMP_DIR}/abba"
TRACE_FILE="${TMP_DIR}/abba-trace.log"
cleanup() {
rm -rf "$TMP_DIR"
}
trap cleanup EXIT
"$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--baseline-revision baseline-test \
--candidate-revision candidate-test \
--warp-bin /usr/bin/true \
--rounds 3 \
--out-dir "$OUT_DIR" \
--dry-run >"$TRACE_FILE" 2>&1
awk -F',' '
NR == 1 {
if ($1 != "sync_label" || $6 != "leg" || $7 != "phase" || $10 != "bucket" || $11 != "dataset_setup") exit 1
next
}
{
key = $1 SUBSEP $3
seen[key]++
leg[key, seen[key]] = $6
phase[key, seen[key]] = $7
bucket[key, seen[key]] = $10
}
END {
for (key in seen) {
if (seen[key] != 4) exit 1
if (leg[key, 1] != "A1" || leg[key, 2] != "B1" || leg[key, 3] != "B2" || leg[key, 4] != "A2") exit 1
if (phase[key, 1] != "baseline" || phase[key, 2] != "candidate" || phase[key, 3] != "candidate" || phase[key, 4] != "baseline") exit 1
for (slot = 1; slot <= 4; slot++) {
if (length(bucket[key, slot]) > 63 || bucket[key, slot] !~ /^rustfs-abba-[a-z0-9-]+$/ || bucket[key, slot] in all_buckets) exit 1
all_buckets[bucket[key, slot]] = 1
}
cells++
}
exit cells == 12 ? 0 : 1
}
' "$OUT_DIR/abba_schedule.csv"
rg -qx 'schedule=ABBA' "$OUT_DIR/manifest.env"
rg -qx 'rounds=3' "$OUT_DIR/manifest.env"
rg -qx 'baseline_revision=baseline-test' "$OUT_DIR/manifest.env"
rg -qx 'candidate_revision=candidate-test' "$OUT_DIR/manifest.env"
rg -qx 'external_isolation=local-per-cell-disks' "$OUT_DIR/manifest.env"
rg -qx 'bucket_isolation=per-leg' "$OUT_DIR/manifest.env"
rg -qx 'dataset_setup=get-and-mixed-via-warp-put' "$OUT_DIR/manifest.env"
[[ "$(rg -c -- '--extra-args --noclear' "$TRACE_FILE")" == "64" ]]
! rg -q -- 'rustfs-bench' "$TRACE_FILE"
if "$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--warp-bin /usr/bin/true \
--rounds 3 \
--out-dir "${TMP_DIR}/missing-revisions" >/dev/null 2>&1; then
echo "expected formal mode to reject unknown binary revisions" >&2
exit 1
fi
mkdir -p "${TMP_DIR}/data/reused/put-4kib-A1-sync-true"
printf '%s\n' sentinel >"${TMP_DIR}/data/reused/put-4kib-A1-sync-true/sentinel"
if "$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--baseline-revision baseline-test \
--candidate-revision candidate-test \
--warp-bin /usr/bin/true \
--rounds 3 \
--data-root "${TMP_DIR}/data" \
--out-dir "${TMP_DIR}/reused" >/dev/null 2>&1; then
echo "expected local mode to reject a reused run namespace" >&2
exit 1
fi
if "$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--endpoint 127.0.0.1:9000 \
--warp-bin /usr/bin/true \
--rounds 3 \
--out-dir "${TMP_DIR}/external" \
--dry-run >/dev/null 2>&1; then
echo "expected formal external mode to require a deploy hook" >&2
exit 1
fi
if "$RUNNER" \
--baseline-bin /usr/bin/true \
--candidate-bin /usr/bin/true \
--baseline-revision baseline-test \
--candidate-revision candidate-test \
--endpoint 127.0.0.1:9000 \
--deploy-hook ':' \
--warp-bin /usr/bin/true \
--rounds 3 \
--out-dir "${TMP_DIR}/missing-deploy-evidence" >/dev/null 2>&1; then
echo "expected a formal external deploy hook to write evidence" >&2
exit 1
fi
echo "hotpath warp ABBA tests passed"
scripts/test_object_batch_bench_enhanced.sh
+31
View File
@@ -102,3 +102,34 @@ chmod +x "$FAKE_WARP"
--require-server-provenance >/dev/null 2>&1
rg -q '^32767B,warp,1,1,128,ok,0,[^,]+,[^,]+,653.90 MiB/s,685663846.400000,20925.58,3.5 ms,3.500000,[^,]+,3.6 ms,3.600000,24.1 ms,24.100000$' "${TMP_DIR}/fake-warp-run/round_results.csv"
"$RUNNER" \
--tool warp \
--endpoint http://127.0.0.1:9000 \
--access-key test-access \
--secret-key test-secret \
--sizes 32767B \
--rounds 1 \
--retry-per-round 1 \
--cooldown-secs 0 \
--duration 1s \
--out-dir "${TMP_DIR}/fake-warp-candidate" \
--warp-bin "$FAKE_WARP" \
--baseline-csv "${TMP_DIR}/fake-warp-run/median_summary.csv" \
--server-image-ref rustfs/rustfs:bench \
--server-image-digest sha256:0123456789abcdef \
--server-revision 9f61bad94 \
--require-server-provenance >/dev/null 2>&1
awk -F',' '
NR == 1 {
if (NF != 25 || $15 != "delta_p90_latency_pct" || $18 != "delta_p99_latency_pct" || $25 != "delta_error_rate_pct") exit 1
}
NR == 2 {
if ($15 != "0.00" || $18 != "0.00" || $23 != "0.00" || $25 != "0.00") exit 1
found = 1
}
END { exit found ? 0 : 1 }
' "${TMP_DIR}/fake-warp-candidate/baseline_compare.csv"
echo "object batch benchmark enhanced tests passed"