mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-23 04:39:04 +00:00
fix(ci): bound Warp ABBA dataset preparation
This commit is contained in:
@@ -34,8 +34,8 @@ CONCURRENCY=8
|
||||
DURATION="60s"
|
||||
ROUNDS=3
|
||||
COOLDOWN_SECS=20
|
||||
DATASET_SETUP_DURATION="10s"
|
||||
HEALTH_TIMEOUT_SECS=180
|
||||
DATASET_OBJECTS_PER_WORKER=8
|
||||
FAIL_PCT=10
|
||||
WARN_PCT=5
|
||||
ALLOW_REGRESSION=false
|
||||
@@ -100,9 +100,6 @@ Benchmark:
|
||||
--duration <dur> warp duration per cell (default 60s).
|
||||
--rounds <n> rounds per cell; must be >= 3 (default 3).
|
||||
--cooldown <n> cooldown seconds between rounds/sizes (default 20).
|
||||
--dataset-setup-duration <dur>
|
||||
isolated Warp PUT warm-up for get/mixed legs
|
||||
(default 10s; not included in the measurement).
|
||||
--concurrency <n> warp concurrency (default 8).
|
||||
--warp-bin <path> warp binary (default warp).
|
||||
|
||||
@@ -173,7 +170,6 @@ while [[ $# -gt 0 ]]; do
|
||||
--duration) DURATION="$2"; shift 2 ;;
|
||||
--rounds) ROUNDS="$2"; shift 2 ;;
|
||||
--cooldown) COOLDOWN_SECS="$2"; shift 2 ;;
|
||||
--dataset-setup-duration) DATASET_SETUP_DURATION="$2"; shift 2 ;;
|
||||
--health-timeout) HEALTH_TIMEOUT_SECS="$2"; shift 2 ;;
|
||||
--fail-pct) FAIL_PCT="$2"; shift 2 ;;
|
||||
--warn-pct) WARN_PCT="$2"; shift 2 ;;
|
||||
@@ -366,29 +362,18 @@ measure() {
|
||||
--duration "$DURATION" --rounds "$ROUNDS" --cooldown-secs "$COOLDOWN_SECS"
|
||||
--out-dir "$cell"
|
||||
)
|
||||
[[ "$mode" == "put" ]] || args+=(--extra-args "--noclear")
|
||||
if [[ "$mode" != "put" ]]; then
|
||||
# Warp defaults to 2,500 setup objects per round. At 10 MiB that writes
|
||||
# 25 GiB before every 12-second measurement, so the matrix cannot finish
|
||||
# inside the workflow budget. Eight objects per worker keeps preparation
|
||||
# bounded while retaining a multi-object working set for relative A/B.
|
||||
args+=(--extra-args "--objects $((CONCURRENCY * DATASET_OBJECTS_PER_WORKER)) --noclear")
|
||||
fi
|
||||
[[ -n "$baseline_csv" ]] && args+=(--baseline-csv "$baseline_csv")
|
||||
run "$ENHANCED_BENCH" "${args[@]}" >&2
|
||||
echo "$cell"
|
||||
}
|
||||
|
||||
prepare_dataset() {
|
||||
local leg="$1" workload="$2" mode="$3" size="$4" sync_label="$5" bucket="$6"
|
||||
[[ "$mode" != "put" ]] || return 0
|
||||
|
||||
local setup_cell="$OUT_DIR/$workload/$sync_label/$leg/dataset-setup"
|
||||
local args=(
|
||||
--tool warp --warp-bin "$WARP_BIN" --warp-mode put
|
||||
--endpoint "$ADDRESS" --access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY"
|
||||
--region "$REGION" --bucket "$bucket" --sizes "$size" --concurrency "$CONCURRENCY"
|
||||
--duration "$DATASET_SETUP_DURATION" --rounds 1 --cooldown-secs 0
|
||||
--extra-args "--noclear"
|
||||
--out-dir "$setup_cell"
|
||||
)
|
||||
log "preparing isolated dataset: $sync_label/$workload/$leg bucket=$bucket"
|
||||
run "$ENHANCED_BENCH" "${args[@]}" >&2
|
||||
}
|
||||
|
||||
write_schedule_header() {
|
||||
echo "sync_label,drive_sync,workload,mode,size,leg,phase,binary,out_dir,bucket,dataset_setup" >"$OUT_DIR/abba_schedule.csv"
|
||||
}
|
||||
@@ -399,7 +384,7 @@ append_schedule() {
|
||||
phase="$(phase_for_leg "$leg")"
|
||||
bin="$(binary_for_leg "$leg")"
|
||||
local dataset_setup="none"
|
||||
[[ "$mode" == "put" ]] || dataset_setup="warp-put"
|
||||
[[ "$mode" == "put" ]] || dataset_setup="warp-native-bounded"
|
||||
echo "$sync_label,$drive_sync,$workload,$mode,$size,$leg,$phase,$bin,$OUT_DIR/$workload/$sync_label/$leg,$bucket,$dataset_setup" >>"$OUT_DIR/abba_schedule.csv"
|
||||
}
|
||||
|
||||
@@ -481,7 +466,8 @@ dataset_namespace=$DATASET_NAMESPACE
|
||||
local_run_data_root=$RUN_DATA_ROOT
|
||||
bucket_isolation=per-leg
|
||||
bucket_prefix=rustfs-abba-$DATASET_NAMESPACE
|
||||
dataset_setup=get-and-mixed-via-warp-put
|
||||
dataset_setup=get-and-mixed-via-bounded-warp-native
|
||||
dataset_objects=$((CONCURRENCY * DATASET_OBJECTS_PER_WORKER))
|
||||
endpoint=$ADDRESS
|
||||
warp_version=$("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown)
|
||||
EOF
|
||||
@@ -502,7 +488,6 @@ for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do
|
||||
log "=== $sync_label $workload leg $leg ($(phase_for_leg "$leg")) ==="
|
||||
bucket="$(bucket_for_leg "$sync_label" "$workload" "$leg")"
|
||||
bring_up "$leg" "$drive_sync" "$workload" "$mode" "$size" "$sync_label" "$bucket"
|
||||
prepare_dataset "$leg" "$workload" "$mode" "$size" "$sync_label" "$bucket"
|
||||
append_schedule "$sync_label" "$drive_sync" "$workload" "$mode" "$size" "$leg" "$bucket"
|
||||
|
||||
baseline_csv=""
|
||||
|
||||
@@ -58,8 +58,13 @@ rg -qx 'evidence_mode=dry-run' "$OUT_DIR/manifest.env"
|
||||
rg -qx 'formal_evidence=false' "$OUT_DIR/manifest.env"
|
||||
rg -qx 'performance_conclusion=not_measured_dry_run' "$OUT_DIR/manifest.env"
|
||||
rg -qx 'bucket_isolation=per-leg' "$OUT_DIR/manifest.env"
|
||||
rg -qx 'dataset_setup=get-and-mixed-via-warp-put' "$OUT_DIR/manifest.env"
|
||||
[[ "$(rg -c -- '--extra-args --noclear' "$TRACE_FILE")" == "64" ]]
|
||||
rg -qx 'dataset_setup=get-and-mixed-via-bounded-warp-native' "$OUT_DIR/manifest.env"
|
||||
rg -qx 'dataset_objects=64' "$OUT_DIR/manifest.env"
|
||||
[[ "$(rg -c -- '--extra-args --objects\\ 64\\ --noclear' "$TRACE_FILE")" == "32" ]]
|
||||
if rg -q -- 'dataset-setup' "$TRACE_FILE"; then
|
||||
echo "unexpected redundant dataset setup command" >&2
|
||||
exit 1
|
||||
fi
|
||||
! rg -q -- 'rustfs-bench' "$TRACE_FILE"
|
||||
|
||||
if "$RUNNER" \
|
||||
|
||||
Reference in New Issue
Block a user