diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml new file mode 100644 index 000000000..28a29cdfe --- /dev/null +++ b/.github/workflows/performance-ab.yml @@ -0,0 +1,139 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4). +# +# Two entry points, honestly scoped: +# * schedule (nightly, on main): post-merge detection — catches a regression +# within 24h of landing, not before merge. +# * pull_request labeled `perf-ab`: opt-in pre-merge gate for a specific PR. +# The `perf-deliberate-tradeoff` label runs the gate with --allow-regression so +# a deliberate correctness cost (e.g. the #4221 fsync durability fix) is +# recorded but does not block (rustfs/backlog#935 correction 1). + +name: Performance A/B + +on: + schedule: + - cron: "0 6 * * *" # 06:00 UTC nightly, against main + workflow_dispatch: + inputs: + duration: + description: "warp duration per round" + required: false + default: "30s" + type: string + allow_regression: + description: "Pass the gate despite a FAIL (deliberate tradeoff)" + required: false + default: false + type: boolean + pull_request: + types: [labeled, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + warp-ab: + name: Warp A/B budget gate + # Opt-in on PRs: only run when the `perf-ab` label is present. Always run on + # schedule / manual dispatch. + if: >- + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'perf-ab') + runs-on: sm-standard-2 + timeout-minutes: 90 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + fetch-depth: 0 # baseline is built from origin/main + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }} + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install warp + run: | + set -euo pipefail + WARP_VERSION="1.0.0" + curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \ + | sudo tar -xz -C /usr/local/bin warp + warp --version + + - name: Decide exemption + id: exempt + run: | + allow="false" + if [[ "${{ github.event_name }}" == "pull_request" ]] \ + && ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then + allow="true" + fi + if [[ "${{ github.event.inputs.allow_regression }}" == "true" ]]; then + allow="true" + fi + echo "allow_regression=$allow" >> "$GITHUB_OUTPUT" + + - name: Run warp A/B and gate + id: ab + run: | + set -euo pipefail + args=(--baseline-ref origin/main --duration "${{ github.event.inputs.duration || '30s' }}") + if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then + args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override") + fi + # Do not let a gate FAIL abort the job here; capture status and surface + # it after the PR comment is posted. + set +e + bash scripts/run_hotpath_warp_ab.sh "${args[@]}" + echo "status=$?" >> "$GITHUB_OUTPUT" + set -e + # Locate the newest gate.md for the comment/artifact steps. + gate_md="$(ls -t target/hotpath-ab/*/gate.md 2>/dev/null | head -n1 || true)" + echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT" + + - name: Upload A/B results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: hotpath-warp-ab-${{ github.run_number }} + path: target/hotpath-ab/ + if-no-files-found: warn + + - name: Comment gate result on PR + if: always() && github.event_name == 'pull_request' && steps.ab.outputs.gate_md != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr comment "${{ github.event.pull_request.number }}" --body-file "${{ steps.ab.outputs.gate_md }}" + + - name: Enforce gate + if: always() + run: | + status="${{ steps.ab.outputs.status }}" + if [[ "$status" != "0" ]]; then + echo "::error::warp A/B budget gate failed (exit $status). See the PR comment / gate.md artifact." >&2 + exit "$status" + fi + echo "warp A/B budget gate passed." diff --git a/docs/operations/hotpath-warp-ab-runbook.md b/docs/operations/hotpath-warp-ab-runbook.md new file mode 100644 index 000000000..8bb2f484b --- /dev/null +++ b/docs/operations/hotpath-warp-ab-runbook.md @@ -0,0 +1,102 @@ +# Hotpath warp A/B runbook + +Relative-budget A/B gate for the hotpath series (rustfs/backlog#935 HP-14). It +runs the same warp workloads against a **baseline** binary and a **candidate** +binary, across the drive-sync on/off matrix, then applies a relative budget: +a metric regressing past the fail budget fails the gate, past the warn budget +warns. This is how the macOS profiling conclusions of the HP series get +confirmed or corrected on Linux — structural wins (call counts, read +amplification) should hold; absolute numbers are whatever the rig measures. + +Pieces: + +- `scripts/run_hotpath_warp_ab.sh` — orchestrator (baseline vs candidate, + workload × drive-sync matrix). +- `scripts/hotpath_warp_ab_gate.sh` — the budget gate over the + `baseline_compare.csv` deltas the load driver emits. +- `scripts/run_object_batch_bench_enhanced.sh` — the warp driver + median + + `baseline_compare.csv` (reused, not reimplemented). +- `.github/workflows/performance-ab.yml` — nightly on `main` (post-merge + detection) plus opt-in pre-merge via the `perf-ab` label. + +Metric directions: `reqps` (put obj/s) and `throughput` (get MiB/s) are +higher-is-better; `latency` / p99 (mixed) is lower-is-better. warp is assumed +pre-installed, as elsewhere in `scripts/`. + +## Local mode (quick / CI smoke) + +Builds both binaries and runs a throwaway single-node server on local disks. + +```bash +scripts/run_hotpath_warp_ab.sh --baseline-ref origin/main +# or with prebuilt binaries: +scripts/run_hotpath_warp_ab.sh --skip-build \ + --baseline-bin ./rustfs-main --candidate-bin ./target/release/rustfs +``` + +Preview the full plan without running anything: + +```bash +scripts/run_hotpath_warp_ab.sh --dry-run --skip-build \ + --baseline-bin /tmp/base --candidate-bin /tmp/cand +``` + +## External mode (real cluster, ansible-deployed) + +For the production-representative run, warp targets an already-running cluster +and a `--deploy-hook` swaps in each phase's binary and durability config +between the baseline and candidate phases. The hook receives context via the +environment: + +- `HOTPATH_AB_PHASE` — `baseline` or `candidate` +- `HOTPATH_AB_BINARY` — binary path (or empty; the hook may build its own) +- `HOTPATH_AB_DRIVE_SYNC` — `true` or `false` for this matrix cell + +This maps directly onto the team's ansible harness. Build the candidate with +the cross toolchain, stage both binaries, then let the hook drive +`rustfs-manage.yml`: + +```bash +# 1. Build the candidate (cross-compile for the cluster target). +cargo zigbuild --release --target x86_64-unknown-linux-gnu -p rustfs --bins + +# 2. Run the A/B against the cluster; the hook deploys the phase's binary and +# applies the drive-sync config, then restarts, before each phase. +scripts/run_hotpath_warp_ab.sh \ + --endpoint "$CLUSTER_ENDPOINT" \ + --deploy-hook ' + set -euo pipefail + cd /home/xiaomage/xiaomage/ansible + # Select the phase binary and the drive-sync value for this cell. + cp "${HOTPATH_AB_BINARY:?}" ./roles/rustfs/files/rustfs + export RUSTFS_DRIVE_SYNC_ENABLE="$HOTPATH_AB_DRIVE_SYNC" + ansible-playbook -f 4 -l testing rustfs-manage.yml --tags stop + ansible-playbook -f 4 -l testing rustfs-manage.yml --tags config + ansible-playbook -f 4 -l testing rustfs-manage.yml --tags binary-copy + ansible-playbook -f 4 -l testing rustfs-manage.yml --tags start + ' \ + --baseline-bin /path/to/rustfs-main \ + --candidate-bin ./target/x86_64-unknown-linux-gnu/release/rustfs +``` + +The `config` tag is responsible for threading `RUSTFS_DRIVE_SYNC_ENABLE` (or +the finer `RUSTFS_DURABILITY_MODE`) into the deployed unit — the hook exports +it so the config template can pick it up. The rig itself never restarts the +cluster; lifecycle stays with ansible. + +## Budget and exemptions + +Default budget: a metric regressing more than **10%** vs baseline fails, +more than **5%** warns. Tune with `--fail-pct` / `--warn-pct`. + +Some regressions are the correct trade — #4221 deliberately paid a large write +cost to restore power-loss durability. For those, run with +`--allow-regression` (or add the `perf-deliberate-tradeoff` label in CI): the +FAIL is recorded and rendered as an exempted WARN, and the gate exits 0. + +## Scope note + +The gate logic is unit-validated across pass/warn/fail/exempt outcomes; the +orchestrator and workflow are shellcheck- and `--dry-run`-validated. The first +real warp measurement belongs on a Linux runner or the ansible cluster — there +is no warp/multi-disk rig in the repo's local checkout. diff --git a/scripts/hotpath_warp_ab_gate.sh b/scripts/hotpath_warp_ab_gate.sh new file mode 100755 index 000000000..430d2790d --- /dev/null +++ b/scripts/hotpath_warp_ab_gate.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# hotpath_warp_ab_gate.sh — relative-budget gate for the hotpath warp A/B rig +# (rustfs/backlog#935 HP-14, item 4). +# +# The load driver scripts/run_object_batch_bench_enhanced.sh already emits a +# baseline_compare.csv with delta_{reqps,latency,throughput}_pct columns but +# does not itself decide pass/fail. This script applies the relative budget: +# a metric that regresses past --fail-pct fails the gate, past --warn-pct +# warns, otherwise passes. Because #4221 shows a deliberate 26x cost is +# sometimes the correct durability fix, --allow-regression downgrades every +# FAIL to an exempted WARN and records the reason instead of blocking. +# +# Metric direction: +# reqps, throughput -> higher is better; regression is a negative delta. +# latency (incl p99) -> lower is better; regression is a positive delta. +# +# Input CSV header (from run_object_batch_bench_enhanced.sh compare_baseline): +# size,tool,concurrency,new_median_reqps,baseline_median_reqps,delta_reqps_pct, +# new_median_latency_ms,baseline_median_latency_ms,delta_latency_pct, +# new_median_throughput_bps,baseline_median_throughput_bps,delta_throughput_pct +# +# Exit code: 0 when no unexempted FAIL (PASS or WARN), 1 otherwise. + +set -euo pipefail + +FAIL_PCT=10 +WARN_PCT=5 +ALLOW_REGRESSION="false" +MARKDOWN_OUT="" +EXEMPTION_REASON="deliberate correctness tradeoff" +declare -a COMPARE_CSVS=() + +usage() { + cat <<'USAGE' +Usage: hotpath_warp_ab_gate.sh --compare-csv [--compare-csv ...] [options] + + --compare-csv baseline_compare.csv to evaluate (repeatable). + --fail-pct Regression budget that fails the gate (default 10). + --warn-pct Regression budget that warns (default 5). + --allow-regression Downgrade every FAIL to an exempted WARN (deliberate + correctness tradeoff); the gate then always exits 0. + --exemption-reason Reason recorded when --allow-regression is set. + --markdown Also write the result table as Markdown to . + -h, --help Show this help. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --compare-csv) COMPARE_CSVS+=("$2"); shift 2 ;; + --fail-pct) FAIL_PCT="$2"; shift 2 ;; + --warn-pct) WARN_PCT="$2"; shift 2 ;; + --allow-regression) ALLOW_REGRESSION="true"; shift ;; + --exemption-reason) EXEMPTION_REASON="$2"; shift 2 ;; + --markdown) MARKDOWN_OUT="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ ${#COMPARE_CSVS[@]} -eq 0 ]]; then + echo "error: at least one --compare-csv is required" >&2 + usage >&2 + exit 2 +fi +for csv in "${COMPARE_CSVS[@]}"; do + [[ -f "$csv" ]] || { echo "error: compare CSV not found: $csv" >&2; exit 2; } +done + +# One awk pass over all CSVs. Emits TSV rows "verdict\tworkload\tmetric\tdelta" +# on stdout and a final "OVERALL\t" line. Verdict is PASS/WARN/FAIL. +gate_output="$( + awk -v fail_pct="$FAIL_PCT" -v warn_pct="$WARN_PCT" ' + function classify(delta, higher_is_better, regress) { + if (delta == "" || delta == "N/A") return "SKIP" + # Signed regression magnitude: positive means "worse than baseline". + regress = higher_is_better ? -delta : delta + if (regress > fail_pct) return "FAIL" + if (regress > warn_pct) return "WARN" + return "PASS" + } + function emit(v, workload, metric, delta, rank) { + if (v == "SKIP") return + print v "\t" workload "\t" metric "\t" delta "%" + rank = (v == "FAIL") ? 3 : (v == "WARN") ? 2 : 1 + if (rank > worst) worst = rank + } + BEGIN { worst = 1 } + FNR == 1 { next } # skip each file header + { + n = split($0, f, ",") + if (n < 12) next + workload = f[1] "/" f[2] "@" f[3] # size/tool@concurrency + emit(classify(f[6], 1), workload, "reqps", f[6]) + emit(classify(f[9], 0), workload, "latency", f[9]) + emit(classify(f[12], 1), workload, "throughput", f[12]) + } + END { + print "OVERALL\t" (worst == 3 ? "FAIL" : worst == 2 ? "WARN" : "PASS") + } + ' "${COMPARE_CSVS[@]}" +)" + +overall="$(printf '%s\n' "$gate_output" | awk -F'\t' '$1=="OVERALL"{print $2}')" +rows="$(printf '%s\n' "$gate_output" | awk -F'\t' '$1!="OVERALL"')" + +exempted="false" +effective="$overall" +if [[ "$overall" == "FAIL" && "$ALLOW_REGRESSION" == "true" ]]; then + exempted="true" + effective="WARN" +fi + +emoji() { case "$1" in FAIL) echo '❌';; WARN) echo '⚠️';; PASS) echo '✅';; *) echo '';; esac; } + +render() { + echo "### Hotpath warp A/B gate — $(emoji "$effective") $effective" + echo + echo "Budget: regression > ${FAIL_PCT}% fails, > ${WARN_PCT}% warns (relative to baseline)." + if [[ "$exempted" == "true" ]]; then + echo + echo "> Gate FAIL exempted via \`--allow-regression\`: ${EXEMPTION_REASON}." + fi + echo + echo "| Workload | Metric | Δ vs baseline | Verdict |" + echo "| --- | --- | --- | --- |" + if [[ -z "$rows" ]]; then + echo "| _(no rows)_ | | | |" + else + printf '%s\n' "$rows" | while IFS=$'\t' read -r v workload metric delta; do + [[ -z "$v" ]] && continue + echo "| $workload | $metric | $delta | $(emoji "$v") $v |" + done + fi +} + +table="$(render)" +printf '%s\n' "$table" +if [[ -n "$MARKDOWN_OUT" ]]; then + printf '%s\n' "$table" >"$MARKDOWN_OUT" +fi + +if [[ "$overall" == "FAIL" && "$exempted" != "true" ]]; then + exit 1 +fi +exit 0 diff --git a/scripts/run_hotpath_warp_ab.sh b/scripts/run_hotpath_warp_ab.sh new file mode 100755 index 000000000..0d1395ec5 --- /dev/null +++ b/scripts/run_hotpath_warp_ab.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# run_hotpath_warp_ab.sh — Linux warp A/B rig for the hotpath series +# (rustfs/backlog#935 HP-14, item 4). +# +# Runs the same warp workloads against a baseline binary and a candidate +# binary, across the drive-sync on/off matrix, then applies the relative-budget +# gate (hotpath_warp_ab_gate.sh). This is how the macOS profiling conclusions +# of the HP series get confirmed or corrected on Linux: structural wins (call +# counts, read amplification) should hold; absolute numbers are whatever this +# rig measures. +# +# Two deployment modes: +# * local (default): build both binaries and run a throwaway single-node +# server on local disks — good for a quick or CI smoke A/B. +# * external (--endpoint ): warp targets an already-running +# cluster; a --deploy-hook command swaps in the right binary / durability +# config between phases. This maps onto the team's ansible harness +# (cargo zigbuild -> ansible rustfs-install / rustfs-manage --tags +# stop,config,binary-copy,start -> warp). See +# docs/operations/hotpath-warp-ab-runbook.md. +# +# Load generation and result parsing are delegated to the existing +# scripts/run_object_batch_bench_enhanced.sh (warp driver + median + +# baseline_compare deltas). warp is assumed pre-installed, as elsewhere in +# scripts/. +# +# The rig is shellcheck- and --dry-run-validated; its first real measurement +# run belongs on a Linux runner / cluster with warp installed. + +set -euo pipefail + +PROJECT_ROOT="$(git rev-parse --show-toplevel)" +ENHANCED_BENCH="${PROJECT_ROOT}/scripts/run_object_batch_bench_enhanced.sh" +GATE="${PROJECT_ROOT}/scripts/hotpath_warp_ab_gate.sh" + +# --- Configuration (overridable via flags) ---------------------------------- +BASELINE_REF="origin/main" +BASELINE_BIN="" +CANDIDATE_BIN="" +DISKS=4 +CONCURRENCY=8 +DURATION="30s" +ROUNDS=3 +LOCAL_ADDRESS="127.0.0.1:9000" +ACCESS_KEY="rustfsadmin" +SECRET_KEY="rustfsadmin" +REGION="us-east-1" +WARP_BIN="warp" +FAIL_PCT=10 +WARN_PCT=5 +ALLOW_REGRESSION="false" +EXEMPTION_REASON="deliberate correctness tradeoff" +DATA_ROOT="/tmp/rustfs-hotpath-ab" +OUT_DIR="${PROJECT_ROOT}/target/hotpath-ab/$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)" +DRY_RUN="false" +SKIP_BUILD="false" +EXTERNAL_ENDPOINT="" # set -> external mode (warp targets this cluster) +DEPLOY_HOOK="" # external mode: command to deploy a phase's binary +HEALTH_PATH="/health" + +# Workloads: name|warp-mode|size. put-4mib obj/s, get-4mib MiB/s, mixed-256k p99. +WORKLOADS=("put-4mib|put|4MiB" "get-4mib|get|4MiB" "mixed-256k|mixed|256KiB") +# Durability matrix: label|RUSTFS_DRIVE_SYNC_ENABLE value. +DRIVE_SYNC_MATRIX=("sync-on|true" "sync-off|false") + +usage() { + cat <<'USAGE' +Usage: run_hotpath_warp_ab.sh [options] + +Local mode (default): build + run a throwaway single-node server. + --baseline-ref Git ref to build the baseline from (default origin/main). + --baseline-bin Prebuilt baseline binary (skips building the ref). + --candidate-bin Prebuilt candidate binary (default: build the worktree). + --disks Local disks for the single node (default 4). + --skip-build Do not build; requires --baseline-bin/--candidate-bin. + +External mode: warp targets an already-running cluster (e.g. ansible-deployed). + --endpoint Cluster S3 endpoint; enables external mode. + --deploy-hook Command run before each phase to deploy that phase's + binary and durability config. It receives context via + env: HOTPATH_AB_PHASE (baseline|candidate), + HOTPATH_AB_BINARY (path or empty), + HOTPATH_AB_DRIVE_SYNC (true|false). Non-zero aborts. + --health-path Readiness path polled after deploy (default /health). + +Common: + --concurrency warp --concurrent (default 8). + --duration warp --duration (default 30s). + --rounds Rounds per cell for the median (default 3). + --fail-pct Gate fail budget (default 10). + --warn-pct Gate warn budget (default 5). + --allow-regression Pass the gate despite a FAIL (deliberate tradeoff). + --exemption-reason Reason recorded with --allow-regression. + --out-dir Output directory (default target/hotpath-ab/). + --dry-run Print the plan and commands without running them. + -h, --help Show this help. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --baseline-ref) BASELINE_REF="$2"; shift 2 ;; + --baseline-bin) BASELINE_BIN="$2"; shift 2 ;; + --candidate-bin) CANDIDATE_BIN="$2"; shift 2 ;; + --disks) DISKS="$2"; shift 2 ;; + --endpoint) EXTERNAL_ENDPOINT="$2"; shift 2 ;; + --deploy-hook) DEPLOY_HOOK="$2"; shift 2 ;; + --health-path) HEALTH_PATH="$2"; shift 2 ;; + --concurrency) CONCURRENCY="$2"; shift 2 ;; + --duration) DURATION="$2"; shift 2 ;; + --rounds) ROUNDS="$2"; shift 2 ;; + --fail-pct) FAIL_PCT="$2"; shift 2 ;; + --warn-pct) WARN_PCT="$2"; shift 2 ;; + --allow-regression) ALLOW_REGRESSION="true"; shift ;; + --exemption-reason) EXEMPTION_REASON="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --skip-build) SKIP_BUILD="true"; shift ;; + --dry-run) DRY_RUN="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +EXTERNAL="false" +[[ -n "$EXTERNAL_ENDPOINT" ]] && EXTERNAL="true" +ADDRESS="$EXTERNAL_ENDPOINT" +[[ "$EXTERNAL" == "true" ]] || ADDRESS="$LOCAL_ADDRESS" + +log() { printf '[hotpath-ab] %s\n' "$*" >&2; } + +run() { + # DRY-RUN lines go to stderr so functions that return a value on stdout stay + # uncontaminated. + if [[ "$DRY_RUN" == "true" ]]; then + { printf 'DRY-RUN:'; printf ' %q' "$@"; printf '\n'; } >&2 + return 0 + fi + "$@" +} + +[[ -x "$ENHANCED_BENCH" ]] || { echo "missing load driver: $ENHANCED_BENCH" >&2; exit 2; } +[[ -x "$GATE" ]] || { echo "missing gate: $GATE" >&2; exit 2; } +if [[ "$DRY_RUN" != "true" ]] && ! command -v "$WARP_BIN" >/dev/null 2>&1; then + echo "error: warp not found on PATH (install warp or use --dry-run)" >&2 + exit 2 +fi +if [[ "$EXTERNAL" == "true" && -z "$DEPLOY_HOOK" ]]; then + log "warning: external mode without --deploy-hook — baseline and candidate must be swapped out of band" +fi + +mkdir -p "$OUT_DIR" +log "mode: $([[ "$EXTERNAL" == "true" ]] && echo "external ($ADDRESS)" || echo "local ($ADDRESS)")" +log "output dir: $OUT_DIR" + +# --- Build binaries (local mode only) --------------------------------------- +build_ref_binary() { + local ref="$1" dest="$OUT_DIR/rustfs-baseline" + if [[ -n "$BASELINE_BIN" ]]; then echo "$BASELINE_BIN"; return; fi + local wt="$OUT_DIR/baseline-worktree" + run git worktree add --detach "$wt" "$ref" + run bash -c "cd '$wt' && cargo build --release --bin rustfs" + run cp "$wt/target/release/rustfs" "$dest" 2>/dev/null || true + run git worktree remove --force "$wt" 2>/dev/null || true + echo "$dest" +} + +if [[ "$EXTERNAL" == "true" ]]; then + # Binaries are optional context handed to the deploy hook. + log "baseline binary: ${BASELINE_BIN:-}" + log "candidate binary: ${CANDIDATE_BIN:-}" +elif [[ "$SKIP_BUILD" == "true" ]]; then + [[ -n "$BASELINE_BIN" && -n "$CANDIDATE_BIN" ]] || { + echo "error: --skip-build requires --baseline-bin and --candidate-bin" >&2; exit 2; } +else + [[ -n "$CANDIDATE_BIN" ]] || { + log "building candidate (current worktree)" + run cargo build --release --bin rustfs + CANDIDATE_BIN="${PROJECT_ROOT}/target/release/rustfs" + } + [[ -n "$BASELINE_BIN" ]] || { + log "building baseline ($BASELINE_REF)" + BASELINE_BIN="$(build_ref_binary "$BASELINE_REF")" + } +fi + +# --- Deployment: bring a phase up, tear it down ----------------------------- +SERVER_PID="" +tear_down() { + # Local mode owns the server process; external mode leaves lifecycle to the + # deploy hook / cluster orchestrator. + [[ "$EXTERNAL" == "true" ]] && return 0 + [[ -n "$SERVER_PID" ]] || return 0 + run kill "$SERVER_PID" 2>/dev/null || true + SERVER_PID="" +} +trap tear_down EXIT INT TERM + +wait_health() { + [[ "$DRY_RUN" == "true" ]] && return 0 + local i + for ((i = 0; i < 60; i++)); do + if curl -fsS "http://${ADDRESS}${HEALTH_PATH}" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + echo "error: endpoint http://${ADDRESS}${HEALTH_PATH} did not become healthy" >&2 + return 1 +} + +bring_up() { + # bring_up + local phase="$1" bin="$2" drive_sync="$3" + if [[ "$EXTERNAL" == "true" ]]; then + if [[ -n "$DEPLOY_HOOK" ]]; then + log "deploy hook: phase=$phase drive_sync=$drive_sync" + HOTPATH_AB_PHASE="$phase" HOTPATH_AB_BINARY="$bin" HOTPATH_AB_DRIVE_SYNC="$drive_sync" \ + run bash -c "$DEPLOY_HOOK" + fi + wait_health + return 0 + fi + + # Local: throwaway single-node server on fresh disks. + local node_dir="$DATA_ROOT/$phase-sync-$drive_sync" + local disks=() d + for ((d = 1; d <= DISKS; d++)); do disks+=("$node_dir/d$d"); done + run mkdir -p "${disks[@]}" + if [[ "$DRY_RUN" == "true" ]]; then + { printf 'DRY-RUN: RUSTFS_DRIVE_SYNC_ENABLE=%s %q server' "$drive_sync" "$bin" + printf ' %q' "${disks[@]}"; printf '\n'; } >&2 + SERVER_PID="dry-run" + return 0 + fi + RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \ + RUSTFS_ADDRESS="$ADDRESS" \ + RUSTFS_ACCESS_KEY="$ACCESS_KEY" \ + RUSTFS_SECRET_KEY="$SECRET_KEY" \ + RUSTFS_REGION="$REGION" \ + RUSTFS_CONSOLE_ENABLE=false \ + RUSTFS_DRIVE_SYNC_ENABLE="$drive_sync" \ + "$bin" server "${disks[@]}" >"$node_dir/rustfs.log" 2>&1 & + SERVER_PID=$! + wait_health +} + +# --- Measure one workload against the current deployment -------------------- +measure() { + # measure [baseline_csv] + local phase="$1" wl="$2" mode="$3" size="$4" sync_label="$5" baseline_csv="${6:-}" + local cell="$OUT_DIR/$wl/$sync_label/$phase" + run mkdir -p "$cell" + local args=( + --tool warp --warp-bin "$WARP_BIN" --warp-mode "$mode" + --endpoint "$ADDRESS" --access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" + --region "$REGION" --sizes "$size" --concurrency "$CONCURRENCY" + --duration "$DURATION" --rounds "$ROUNDS" --out-dir "$cell" + ) + [[ -n "$baseline_csv" ]] && args+=(--baseline-csv "$baseline_csv") + run "$ENHANCED_BENCH" "${args[@]}" + echo "$cell" +} + +# --- Drive the matrix: per drive-sync, baseline phase then candidate phase --- +declare -a COMPARE_CSVS=() +for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do + IFS='|' read -r sync_label drive_sync <<<"$ds_spec" + + log "=== drive-sync $sync_label: baseline phase ===" + bring_up "baseline" "$BASELINE_BIN" "$drive_sync" + for wl_spec in "${WORKLOADS[@]}"; do + IFS='|' read -r wl mode size <<<"$wl_spec" + measure "baseline" "$wl" "$mode" "$size" "$sync_label" >/dev/null + done + tear_down + + log "=== drive-sync $sync_label: candidate phase (vs baseline) ===" + bring_up "candidate" "$CANDIDATE_BIN" "$drive_sync" + for wl_spec in "${WORKLOADS[@]}"; do + IFS='|' read -r wl mode size <<<"$wl_spec" + # measure() writes each phase under $OUT_DIR///, so the + # baseline median for this cell is at a deterministic path. + baseline_csv="$OUT_DIR/$wl/$sync_label/baseline/median_summary.csv" + cand_cell="$(measure "candidate" "$wl" "$mode" "$size" "$sync_label" "$baseline_csv")" + COMPARE_CSVS+=("$cand_cell/baseline_compare.csv") + done + tear_down +done + +# --- Gate ------------------------------------------------------------------- +gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/gate.md") +for csv in "${COMPARE_CSVS[@]}"; do gate_args+=(--compare-csv "$csv"); done +[[ "$ALLOW_REGRESSION" == "true" ]] && gate_args+=(--allow-regression --exemption-reason "$EXEMPTION_REASON") + +if [[ "$DRY_RUN" == "true" ]]; then + log "dry-run complete; would gate on ${#COMPARE_CSVS[@]} compare CSV(s)" + { printf 'DRY-RUN:'; printf ' %q' "$GATE" "${gate_args[@]}"; printf '\n'; } >&2 + exit 0 +fi + +log "applying relative-budget gate" +"$GATE" "${gate_args[@]}" +gate_status=$? +log "gate result written to $OUT_DIR/gate.md (exit $gate_status)" +exit "$gate_status"