ci(perf): cache the main baseline binary to cut the nightly double build (#4816)

Every push to main now builds the release binary once and stores it in the
actions cache as rustfs-baseline-<sha> (build-baseline-cache job). The A/B job
restores it for origin/main and passes --baseline-bin; on the nightly, where
the candidate commit equals the baseline, one cached binary serves both phases
with --skip-build and the run does zero source builds. A cache miss falls back
to the source double-build via --baseline-ref origin/main.

This removes the ~32min-per-side double build that pushed the 24-cell nightly
past its 120min ceiling (2026-07-11..07-14 all cancelled on timeout).

Also:
- alert-on-failure now fires on cancelled/timed-out, not just failure, so a
  timed-out nightly is no longer silent (the composite action already reports
  cancelled jobs); removed the stale perf-2 TODO now that the alert job exists.
- run_hotpath_warp_ab.sh appends a Provenance section to gate.md (baseline and
  candidate SHAs + binary source, runner, warp version, matrix params) via a
  repeatable --provenance-note; the gate exit code is preserved.

Refs rustfs/backlog#1152 (perf-3).

.github/workflows/performance-ab.yml

@@ -41,6 +41,10 @@ on:
         type: boolean
   pull_request:
     types: [labeled, synchronize, reopened]
+  push:
+    # Every main commit pre-builds and caches its release binary (perf-3) so the
+    # nightly A/B restores a ready baseline instead of paying the double build.
+    branches: [main]
 
 permissions:
   contents: read
@@ -58,22 +62,67 @@ env:
   RUST_BACKTRACE: 1
 
 jobs:
+  # perf-3: on every push to main, build the release binary once and cache it
+  # keyed by commit SHA (rustfs-baseline-<sha>). The nightly A/B (and, later, the
+  # perf-7 PR gate) restore this instead of paying the ~32min-per-side source
+  # build. That double build is what pushed the expanded 24-cell nightly past its
+  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
+  # builds off the shared cargo cache keep each push cheap, and building on the
+  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
+  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
+  # are cancelled ~7/8 of the time and are not a reliable baseline.
+  build-baseline-cache:
+    name: Build + cache baseline binary
+    if: github.event_name == 'push'
+    runs-on: sm-standard-2
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build release rustfs
+        run: cargo build --release --bin rustfs
+
+      - name: Stage binary for cache
+        run: |
+          set -euo pipefail
+          mkdir -p baseline-bin
+          cp target/release/rustfs baseline-bin/rustfs
+
+      - name: Cache baseline binary by SHA
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ github.sha }}
+
   warp-ab:
     name: Warp A/B budget gate
-    # Opt-in on PRs: only run when the `perf-ab` label is present, and for
+    # Always run on schedule / manual dispatch. Opt-in on PRs: only when the
-    # `labeled` events only when the label being added is `perf-ab` itself —
+    # `perf-ab` label is present, and for `labeled` events only when the label
-    # adding an unrelated label to an opted-in PR must not re-run the gate.
+    # being added is `perf-ab` itself (adding an unrelated label to an opted-in
-    # Always run on schedule / manual dispatch.
+    # PR must not re-run the gate). Never on push — that event only feeds
+    # build-baseline-cache above.
     if: >-
-      github.event_name != 'pull_request' ||
+      github.event_name == 'schedule' ||
-      (contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+       contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
        (github.event.action != 'labeled' || github.event.label.name == 'perf-ab'))
     runs-on: sm-standard-2
-    # Phase-0 stopgap: the baseline+candidate release double-build alone is
+    # With perf-3's cached baseline binary the common (cache-hit) nightly is
-    # ~65min on this runner, so 90min left no room for a real full-matrix
+    # measurement-only and finishes well under 50min. This ceiling stays
-    # measurement (the earlier nightly runs only ever failed *before*
+    # generous only to absorb the rare cache-miss fallback, which reverts to the
-    # measuring). 120min gives the 24-cell short matrix headroom until perf-3
+    # ~65min source double-build; a timeout there now surfaces via the
-    # caches the baseline binary and restores a tighter budget.
+    # alert-on-failure job (it fires on cancelled/timed-out, not just failure).
+    # perf-6 recalibrates the budget once the noise study lands.
     timeout-minutes: 120
     steps:
       - name: Checkout repository
@@ -110,21 +159,74 @@ jobs:
           fi
           echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
 
+      # perf-3: resolve the commits so the cache can be keyed by SHA. The
+      # baseline is origin/main; the candidate is the checked-out ref. On the
+      # nightly (checkout == main) they are the same commit, so one cached binary
+      # serves both phases and the run does zero source builds.
+      - name: Resolve baseline / candidate commits
+        id: commits
+        run: |
+          set -euo pipefail
+          baseline_sha="$(git rev-parse origin/main)"
+          candidate_sha="$(git rev-parse HEAD)"
+          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
+          echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
+          echo "baseline commit: $baseline_sha"
+          echo "candidate commit: $candidate_sha"
+
+      # Exact-key restore of the baseline binary built by build-baseline-cache
+      # when origin/main last landed. A miss (binary evicted or not built yet)
+      # leaves cache-hit unset and the rig falls back to a source build.
+      - name: Restore cached baseline binary
+        id: baseline_cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: baseline-bin/rustfs
+          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
+
       - name: Run warp A/B and gate
         id: ab
         run: |
           set -euo pipefail
-          # Budget note: the baseline+candidate release double-build (~65 min,
+          # Budget note: with perf-3's cached baseline the nightly does no source
-          # cached away later by perf-3) dominates the 90-min job, so the
+          # build on a cache hit, so the wall-clock is dominated by the short warp
-          # measurement runs a short warp matrix — duration/rounds/cooldown are
+          # matrix — duration/rounds/cooldown are kept small to fit all 24 cells
-          # kept small to fit all 24 cells (6 workloads x 2 phases x 2 drive-sync)
+          # (6 workloads x 2 phases x 2 drive-sync) rather than dropping cells.
-          # under budget rather than dropping cells. --health-timeout 180 outlasts
+          # --health-timeout 180 outlasts the server's own 120s startup-readiness
-          # the server's own 120s startup-readiness budget, which is what the
+          # budget, which the rig's previous 60s health poll undershot (the first
-          # rig's previous 60s health poll undershot (the first two nightly
+          # two nightly failures). perf-6 recalibrates these once the noise study
-          # failures). perf-6 will recalibrate these once the pipeline is green.
+          # lands.
           duration="${{ github.event.inputs.duration || '12s' }}"
-          args=(--baseline-ref origin/main
+          baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
-            --duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
+          candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
+          baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
+
+          args=(--duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
+
+          if [[ "$baseline_hit" == "true" ]]; then
+            chmod +x baseline-bin/rustfs
+            base_bin="$PWD/baseline-bin/rustfs"
+            args+=(--baseline-bin "$base_bin")
+            base_src="actions-cache (rustfs-baseline-$baseline_sha)"
+            if [[ "$candidate_sha" == "$baseline_sha" ]]; then
+              # Nightly on main: the candidate is the same commit as the baseline,
+              # so reuse the one cached binary for both phases and skip all builds.
+              args+=(--candidate-bin "$base_bin" --skip-build)
+              cand_src="actions-cache (same commit as baseline)"
+            else
+              cand_src="source build of the checked-out ref"
+            fi
+          else
+            # Cache miss (binary evicted or not built for this SHA): fall back to
+            # the slow source double-build so the run still produces a result.
+            args+=(--baseline-ref origin/main)
+            base_src="source build of origin/main (cache miss)"
+            cand_src="source build of the checked-out ref"
+          fi
+
+          args+=(--provenance-note "baseline commit: $baseline_sha - $base_src")
+          args+=(--provenance-note "candidate commit: $candidate_sha - $cand_src")
+
           if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
             args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override")
           fi
@@ -201,12 +303,8 @@ jobs:
         run: |
           gh pr comment "${{ github.event.pull_request.number }}" --body-file "${{ steps.ab.outputs.gate_md }}"
 
-      # TODO(ci-8/perf-2): scheduled/dispatch failures are currently silent. Once
+      # Scheduled failure alerting is handled by the alert-on-failure job below
-      # ci-8 lands the .github/actions/schedule-failure-issue composite action,
+      # (perf-2 consuming ci-8's schedule-failure-issue composite action).
-      # perf-2 adds a step here guarded by
-      #   if: failure() && github.event_name != 'pull_request'
-      # that calls it (label perf-nightly-failure, append to an existing open
-      # issue) instead of hand-rolling gh CLI dedup. Do not implement it here.
 
       - name: Enforce gate
         if: always()
@@ -224,7 +322,16 @@ jobs:
     # `always()` is required: without it this job is skipped when a needed
     # job fails. Alerts only for scheduled (nightly) runs (backlog#1149
     # ci-8); PR and manual dispatch failures are already watched by a human.
-    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    # `cancelled` is included alongside `failure` on purpose: a job that hits
+    # timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
+    # timeouts went silent precisely because the guard was failure-only. The
+    # composite action already reports cancelled/timed-out jobs in the issue
+    # body. (Scheduled runs get a unique concurrency group with
+    # cancel-in-progress off, so a cancellation here means a timeout/manual
+    # abort, never a superseding run.)
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
     runs-on: ubuntu-latest
     timeout-minutes: 10
     permissions:

docs/operations/hotpath-warp-ab-runbook.md

@@ -37,15 +37,31 @@ Six workloads × the drive-sync on/off matrix × baseline/candidate = 24 cells:
 Sizes are passed to the load driver via `--sizes` (one size per cell); the
 driver's `DEFAULT_SIZES` covers 1KiB..10MiB, so any of those can be added by
 editing `WORKLOADS` in `scripts/run_hotpath_warp_ab.sh`. A 1KiB cell is left
-out for now to keep the nightly matrix under the 90-minute job budget (the
+out for now to keep the nightly matrix comfortably under budget; re-enable it
-baseline+candidate double build dominates until perf-3 caches it); re-enable it
+(one line in `WORKLOADS`) once perf-6 recalibrates the warp params.
-once that lands.
 
 CI runs a **short** warp matrix (`--duration`/`--rounds`/`--cooldown` tuned in
 `.github/workflows/performance-ab.yml`) so all 24 cells fit the budget without
 dropping cells. These params are deliberately noisy-but-fast for the Phase-0
 "keep the pipeline alive" goal; perf-6 recalibrates them.
 
+## Baseline binary cache (CI)
+
+The nightly no longer builds both binaries from source. Every push to `main`
+runs a `build-baseline-cache` job that builds the release binary once and stores
+it in the actions cache under `rustfs-baseline-<sha>` (perf-3). The A/B job
+restores the binary for `origin/main` by that key and passes it as
+`--baseline-bin`; on the nightly, where the candidate commit equals the baseline
+commit, the same cached binary serves both phases (`--skip-build`) and the run
+does zero source builds — the common path finishes well under 50 minutes. A
+cache miss (binary evicted, or not built for that SHA yet) transparently falls
+back to the source double-build via `--baseline-ref origin/main`.
+
+Each `gate.md` ends with a **Provenance** section recording the baseline and
+candidate commit SHAs and whether each binary came from the cache or a source
+build, plus the runner, warp version, and matrix params. `perf-5`/`perf-12`
+reuse this contract for their archived baselines.
+
 ## Local mode (quick / CI smoke)
 
 Builds both binaries and runs a throwaway single-node server on local disks.

scripts/run_hotpath_warp_ab.sh

@@ -55,6 +55,11 @@ DATA_ROOT="/tmp/rustfs-hotpath-ab"
 OUT_DIR="${PROJECT_ROOT}/target/hotpath-ab/$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || echo run)"
 DRY_RUN="false"
 SKIP_BUILD="false"
+# Free-form provenance lines appended to gate.md under a "Provenance" section
+# (repeatable --provenance-note). CI passes the baseline/candidate commit SHAs
+# and whether each binary came from the actions cache or a source build, so a
+# gate.md is self-describing about *what* it measured (perf-3, backlog#1152).
+PROVENANCE_NOTES=()
 EXTERNAL_ENDPOINT=""     # set -> external mode (warp targets this cluster)
 DEPLOY_HOOK=""           # external mode: command to deploy a phase's binary
 HEALTH_PATH="/health"
@@ -119,6 +124,9 @@ Common:
   --allow-regression      Pass the gate despite a FAIL (deliberate tradeoff).
   --exemption-reason <s>  Reason recorded with --allow-regression.
   --out-dir <path>        Output directory (default target/hotpath-ab/<ts>).
+  --provenance-note <s>   Extra line appended to gate.md's Provenance section
+                          (repeatable). CI records the baseline/candidate SHAs
+                          and their binary source (actions cache vs source build).
   --dry-run               Print the plan and commands without running them.
   -h, --help              Show this help.
 USAGE
@@ -144,6 +152,7 @@ while [[ $# -gt 0 ]]; do
     --exemption-reason) EXEMPTION_REASON="$2"; shift 2 ;;
     --out-dir) OUT_DIR="$2"; shift 2 ;;
     --skip-build) SKIP_BUILD="true"; shift ;;
+    --provenance-note) PROVENANCE_NOTES+=("$2"); shift 2 ;;
     --dry-run) DRY_RUN="true"; shift ;;
     -h|--help) usage; exit 0 ;;
     *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
@@ -358,6 +367,31 @@ for ds_spec in "${DRIVE_SYNC_MATRIX[@]}"; do
   tear_down
 done
 
+# Append a Provenance section to gate.md so a stored or attached gate result is
+# self-describing: which binaries were compared (SHA + source), on what runner,
+# with which warp version and matrix params. The baseline binary's origin SHA is
+# the perf-3 acceptance item; perf-5/perf-12 reuse this same contract for their
+# archived baselines.
+write_provenance() {
+  local md="$OUT_DIR/gate.md"
+  [[ -f "$md" ]] || return 0
+  {
+    echo
+    echo "## Provenance"
+    echo
+    echo "- generated: $(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
+    echo "- runner: $(uname -srm 2>/dev/null || echo unknown)"
+    echo "- warp: $("$WARP_BIN" --version 2>/dev/null | head -n1 || echo unknown)"
+    echo "- matrix: duration=$DURATION rounds=$ROUNDS cooldown=${COOLDOWN_SECS:-driver-default} disks=$DISKS concurrency=$CONCURRENCY"
+    echo "- baseline binary: ${BASELINE_BIN:-<built from $BASELINE_REF>}"
+    echo "- candidate binary: ${CANDIDATE_BIN:-<built from worktree>}"
+    local note
+    for note in ${PROVENANCE_NOTES[@]+"${PROVENANCE_NOTES[@]}"}; do
+      echo "- $note"
+    done
+  } >>"$md"
+}
+
 # --- Gate -------------------------------------------------------------------
 gate_args=(--fail-pct "$FAIL_PCT" --warn-pct "$WARN_PCT" --markdown "$OUT_DIR/gate.md")
 for csv in "${COMPARE_CSVS[@]}"; do gate_args+=(--compare-csv "$csv"); done
@@ -370,7 +404,12 @@ if [[ "$DRY_RUN" == "true" ]]; then
 fi
 
 log "applying relative-budget gate"
+# Do not let a gate FAIL (exit 1) trip set -e before provenance is appended and
+# the status is returned; the gate's exit code is captured and re-emitted.
+set +e
 "$GATE" "${gate_args[@]}"
 gate_status=$?
+set -e
 log "gate result written to $OUT_DIR/gate.md (exit $gate_status)"
+write_provenance
 exit "$gate_status"