From 242424b0fc24efe39bd84f94bb8b390a8175c83f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 15 Jul 2026 15:58:37 +0800 Subject: [PATCH] ci(perf): fit the baseline cache build to the post-LTO profile and self-heal nightly misses (#4841) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(perf): fit the baseline cache build to the post-LTO profile and self-heal nightly misses Every build-baseline-cache run on 2026-07-15 died on its 60min ceiling ('exceeded the maximum execution time of 1h0m0s'): #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a single release build past 60min on sm-standard-2, so the baseline cache never populated. The measured binary must keep the production profile, so raise the job budget to 100min instead of weakening the profile. Also make the A/B job self-heal a same-commit cache miss: when the candidate commit equals the baseline commit (nightly/dispatch on main) and the restore misses, build the binary once, reuse it for both phases, and save it back to the cache under rustfs-baseline- — previously that miss made the rig build the identical commit twice, which no job budget fits post-#4806. The PR-context miss (candidate != baseline, opt-in gate) keeps the double-build fallback and now documents that it will overrun and alert. Refs rustfs/backlog#1152 (perf-3 follow-up). * ci(perf): latest-wins concurrency for the baseline cache build Consumers only restore the binary for the current origin/main tip, so a superseded push build's output is dead weight; cancel it instead of stacking ~65min jobs on the shared sm-standard-2 pool when main merges quickly. A skipped intermediate SHA at most costs one same-commit self-heal in the A/B job. --- .github/workflows/performance-ab.yml | 68 +++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index 796cc2e92..b9848e397 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -75,7 +75,20 @@ jobs: name: Build + cache baseline binary if: github.event_name == 'push' runs-on: sm-standard-2 - timeout-minutes: 60 + # Latest-wins: consumers only ever restore the binary for the *current* + # origin/main tip, so when pushes land faster than the ~65min build, a + # superseded build's output is dead weight — cancel it instead of stacking + # hour-long jobs on the shared runner pool. A skipped intermediate SHA at + # most costs one same-commit self-heal in the A/B job. + concurrency: + group: perf-baseline-build-main + cancel-in-progress: true + # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a + # single release build past 60min on this runner — every cache build on + # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution + # time of 1h0m0s") and the cache never populated. The measured binary must + # keep the production profile, so the budget absorbs the build instead. + timeout-minutes: 100 steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -119,10 +132,11 @@ jobs: runs-on: sm-standard-2 # With perf-3's cached baseline binary the common (cache-hit) nightly is # measurement-only and finishes well under 50min. This ceiling stays - # generous only to absorb the rare cache-miss fallback, which reverts to the - # ~65min source double-build; a timeout there now surfaces via the - # alert-on-failure job (it fires on cancelled/timed-out, not just failure). - # perf-6 recalibrates the budget once the noise study lands. + # generous only to absorb the same-commit cache-miss self-heal (~65min + # single build with the post-#4806 LTO profile + measurement). A timeout + # surfaces via the alert-on-failure job (it fires on cancelled/timed-out, + # not just failure). perf-6 recalibrates the budget once the noise study + # lands. timeout-minutes: 120 steps: - name: Checkout repository @@ -184,6 +198,30 @@ jobs: path: baseline-bin/rustfs key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }} + # Self-heal: on a nightly/dispatch run where the candidate commit IS the + # baseline commit, a cache miss would make the rig build the same commit + # twice (~65min per side with the post-#4806 LTO profile — no job budget + # fits that). Build it once here, reuse it for both phases, and save it + # back to the cache so the next run hits. + - name: Build baseline on cache miss (same-commit self-heal) + id: selfheal + if: >- + steps.baseline_cache.outputs.cache-hit != 'true' && + steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha + run: | + set -euo pipefail + cargo build --release --bin rustfs + mkdir -p baseline-bin + cp target/release/rustfs baseline-bin/rustfs + echo "built=true" >> "$GITHUB_OUTPUT" + + - name: Save self-healed baseline to cache + if: steps.selfheal.outputs.built == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: baseline-bin/rustfs + key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }} + - name: Run warp A/B and gate id: ab run: | @@ -200,25 +238,33 @@ jobs: baseline_sha="${{ steps.commits.outputs.baseline_sha }}" candidate_sha="${{ steps.commits.outputs.candidate_sha }}" baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}" + selfheal_built="${{ steps.selfheal.outputs.built }}" args=(--duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180) - if [[ "$baseline_hit" == "true" ]]; then + if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" ]]; then chmod +x baseline-bin/rustfs base_bin="$PWD/baseline-bin/rustfs" args+=(--baseline-bin "$base_bin") - base_src="actions-cache (rustfs-baseline-$baseline_sha)" + if [[ "$baseline_hit" == "true" ]]; then + base_src="actions-cache (rustfs-baseline-$baseline_sha)" + else + base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)" + fi if [[ "$candidate_sha" == "$baseline_sha" ]]; then # Nightly on main: the candidate is the same commit as the baseline, - # so reuse the one cached binary for both phases and skip all builds. + # so reuse the one binary for both phases and skip all builds. args+=(--candidate-bin "$base_bin" --skip-build) - cand_src="actions-cache (same commit as baseline)" + cand_src="same binary as baseline (same commit)" else cand_src="source build of the checked-out ref" fi else - # Cache miss (binary evicted or not built for this SHA): fall back to - # the slow source double-build so the run still produces a result. + # Cache miss with candidate != baseline (opt-in PR gate only): fall + # back to the source double-build. With the post-#4806 LTO profile + # this will overrun the job budget and alert; rerun once the push + # cache build for origin/main has completed, or wait for perf-7's + # merge-base caching. args+=(--baseline-ref origin/main) base_src="source build of origin/main (cache miss)" cand_src="source build of the checked-out ref"