diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index a7ecf1d4b..de4986388 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -22,13 +22,6 @@ # correctness cost (e.g. the #4221 fsync durability fix) is recorded, not # blocked (rustfs/backlog#935 correction 1). -# DISABLED. This workflow is switched off in the repository's Actions settings -# (state: disabled_manually) and does not run on any trigger, including its cron -# and workflow_dispatch. That state lives in GitHub's UI and is invisible when -# reading this file, which has already misled at least one audit — hence this -# banner. Re-enabling is a UI action; anyone doing so should first check that the -# workflow still matches the current CI layout. See rustfs/backlog#1603. -# name: Performance A/B on: @@ -37,7 +30,7 @@ on: workflow_dispatch: inputs: duration: - description: "warp duration per round (short by default to fit the double-build budget)" + description: "warp duration per round" required: false default: "12s" type: string @@ -46,12 +39,8 @@ on: required: false default: false type: boolean - push: - # Every main commit pre-builds and caches its release binary (perf-3) so the - # nightly A/B restores a ready baseline instead of paying the double build. - branches: [main] - permissions: + actions: read contents: read env: @@ -59,83 +48,19 @@ env: RUST_BACKTRACE: 1 jobs: - # perf-3: on every push to main, build the release binary once and cache it - # keyed by commit SHA (rustfs-baseline-). The warp-ab measurements - # restore this instead of paying the ~32min-per-side source - # build. That double build is what pushed the expanded 24-cell nightly past its - # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental - # builds off the shared cargo cache keep each push cheap, and building on the - # same sm-standard-2 runner the A/B measures on guarantees the cached binary is - # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those - # are cancelled ~7/8 of the time and are not a reliable baseline. - build-baseline-cache: - name: Build + cache baseline binary - if: github.event_name == 'push' - runs-on: sm-standard-2 - # Latest-wins: consumers only ever restore the binary for the *current* - # origin/main tip, so when pushes land faster than the ~65min build, a - # superseded build's output is dead weight — cancel it instead of stacking - # hour-long jobs on the shared runner pool. A skipped intermediate SHA at - # most costs one same-commit self-heal in the A/B job. - concurrency: - group: perf-baseline-build-main - cancel-in-progress: true - # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a - # single release build past 60min on this runner — every cache build on - # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution - # time of 1h0m0s") and the cache never populated. The measured binary must - # keep the production profile, so the budget absorbs the build instead. - timeout-minutes: 100 - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - - name: Setup Rust environment - uses: ./.github/actions/setup - with: - rust-version: stable - cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }} - cache-save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build release rustfs - run: cargo build --release --bin rustfs - - - name: Stage binary for cache - run: | - set -euo pipefail - mkdir -p baseline-bin - cp target/release/rustfs baseline-bin/rustfs - - - name: Cache baseline binary by SHA - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: baseline-bin/rustfs - key: rustfs-baseline-${{ github.sha }} - warp-ab: name: Warp A/B budget gate - # Always run on schedule / manual dispatch. Never on push — that event only - # feeds build-baseline-cache above. - if: >- - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' runs-on: sm-standard-2 - # With perf-3's cached baseline binary the common (cache-hit) nightly is - # measurement-only and finishes well under 50min. This ceiling stays - # generous only to absorb the same-commit cache-miss self-heal (~65min - # single build with the post-#4806 LTO profile + measurement). A timeout - # surfaces via the alert-on-failure job (it fires on cancelled/timed-out, - # not just failure). perf-6 recalibrates the budget once the noise study - # lands. - timeout-minutes: 120 + # A normal nightly restores the last successful binary and builds only the + # candidate; daily access keeps that cache warm. A cache miss may build both + # and needs room for the A/B run plus artifact and cache publication. + timeout-minutes: 180 steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - fetch-depth: 0 # baseline is built from origin/main + fetch-depth: 0 # baseline may be an earlier successful scheduled head - name: Setup Rust environment uses: ./.github/actions/setup @@ -163,24 +88,55 @@ jobs: fi echo "allow_regression=$allow" >> "$GITHUB_OUTPUT" - # perf-3: resolve the commits so the cache can be keyed by SHA. The - # baseline is origin/main; the candidate is the checked-out ref. On the - # nightly (checkout == main) they are the same commit, so one cached binary - # serves both phases and the run does zero source builds. + # A failed regression run must keep comparing against the last known-good + # scheduled head. Otherwise the next nightly would absorb the regression + # into its baseline and turn green without a fix. + - name: Find last successful scheduled baseline + id: scheduled_baseline + if: github.event_name == 'schedule' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + result-encoding: string + script: | + const { data } = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "performance-ab.yml", + event: "schedule", + status: "success", + per_page: 1, + }); + return data.workflow_runs[0]?.head_sha ?? ""; + + # Manual runs compare a selected ref with current main. Scheduled runs + # compare current main with the last successful scheduled head. With no + # history, the first run measures the candidate against itself and seeds + # that head only if the complete rig succeeds. - name: Resolve baseline / candidate commits id: commits + env: + SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }} run: | set -euo pipefail - baseline_sha="$(git rev-parse origin/main)" candidate_sha="$(git rev-parse HEAD)" + if [[ "${{ github.event_name }}" == "schedule" ]]; then + baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}" + if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then + echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2 + exit 1 + fi + else + baseline_sha="$(git rev-parse origin/main)" + fi + git cat-file -e "${baseline_sha}^{commit}" echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT" echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT" echo "baseline commit: $baseline_sha" echo "candidate commit: $candidate_sha" - # Exact-key restore of the baseline binary built by build-baseline-cache - # when origin/main last landed. A miss (binary evicted or not built yet) - # leaves cache-hit unset and the rig falls back to a source build. + # Exact-key restore of the candidate binary saved by its successful + # scheduled run. A miss leaves cache-hit unset and falls back to a source + # build of that known-good head. - name: Restore cached baseline binary id: baseline_cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 @@ -270,11 +226,11 @@ jobs: elif [[ "$selfheal_built" == "true" ]]; then base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)" else - base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)" + base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)" fi if [[ "$candidate_sha" == "$baseline_sha" ]]; then - # Nightly on main: the candidate is the same commit as the baseline, - # so reuse the one binary for both phases and skip all builds. + # No commits landed since the last successful baseline, so reuse + # the one binary for both phases and measure only rig drift. args+=(--candidate-bin "$base_bin") cand_src="same binary as baseline (same commit)" elif [[ "$candidate_built" == "true" ]]; then @@ -362,6 +318,23 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" + - name: Stage successful candidate baseline + if: >- + steps.ab.outputs.status == '0' && + steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha + run: | + set -euo pipefail + cp candidate-bin/rustfs baseline-bin/rustfs + + - name: Cache successful candidate baseline + if: >- + steps.ab.outputs.status == '0' && + steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: baseline-bin/rustfs + key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }} + # Scheduled failure alerting is handled by the alert-on-failure job below # (perf-2 consuming ci-8's schedule-failure-issue composite action). diff --git a/scripts/security/check_performance_ab_workflow.sh b/scripts/security/check_performance_ab_workflow.sh index 5465e4fdf..09b97cc0d 100755 --- a/scripts/security/check_performance_ab_workflow.sh +++ b/scripts/security/check_performance_ab_workflow.sh @@ -13,8 +13,29 @@ require_absent_pattern() { fi } +require_present_pattern() { + local pattern="$1" + local description="$2" + + if ! grep -Eq -- "$pattern" "$workflow"; then + echo "invalid performance A/B workflow contract: $description" >&2 + exit 1 + fi +} + require_absent_pattern '(^|[^[:alnum:]_])pull_request(_target)?([^[:alnum:]_]|$)' "the workflow must not contain PR event handling" require_absent_pattern 'pull-requests[[:space:]]*:[[:space:]]*write' "the workflow must not receive PR write permission" require_absent_pattern 'permissions[[:space:]]*:[[:space:]]*write-all' "the workflow must not receive broad write permission" +require_absent_pattern '^[[:space:]]*push:' "the workflow must not spend a release build on every main push" +require_present_pattern 'listWorkflowRuns' "the scheduled baseline must come from workflow history" +require_present_pattern 'status:[[:space:]]*"success"' "the scheduled baseline must be a successful run" +require_present_pattern 'SCHEDULED_BASELINE_SHA' "the resolved scheduled baseline must reach the comparison" +require_present_pattern "SCHEDULED_BASELINE_SHA:-\\\$candidate_sha" "the first scheduled run must seed from its verified candidate" +require_present_pattern 'git merge-base --is-ancestor' "the scheduled baseline must stay on candidate history" +require_present_pattern 'Cache successful candidate baseline' "a successful candidate must become the next cached baseline" +if ! sed -n '/^ warp-ab:/,/^ alert-on-failure:/p' "$workflow" | grep -Eq '^ timeout-minutes:[[:space:]]*180([[:space:]]|$)'; then + echo "invalid performance A/B workflow contract: the cold-cache path must fit both builds, the A/B run, and evidence publication" >&2 + exit 1 +fi -echo "Performance A/B workflow trust boundary ok." +echo "Performance A/B workflow contract ok."