mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
Zhengchao An
d73c8a783a
Every push to main now builds the release binary once and stores it in the actions cache as rustfs-baseline-<sha> (build-baseline-cache job). The A/B job restores it for origin/main and passes --baseline-bin; on the nightly, where the candidate commit equals the baseline, one cached binary serves both phases with --skip-build and the run does zero source builds. A cache miss falls back to the source double-build via --baseline-ref origin/main. This removes the ~32min-per-side double build that pushed the 24-cell nightly past its 120min ceiling (2026-07-11..07-14 all cancelled on timeout). Also: - alert-on-failure now fires on cancelled/timed-out, not just failure, so a timed-out nightly is no longer silent (the composite action already reports cancelled jobs); removed the stale perf-2 TODO now that the alert job exists. - run_hotpath_warp_ab.sh appends a Provenance section to gate.md (baseline and candidate SHAs + binary source, runner, warp version, matrix params) via a repeatable --provenance-note; the gate exit code is preserved. Refs rustfs/backlog#1152 (perf-3).
346 lines
15 KiB
YAML
346 lines
15 KiB
YAML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4).
|
|
#
|
|
# Two entry points, honestly scoped:
|
|
# * schedule (nightly, on main): post-merge detection — catches a regression
|
|
# within 24h of landing, not before merge.
|
|
# * pull_request labeled `perf-ab`: opt-in pre-merge gate for a specific PR.
|
|
# The `perf-deliberate-tradeoff` label runs the gate with --allow-regression so
|
|
# a deliberate correctness cost (e.g. the #4221 fsync durability fix) is
|
|
# recorded but does not block (rustfs/backlog#935 correction 1).
|
|
|
|
name: Performance A/B
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "0 6 * * *" # 06:00 UTC nightly, against main
|
|
workflow_dispatch:
|
|
inputs:
|
|
duration:
|
|
description: "warp duration per round (short by default to fit the double-build budget)"
|
|
required: false
|
|
default: "12s"
|
|
type: string
|
|
allow_regression:
|
|
description: "Pass the gate despite a FAIL (deliberate tradeoff)"
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
pull_request:
|
|
types: [labeled, synchronize, reopened]
|
|
push:
|
|
# Every main commit pre-builds and caches its release binary (perf-3) so the
|
|
# nightly A/B restores a ready baseline instead of paying the double build.
|
|
branches: [main]
|
|
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
|
|
# Per-PR: a new push cancels the previous (up to 90-minute) A/B run instead of
|
|
# stacking them. Nightly schedule and manual dispatch get a unique group and
|
|
# always run to completion.
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
RUST_BACKTRACE: 1
|
|
|
|
jobs:
|
|
# perf-3: on every push to main, build the release binary once and cache it
|
|
# keyed by commit SHA (rustfs-baseline-<sha>). The nightly A/B (and, later, the
|
|
# perf-7 PR gate) restore this instead of paying the ~32min-per-side source
|
|
# build. That double build is what pushed the expanded 24-cell nightly past its
|
|
# ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
|
|
# builds off the shared cargo cache keep each push cheap, and building on the
|
|
# same sm-standard-2 runner the A/B measures on guarantees the cached binary is
|
|
# ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
|
|
# are cancelled ~7/8 of the time and are not a reliable baseline.
|
|
build-baseline-cache:
|
|
name: Build + cache baseline binary
|
|
if: github.event_name == 'push'
|
|
runs-on: sm-standard-2
|
|
timeout-minutes: 60
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Build release rustfs
|
|
run: cargo build --release --bin rustfs
|
|
|
|
- name: Stage binary for cache
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p baseline-bin
|
|
cp target/release/rustfs baseline-bin/rustfs
|
|
|
|
- name: Cache baseline binary by SHA
|
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: baseline-bin/rustfs
|
|
key: rustfs-baseline-${{ github.sha }}
|
|
|
|
warp-ab:
|
|
name: Warp A/B budget gate
|
|
# Always run on schedule / manual dispatch. Opt-in on PRs: only when the
|
|
# `perf-ab` label is present, and for `labeled` events only when the label
|
|
# being added is `perf-ab` itself (adding an unrelated label to an opted-in
|
|
# PR must not re-run the gate). Never on push — that event only feeds
|
|
# build-baseline-cache above.
|
|
if: >-
|
|
github.event_name == 'schedule' ||
|
|
github.event_name == 'workflow_dispatch' ||
|
|
(github.event_name == 'pull_request' &&
|
|
contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
|
|
(github.event.action != 'labeled' || github.event.label.name == 'perf-ab'))
|
|
runs-on: sm-standard-2
|
|
# With perf-3's cached baseline binary the common (cache-hit) nightly is
|
|
# measurement-only and finishes well under 50min. This ceiling stays
|
|
# generous only to absorb the rare cache-miss fallback, which reverts to the
|
|
# ~65min source double-build; a timeout there now surfaces via the
|
|
# alert-on-failure job (it fires on cancelled/timed-out, not just failure).
|
|
# perf-6 recalibrates the budget once the noise study lands.
|
|
timeout-minutes: 120
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
with:
|
|
fetch-depth: 0 # baseline is built from origin/main
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: stable
|
|
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Install warp
|
|
run: |
|
|
set -euo pipefail
|
|
WARP_VERSION="1.0.0"
|
|
curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \
|
|
| sudo tar -xz -C /usr/local/bin warp
|
|
warp --version
|
|
|
|
- name: Decide exemption
|
|
id: exempt
|
|
run: |
|
|
allow="false"
|
|
if [[ "${{ github.event_name }}" == "pull_request" ]] \
|
|
&& ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then
|
|
allow="true"
|
|
fi
|
|
if [[ "${{ github.event.inputs.allow_regression }}" == "true" ]]; then
|
|
allow="true"
|
|
fi
|
|
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
|
|
|
|
# perf-3: resolve the commits so the cache can be keyed by SHA. The
|
|
# baseline is origin/main; the candidate is the checked-out ref. On the
|
|
# nightly (checkout == main) they are the same commit, so one cached binary
|
|
# serves both phases and the run does zero source builds.
|
|
- name: Resolve baseline / candidate commits
|
|
id: commits
|
|
run: |
|
|
set -euo pipefail
|
|
baseline_sha="$(git rev-parse origin/main)"
|
|
candidate_sha="$(git rev-parse HEAD)"
|
|
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
|
|
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
|
|
echo "baseline commit: $baseline_sha"
|
|
echo "candidate commit: $candidate_sha"
|
|
|
|
# Exact-key restore of the baseline binary built by build-baseline-cache
|
|
# when origin/main last landed. A miss (binary evicted or not built yet)
|
|
# leaves cache-hit unset and the rig falls back to a source build.
|
|
- name: Restore cached baseline binary
|
|
id: baseline_cache
|
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: baseline-bin/rustfs
|
|
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
|
|
|
|
- name: Run warp A/B and gate
|
|
id: ab
|
|
run: |
|
|
set -euo pipefail
|
|
# Budget note: with perf-3's cached baseline the nightly does no source
|
|
# build on a cache hit, so the wall-clock is dominated by the short warp
|
|
# matrix — duration/rounds/cooldown are kept small to fit all 24 cells
|
|
# (6 workloads x 2 phases x 2 drive-sync) rather than dropping cells.
|
|
# --health-timeout 180 outlasts the server's own 120s startup-readiness
|
|
# budget, which the rig's previous 60s health poll undershot (the first
|
|
# two nightly failures). perf-6 recalibrates these once the noise study
|
|
# lands.
|
|
duration="${{ github.event.inputs.duration || '12s' }}"
|
|
baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
|
|
candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
|
|
baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
|
|
|
|
args=(--duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)
|
|
|
|
if [[ "$baseline_hit" == "true" ]]; then
|
|
chmod +x baseline-bin/rustfs
|
|
base_bin="$PWD/baseline-bin/rustfs"
|
|
args+=(--baseline-bin "$base_bin")
|
|
base_src="actions-cache (rustfs-baseline-$baseline_sha)"
|
|
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
|
|
# Nightly on main: the candidate is the same commit as the baseline,
|
|
# so reuse the one cached binary for both phases and skip all builds.
|
|
args+=(--candidate-bin "$base_bin" --skip-build)
|
|
cand_src="actions-cache (same commit as baseline)"
|
|
else
|
|
cand_src="source build of the checked-out ref"
|
|
fi
|
|
else
|
|
# Cache miss (binary evicted or not built for this SHA): fall back to
|
|
# the slow source double-build so the run still produces a result.
|
|
args+=(--baseline-ref origin/main)
|
|
base_src="source build of origin/main (cache miss)"
|
|
cand_src="source build of the checked-out ref"
|
|
fi
|
|
|
|
args+=(--provenance-note "baseline commit: $baseline_sha - $base_src")
|
|
args+=(--provenance-note "candidate commit: $candidate_sha - $cand_src")
|
|
|
|
if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
|
|
args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override")
|
|
fi
|
|
# Do not let a gate FAIL abort the job here; capture status and surface
|
|
# it after the PR comment is posted.
|
|
set +e
|
|
bash scripts/run_hotpath_warp_ab.sh "${args[@]}"
|
|
echo "status=$?" >> "$GITHUB_OUTPUT"
|
|
set -e
|
|
# Locate the newest run dir + gate.md for the summary/comment/artifact
|
|
# steps. On a startup failure there is no gate.md, but the run dir still
|
|
# holds server-logs/ for diagnosis.
|
|
# Run dirs are UTC-timestamp names (no special chars); ls is safe here.
|
|
# shellcheck disable=SC2012
|
|
run_dir="$(ls -td target/hotpath-ab/*/ 2>/dev/null | head -n1 || true)"
|
|
echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
|
|
# shellcheck disable=SC2012
|
|
gate_md="$(ls -t target/hotpath-ab/*/gate.md 2>/dev/null | head -n1 || true)"
|
|
echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Upload A/B results
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: hotpath-warp-ab-${{ github.run_number }}
|
|
# Includes per-cell median_summary.csv / baseline_compare.csv, gate.md,
|
|
# and server-logs/ (rustfs.log + startup env per phase) so a failed run
|
|
# is diagnosable. Short retention: this is churny nightly debug data.
|
|
path: target/hotpath-ab/
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Write gate summary
|
|
if: always()
|
|
run: |
|
|
set -euo pipefail
|
|
status="${{ steps.ab.outputs.status }}"
|
|
gate_md="${{ steps.ab.outputs.gate_md }}"
|
|
run_dir="${{ steps.ab.outputs.run_dir }}"
|
|
{
|
|
echo "## Hotpath warp A/B — run ${{ github.run_number }}"
|
|
echo
|
|
if [[ "$status" == "0" ]]; then
|
|
echo "Rig/gate exit: \`0\` (pass or warn)."
|
|
else
|
|
echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**."
|
|
fi
|
|
echo
|
|
if [[ -n "$gate_md" && -f "$gate_md" ]]; then
|
|
cat "$gate_md"
|
|
else
|
|
echo "No \`gate.md\` produced — the rig failed **before** the gate"
|
|
echo "(most likely server startup / health). Failing phase(s) below;"
|
|
echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact."
|
|
if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then
|
|
for f in "$run_dir"/server-logs/*.log; do
|
|
[[ -f "$f" ]] || continue
|
|
echo
|
|
echo "<details><summary>$(basename "$f")</summary>"
|
|
echo
|
|
echo '```'
|
|
tail -n 30 "$f"
|
|
echo '```'
|
|
echo "</details>"
|
|
done
|
|
fi
|
|
fi
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Comment gate result on PR
|
|
if: always() && github.event_name == 'pull_request' && steps.ab.outputs.gate_md != ''
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
gh pr comment "${{ github.event.pull_request.number }}" --body-file "${{ steps.ab.outputs.gate_md }}"
|
|
|
|
# Scheduled failure alerting is handled by the alert-on-failure job below
|
|
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
|
|
|
|
- name: Enforce gate
|
|
if: always()
|
|
run: |
|
|
status="${{ steps.ab.outputs.status }}"
|
|
if [[ "$status" != "0" ]]; then
|
|
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / PR comment / gate.md artifact." >&2
|
|
exit "$status"
|
|
fi
|
|
echo "warp A/B budget gate passed."
|
|
|
|
alert-on-failure:
|
|
name: Alert on scheduled failure
|
|
needs: [warp-ab]
|
|
# `always()` is required: without it this job is skipped when a needed
|
|
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
|
# ci-8); PR and manual dispatch failures are already watched by a human.
|
|
# `cancelled` is included alongside `failure` on purpose: a job that hits
|
|
# timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
|
|
# timeouts went silent precisely because the guard was failure-only. The
|
|
# composite action already reports cancelled/timed-out jobs in the issue
|
|
# body. (Scheduled runs get a unique concurrency group with
|
|
# cancel-in-progress off, so a cancellation here means a timeout/manual
|
|
# abort, never a superseding run.)
|
|
if: >-
|
|
always() && github.event_name == 'schedule' &&
|
|
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
issues: write
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
- name: Open or update failure-tracking issue
|
|
uses: ./.github/actions/schedule-failure-issue
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|