diff --git a/.github/actions/schedule-failure-issue/README.md b/.github/actions/schedule-failure-issue/README.md index e704c0e77..d808dfae9 100644 --- a/.github/actions/schedule-failure-issue/README.md +++ b/.github/actions/schedule-failure-issue/README.md @@ -74,7 +74,6 @@ Notes: - `.github/workflows/e2e-s3tests.yml` (weekly full s3-tests sweep) - `.github/workflows/mint.yml` (weekly multi-SDK mint run) - `.github/workflows/fuzz.yml` (nightly fuzz corpus jobs) -- `.github/workflows/performance-ab.yml` (nightly warp A/B gate) - **Pending:** the ci-7 e2e nightly-full workflow does not exist yet (backlog#1149 ci-7). When it lands, wire it up with the snippet above. diff --git a/.github/scheduled-validations.json b/.github/scheduled-validations.json index 060200715..d9c25fac2 100644 --- a/.github/scheduled-validations.json +++ b/.github/scheduled-validations.json @@ -18,7 +18,6 @@ "never_ran_grace_until": "2026-09-08T00:00:00Z" }, { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 }, - { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 }, { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792, diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 209fffead..6dc3492e6 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -29,7 +29,6 @@ on: - 'scripts/test_package_service_scripts.sh' - 'scripts/test_package_versions.sh' - 'scripts/test_docker_workflow.py' - - 'scripts/security/check_performance_ab_workflow.sh' - 'scripts/security/check_preview_release_workflow.sh' - 'scripts/security/check_tier_artifact_workflow.sh' - 'scripts/security/check_workflow_pins.sh' @@ -48,7 +47,6 @@ on: - 'scripts/test_package_service_scripts.sh' - 'scripts/test_package_versions.sh' - 'scripts/test_docker_workflow.py' - - 'scripts/security/check_performance_ab_workflow.sh' - 'scripts/security/check_preview_release_workflow.sh' - 'scripts/security/check_tier_artifact_workflow.sh' - 'scripts/security/check_workflow_pins.sh' @@ -155,9 +153,6 @@ jobs: - name: Check preview release workflow policy run: ./scripts/security/check_preview_release_workflow.sh - - name: Check performance A/B workflow trust boundary - run: ./scripts/security/check_performance_ab_workflow.sh - - name: Check tier evidence workflow isolation run: ./scripts/security/check_tier_artifact_workflow.sh diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6ceefefad..8b2deb1b4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -45,7 +45,7 @@ on: # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00), # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17), - # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes. + # and e2e-replication-nightly (04:00) lanes. - cron: "43 7 * * 0" concurrency: diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index 5fa9ac271..59a8bd849 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -38,8 +38,8 @@ name: e2e-nightly on: workflow_dispatch: schedule: - # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00), - # stale (01:30) and performance-ab (06:00). + # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00) + # and stale (01:30). - cron: "29 4 * * *" # Only alert-on-failure needs more than read access; it declares its own diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml deleted file mode 100644 index aa1ddb528..000000000 --- a/.github/workflows/performance-ab.yml +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright 2024 RustFS Team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4). -# -# Two entry points, honestly scoped: -# * schedule (nightly, on main): post-merge detection — catches a regression -# within 24h of landing, not before merge. -# * workflow_dispatch: an explicitly selected trusted ref. -# The dispatch input can run the gate with --allow-regression so a deliberate -# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not -# blocked (rustfs/backlog#935 correction 1). - -name: Performance A/B - -on: - schedule: - - cron: "31 6 * * *" # 06:31 UTC nightly, against main - workflow_dispatch: - inputs: - duration: - description: "warp duration per round" - required: false - default: "12s" - type: string - allow_regression: - description: "Pass the gate despite a FAIL (deliberate tradeoff)" - required: false - default: false - type: boolean -permissions: - actions: read - contents: read - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: 1 - -jobs: - warp-ab: - if: github.repository == 'rustfs/rustfs' - name: Warp A/B budget gate - runs-on: sm-standard-2 - # A normal nightly restores the last successful binary and builds only the - # candidate; daily access keeps that cache warm. A cache miss may build both - # and needs room for the A/B run plus artifact and cache publication. - timeout-minutes: 180 - steps: - - name: Checkout repository - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 - with: - persist-credentials: false - fetch-depth: 0 # baseline may be an earlier successful scheduled head - - - name: Setup Rust environment - uses: ./.github/actions/setup - with: - rust-version: stable - cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }} - cache-save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Install warp - run: | - set -euo pipefail - WARP_VERSION="1.0.0" - curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \ - | sudo tar -xz -C /usr/local/bin warp - warp --version - - - name: Decide exemption - id: exempt - env: - INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }} - run: | - allow="false" - if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then - allow="true" - fi - echo "allow_regression=$allow" >> "$GITHUB_OUTPUT" - - # A failed regression run must keep comparing against the last known-good - # scheduled head. Otherwise the next nightly would absorb the regression - # into its baseline and turn green without a fix. - - name: Find last successful scheduled baseline - id: scheduled_baseline - if: github.event_name == 'schedule' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - result-encoding: string - script: | - const { data } = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: "performance-ab.yml", - event: "schedule", - status: "success", - per_page: 1, - }); - return data.workflow_runs[0]?.head_sha ?? ""; - - # Manual runs compare a selected ref with current main. Scheduled runs - # compare current main with the last successful scheduled head. With no - # history, the first run measures the candidate against itself and seeds - # that head only if the complete rig succeeds. - - name: Resolve baseline / candidate commits - id: commits - env: - SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }} - run: | - set -euo pipefail - candidate_sha="$(git rev-parse HEAD)" - if [[ "${{ github.event_name }}" == "schedule" ]]; then - baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}" - else - baseline_sha="$(git rev-parse origin/main)" - fi - git cat-file -e "${baseline_sha}^{commit}" - if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then - echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2 - exit 1 - fi - echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT" - echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT" - echo "baseline commit: $baseline_sha" - echo "candidate commit: $candidate_sha" - - # Exact-key restore of the candidate binary saved by its successful - # scheduled run. A miss leaves cache-hit unset and falls back to a source - # build of that known-good head. - - name: Restore cached baseline binary - id: baseline_cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: baseline-bin/rustfs - key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }} - - # Self-heal: on a nightly/dispatch run where the candidate commit IS the - # baseline commit, a cache miss would make the rig build the same commit - # twice (~65min per side with the post-#4806 LTO profile — no job budget - # fits that). Build it once here, reuse it for both phases, and save it - # back to the cache so the next run hits. - - name: Build baseline on cache miss (same-commit self-heal) - id: selfheal - if: >- - steps.baseline_cache.outputs.cache-hit != 'true' && - steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha - run: | - set -euo pipefail - cargo build --release --bin rustfs - mkdir -p baseline-bin - cp target/release/rustfs baseline-bin/rustfs - echo "built=true" >> "$GITHUB_OUTPUT" - - - name: Build baseline on cache miss (different candidate) - id: baseline_build - if: >- - steps.baseline_cache.outputs.cache-hit != 'true' && - steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha - run: | - set -euo pipefail - baseline_root="$RUNNER_TEMP/rustfs-baseline-${{ github.run_id }}" - baseline_target="$RUNNER_TEMP/rustfs-baseline-target-${{ github.run_id }}" - git worktree add --detach "$baseline_root" "${{ steps.commits.outputs.baseline_sha }}" - cargo build --release --manifest-path "$baseline_root/Cargo.toml" --bin rustfs --target-dir "$baseline_target" - mkdir -p baseline-bin - cp "$baseline_target/release/rustfs" baseline-bin/rustfs - git worktree remove --force "$baseline_root" - echo "built=true" >> "$GITHUB_OUTPUT" - - - name: Build candidate binary - id: candidate_build - if: steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha - run: | - set -euo pipefail - cargo build --release --bin rustfs - mkdir -p candidate-bin - cp target/release/rustfs candidate-bin/rustfs - echo "built=true" >> "$GITHUB_OUTPUT" - - - name: Save self-healed baseline to cache - if: steps.selfheal.outputs.built == 'true' || steps.baseline_build.outputs.built == 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: baseline-bin/rustfs - key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }} - - - name: Run warp A/B and gate - id: ab - env: - INPUT_DURATION: ${{ github.event.inputs.duration }} - run: | - set -euo pipefail - # The formal runner executes A1 baseline -> B1 candidate -> B2 candidate - # -> A2 baseline for each workload and drive-sync cell. It requires three - # rounds per leg to emit tail latency and error-rate evidence. - # --health-timeout 180 outlasts the server's own 120s startup-readiness - # budget, which the rig's previous 60s health poll undershot (the first - # two nightly failures). perf-6 recalibrates these once the noise study - # lands. - duration="${INPUT_DURATION:-12s}" - baseline_sha="${{ steps.commits.outputs.baseline_sha }}" - candidate_sha="${{ steps.commits.outputs.candidate_sha }}" - baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}" - selfheal_built="${{ steps.selfheal.outputs.built }}" - baseline_built="${{ steps.baseline_build.outputs.built }}" - candidate_built="${{ steps.candidate_build.outputs.built }}" - - args=(--duration "$duration" --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision "$baseline_sha" --candidate-revision "$candidate_sha") - - if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" || "$baseline_built" == "true" ]]; then - chmod +x baseline-bin/rustfs - base_bin="$PWD/baseline-bin/rustfs" - args+=(--baseline-bin "$base_bin") - if [[ "$baseline_hit" == "true" ]]; then - base_src="actions-cache (rustfs-baseline-$baseline_sha)" - elif [[ "$selfheal_built" == "true" ]]; then - base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)" - else - base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)" - fi - if [[ "$candidate_sha" == "$baseline_sha" ]]; then - # No commits landed since the last successful baseline, so reuse - # the one binary for both phases and measure only rig drift. - args+=(--candidate-bin "$base_bin") - cand_src="same binary as baseline (same commit)" - elif [[ "$candidate_built" == "true" ]]; then - chmod +x candidate-bin/rustfs - args+=(--candidate-bin "$PWD/candidate-bin/rustfs") - cand_src="source build of the checked-out ref" - else - echo "::error::candidate binary was not built" >&2 - exit 2 - fi - else - echo "::error::baseline binary was not restored or built" >&2 - exit 2 - fi - - echo "baseline binary: $base_src" - echo "candidate binary: $cand_src" - - if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then - args+=(--allow-regression --exemption-reason "workflow dispatch override") - fi - # Do not let a gate FAIL abort the job here; capture status and surface - # it after the step summary is written. - set +e - bash scripts/run_hotpath_warp_abba.sh "${args[@]}" - echo "status=$?" >> "$GITHUB_OUTPUT" - set -e - # Locate the newest run dir + gate.md for the summary/comment/artifact - # steps. On a startup failure there is no gate.md, but the run dir still - # holds server-logs/ for diagnosis. - # Run dirs are UTC-timestamp names (no special chars); ls is safe here. - # shellcheck disable=SC2012 - run_dir="$(ls -td target/hotpath-abba/*/ 2>/dev/null | head -n1 || true)" - echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT" - # shellcheck disable=SC2012 - gate_md="$(ls -t target/hotpath-abba/*/candidate_gate.md 2>/dev/null | head -n1 || true)" - echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT" - - - name: Upload A/B results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: hotpath-warp-ab-${{ github.run_number }} - # Includes per-cell median_summary.csv / baseline_compare.csv, both gates, - # and server-logs/ (rustfs.log + startup env per phase) so a failed run - # is diagnosable. Short retention: this is churny nightly debug data. - path: target/hotpath-abba/ - if-no-files-found: warn - retention-days: 14 - - - name: Write gate summary - if: always() - run: | - set -euo pipefail - status="${{ steps.ab.outputs.status }}" - gate_md="${{ steps.ab.outputs.gate_md }}" - run_dir="${{ steps.ab.outputs.run_dir }}" - { - echo "## Hotpath warp A/B — run ${{ github.run_number }}" - echo - if [[ "$status" == "0" ]]; then - echo "Rig/gate exit: \`0\` (pass or warn)." - else - echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**." - fi - echo - if [[ -n "$gate_md" && -f "$gate_md" ]]; then - cat "$gate_md" - else - echo "No \`gate.md\` produced — the rig failed **before** the gate" - echo "(most likely server startup / health). Failing phase(s) below;" - echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact." - if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then - for f in "$run_dir"/server-logs/*.log; do - [[ -f "$f" ]] || continue - echo - echo "
$(basename "$f")" - echo - echo '```' - tail -n 30 "$f" - echo '```' - echo "
" - done - fi - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Stage successful candidate baseline - if: >- - steps.ab.outputs.status == '0' && - steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha - run: | - set -euo pipefail - cp candidate-bin/rustfs baseline-bin/rustfs - - - name: Cache successful candidate baseline - if: >- - steps.ab.outputs.status == '0' && - steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: baseline-bin/rustfs - key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }} - - # Scheduled failure alerting is handled by the alert-on-failure job below - # (perf-2 consuming ci-8's schedule-failure-issue composite action). - - - name: Enforce gate - if: always() - run: | - status="${{ steps.ab.outputs.status }}" - if [[ -z "$status" ]]; then - echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2 - exit 1 - fi - if [[ "$status" != "0" ]]; then - echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2 - exit "$status" - fi - echo "warp A/B budget gate passed." - - alert-on-failure: - name: Alert on scheduled failure - needs: [warp-ab] - # `always()` is required: without it this job is skipped when a needed - # job fails. Alerts only for scheduled (nightly) runs (backlog#1149 - # ci-8); manual dispatch failures are already watched by a human. - # `cancelled` is included alongside `failure` on purpose: a job that hits - # timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly - # timeouts went silent precisely because the guard was failure-only. The - # composite action already reports cancelled/timed-out jobs in the issue - # body. - if: >- - github.repository == 'rustfs/rustfs' && ( - always() && github.event_name == 'schedule' && - (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) - ) - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - issues: write - steps: - - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 - with: - persist-credentials: false - - name: Open or update failure-tracking issue - uses: ./.github/actions/schedule-failure-issue - with: - github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }} diff --git a/.github/workflows/schedule-failure-alert-drill.yml b/.github/workflows/schedule-failure-alert-drill.yml index 17cf13058..97363970e 100644 --- a/.github/workflows/schedule-failure-alert-drill.yml +++ b/.github/workflows/schedule-failure-alert-drill.yml @@ -16,8 +16,8 @@ # # Forces a job failure and then runs .github/actions/schedule-failure-issue # through the exact `needs` + `always() && contains(needs.*.result, -# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz, -# performance-ab). Dispatch it twice to verify both the issue-creation and +# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz). +# Dispatch it twice to verify both the issue-creation and # the dedupe-comment paths, then close the resulting # "[scheduled-failure] Schedule Failure Alert Drill" issue in rustfs/backlog # (the default target repository). Requires the BACKLOG_ISSUE_TOKEN secret diff --git a/.github/workflows/scheduled-validation-watchdog.yml b/.github/workflows/scheduled-validation-watchdog.yml index 9235dcc18..20127e564 100644 --- a/.github/workflows/scheduled-validation-watchdog.yml +++ b/.github/workflows/scheduled-validation-watchdog.yml @@ -28,7 +28,6 @@ on: - "mint" - "minio-interop" - "Nightly GNU Build" - - "Performance A/B" - "Runner Hygiene" types: [completed] diff --git a/docs/operations/hotpath-warp-ab-runbook.md b/docs/operations/hotpath-warp-ab-runbook.md index 4dbf708f3..0f582ea66 100644 --- a/docs/operations/hotpath-warp-ab-runbook.md +++ b/docs/operations/hotpath-warp-ab-runbook.md @@ -1,7 +1,7 @@ # Hotpath warp runbook (A/B gate and ABBA evidence) -**Use this when:** you need S3-face (PutObject / GetObject / mixed) performance evidence for a code change: the nightly relative-budget gate, a quick local A/B, or a formal ABBA run whose numbers will be quoted in a PR. -**Source of truth:** `scripts/run_hotpath_warp_ab.sh` (quick A/B rig), `scripts/run_hotpath_warp_abba.sh` (ABBA runner; also what CI executes), `scripts/hotpath_warp_ab_gate.sh` (relative-budget gate), `scripts/run_object_batch_bench_enhanced.sh` (warp driver, medians, `baseline_compare.csv`), `.github/workflows/performance-ab.yml` (CI gate). +**Use this when:** you need S3-face (PutObject / GetObject / mixed) performance evidence for a code change: a quick local A/B, or a formal ABBA run whose numbers will be quoted in a PR. +**Source of truth:** `scripts/run_hotpath_warp_ab.sh` (quick A/B rig), `scripts/run_hotpath_warp_abba.sh` (ABBA runner), `scripts/hotpath_warp_ab_gate.sh` (relative-budget gate), `scripts/run_object_batch_bench_enhanced.sh` (warp driver, medians, `baseline_compare.csv`). This runbook compares two binaries. To sweep one `RUSTFS_*` runtime knob at a time against a fixed binary, use [object-io-tuning-ab-matrix.md](object-io-tuning-ab-matrix.md) instead. @@ -10,7 +10,7 @@ Two entry points share one workload matrix, one gate, and one deploy-hook shape: | Entry | Script | Legs per cell | Use | | --- | --- | --- | --- | | Quick A/B | `scripts/run_hotpath_warp_ab.sh` | baseline → candidate | local smoke, fast triage of a suspected regression | -| ABBA | `scripts/run_hotpath_warp_abba.sh` | A1 baseline → B1 candidate → B2 candidate → A2 baseline | formal evidence; the CI gate | +| ABBA | `scripts/run_hotpath_warp_abba.sh` | A1 baseline → B1 candidate → B2 candidate → A2 baseline | formal evidence | This rig is the only entry point for S3-face performance coverage. There is deliberately no in-process criterion benchmark for those operations: a criterion harness that stands up an embedded server measures the harness, not the S3 path. Micro-benchmarks stay at function level (EC encode, `xl.meta` parse, `rename_data`). Knob-level sweeps against a fixed binary are a different question; see [object-io-tuning-ab-matrix.md](object-io-tuning-ab-matrix.md). @@ -163,16 +163,6 @@ After each window confirm the `.json.gz` profile and its `.syms.json` sidecar ar Instrumented builds (features in `rustfs/Cargo.toml`): `--features hotpath-alloc` for allocation attribution, `--features hotpath-cpu` for CPU hotpath sections. Compare instrumented binaries only with other builds of the same mode; never use them for throughput acceptance, because the instrumentation changes what is measured. -## CI gate (`performance-ab.yml`) +## Performance gate status -| Aspect | Value | -| --- | --- | -| Triggers | `schedule` (nightly cron `31 6 * * *` UTC against `main`) and `workflow_dispatch` (inputs `duration`, default `12s`; `allow_regression`, boolean). No `pull_request` trigger and no label gating. | -| Jobs | `warp-ab` (runner `sm-standard-2`, `timeout-minutes: 180`); `alert-on-failure` (opens the scheduled-failure issue; scheduled runs only). | -| Baseline commit | scheduled: head of the last successful scheduled run (falls back to the candidate itself when there is none); dispatch: `origin/main`. The baseline must be an ancestor of the candidate. | -| Binary cache | `actions/cache` inside `warp-ab`, key `rustfs-baseline-`. Miss: source build (same-commit runs build once and reuse the binary for both phases); a run whose gate passes saves the candidate binary under `rustfs-baseline-` for the next night. | -| Command | `scripts/run_hotpath_warp_abba.sh --duration --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision --candidate-revision --baseline-bin ... --candidate-bin ...` | -| Exemption | `allow_regression=true` on dispatch adds `--allow-regression --exemption-reason "workflow dispatch override"`. | -| Artifacts | `hotpath-warp-ab-` containing `target/hotpath-abba/` (14-day retention); the step summary renders `candidate_gate.md`, or the server-log tails when the rig failed before the gate. The `Enforce gate` step fails the job on a non-zero rig exit. | - -The nightly detects a regression within a day of landing; it does not block a merge. For pre-merge evidence run the ABBA procedure above and attach the outputs to the PR. +There is no scheduled CI performance gate. The ABBA procedure above is the evidence path for performance comparisons; attach its outputs to the PR that claims a performance effect. A scheduled regression gate requires calibrated budgets: on the previous gate's runner, measured drift between identical binaries at the default 12-second rounds exceeded the former 10% fail budget, so the scheduled entry never seeded a baseline (the workflow only ever passed on its pre-removal push trigger). Restore a scheduled gate only after a noise study grounds the budgets on the target runner. diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 263a5a1c8..d10e6d87e 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -41,7 +41,7 @@ The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`, three A1/B1/B2/A2 groups for both baseline/candidate with background work on, and candidate-only background off/on. A measured release leg lasts at least 7200 seconds; the minimum matrix contains 120 legs (240 hours before setup/oracles). -The existing `performance-ab.yml` supplies the pattern for immutable build +The hotpath warp ABBA rig (`scripts/run_hotpath_warp_abba.sh`) supplies the pattern for immutable build provenance and failure propagation, but its short Warp workload is not this scanner gate. No scheduled workflow starts this matrix automatically. diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 657cf662a..c337d9050 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -99,7 +99,6 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow | | `on-demand-migration-interop.yml` (nightly) | `minio-source`, `cloud-source` (`aws`, `r2`, `gcs`) | report-only provider interop; one JSON report per provider naming cases, timings and source request counts, plus JUnit and MinIO logs. A cloud provider whose `ODM_INTEROP_*` secrets are absent is skipped with a summary note, not failed | no | start the pinned MinIO container as in the job, export the `RUSTFS_ODM_INTEROP_*` variables, run `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test` | -| `performance-ab.yml` (nightly) | `warp-ab` | regression-budget gate; A/B summaries and server logs | yes | `bash scripts/run_hotpath_warp_abba.sh --help` | | `nightly-gnu.yml` (nightly) | `build`, `kms-vault-lane`, `kms-vault-ha-failover` | build, live Vault, and HA failover gates | yes | commands and pinned Vault images in the workflow | | `audit.yml` (nightly) | `cargo-deny`, `workflow-pin-report` | dependency and workflow-pin gates | yes | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow | @@ -111,7 +110,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched | `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/-` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | no | dispatch, optionally with `source_ref` | | `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on missing or stale attempts or completed successes | n/a | dispatch | -Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. A manual performance run may explicitly allow a known regression; that override is not a passing baseline. +Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. ## Packaged functional acceptance diff --git a/scripts/README.md b/scripts/README.md index e2001729f..06f515a0f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -100,7 +100,7 @@ their issue closes. | Entry | Status | Purpose | Wiring / docs | |---|---|---|---| | `run_hotpath_warp_ab.sh` | dev-tool | Linux warp A/B rig for the hotpath series (quick local A/B) | `docs/operations/hotpath-warp-ab-runbook.md` | -| `run_hotpath_warp_abba.sh` | ci-gate | Formal ABBA warp runner (baseline/candidate interleaved, `--rounds >= 3`); the runner performance-ab.yml executes | performance-ab.yml (scheduled); `docs/operations/hotpath-warp-ab-runbook.md` | +| `run_hotpath_warp_abba.sh` | dev-tool | Formal ABBA warp runner (baseline/candidate interleaved, `--rounds >= 3`) | `docs/operations/hotpath-warp-ab-runbook.md` | | `hotpath_warp_ab_gate.sh` | dev-tool | Relative-budget gate evaluated over the warp A/B results | used by `run_hotpath_warp_ab.sh`; hotpath runbook | | `run_internode_grpc_ab_bench.sh` | dev-tool | One-click A/B driver for the internode gRPC optimization stages | `docs/operations/internode-grpc-benchmark-runbook.md` | | `run_internode_transport_baseline.sh` | dev-tool | Internode transport baseline runner | internode runbook; `crates/io-metrics/README.md` | diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 65899459c..54d9e34b0 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -520,7 +520,6 @@ SCHEDULED_ALERT_CHECKOUT_OVERRIDES = { ".github/workflows/ci.yml": "actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a", ".github/workflows/fuzz.yml": "actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a", ".github/workflows/nightly-gnu.yml": "actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a", - ".github/workflows/performance-ab.yml": "actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a", ".github/workflows/runner-hygiene.yml": "actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a", ".github/workflows/scheduled-validation-watchdog.yml": "actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a", ".github/workflows/scheduled-validation-freshness.yml": "actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a", @@ -5953,7 +5952,6 @@ class SelfTests(unittest.TestCase): ".github/workflows/ci.yml", ".github/workflows/fuzz.yml", ".github/workflows/nightly-gnu.yml", - ".github/workflows/performance-ab.yml", ".github/workflows/runner-hygiene.yml", ) with tempfile.TemporaryDirectory() as tmp: diff --git a/scripts/security/check_performance_ab_workflow.sh b/scripts/security/check_performance_ab_workflow.sh deleted file mode 100755 index bb77e9f91..000000000 --- a/scripts/security/check_performance_ab_workflow.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -workflow=".github/workflows/performance-ab.yml" - -require_absent_pattern() { - local pattern="$1" - local description="$2" - - if grep -Eq -- "$pattern" "$workflow"; then - echo "invalid performance A/B workflow contract: $description" >&2 - exit 1 - fi -} - -require_present_pattern() { - local pattern="$1" - local description="$2" - - if ! grep -Eq -- "$pattern" "$workflow"; then - echo "invalid performance A/B workflow contract: $description" >&2 - exit 1 - fi -} - -require_absent_pattern '(^|[^[:alnum:]_])pull_request(_target)?([^[:alnum:]_]|$)' "the workflow must not contain PR event handling" -require_absent_pattern 'pull-requests[[:space:]]*:[[:space:]]*write' "the workflow must not receive PR write permission" -require_absent_pattern 'permissions[[:space:]]*:[[:space:]]*write-all' "the workflow must not receive broad write permission" -require_absent_pattern '^[[:space:]]*push:' "the workflow must not spend a release build on every main push" -require_present_pattern 'listWorkflowRuns' "the scheduled baseline must come from workflow history" -require_present_pattern 'status:[[:space:]]*"success"' "the scheduled baseline must be a successful run" -require_present_pattern 'SCHEDULED_BASELINE_SHA' "the resolved scheduled baseline must reach the comparison" -require_present_pattern "SCHEDULED_BASELINE_SHA:-\\\$candidate_sha" "the first scheduled run must seed from its verified candidate" -require_present_pattern 'git merge-base --is-ancestor' "every baseline must stay on candidate history" -require_present_pattern 'update the selected ref before comparing' "stale manual refs must fail before the benchmark" -require_present_pattern 'setup failed before the rig ran' "an earlier setup failure must not become an empty exit status" -require_present_pattern 'Cache successful candidate baseline' "a successful candidate must become the next cached baseline" -if ! sed -n '/^ warp-ab:/,/^ alert-on-failure:/p' "$workflow" | grep -Eq '^ timeout-minutes:[[:space:]]*180([[:space:]]|$)'; then - echo "invalid performance A/B workflow contract: the cold-cache path must fit both builds, the A/B run, and evidence publication" >&2 - exit 1 -fi - -echo "Performance A/B workflow contract ok."