# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4).
#
# Two entry points, honestly scoped:
#   * schedule (nightly, on main): post-merge detection — catches a regression
#     within 24h of landing, not before merge.
#   * pull_request labeled `perf-ab`: opt-in pre-merge gate for a specific PR.
# The `perf-deliberate-tradeoff` label runs the gate with --allow-regression so
# a deliberate correctness cost (e.g. the #4221 fsync durability fix) is
# recorded but does not block (rustfs/backlog#935 correction 1).

name: Performance A/B

on:
  schedule:
    - cron: "0 6 * * *" # 06:00 UTC nightly, against main
  workflow_dispatch:
    inputs:
      duration:
        description: "warp duration per round (short by default to fit the double-build budget)"
        required: false
        default: "12s"
        type: string
      allow_regression:
        description: "Pass the gate despite a FAIL (deliberate tradeoff)"
        required: false
        default: false
        type: boolean
  pull_request:
    types: [labeled, synchronize, reopened]
  push:
    # Every main commit pre-builds and caches its release binary (perf-3) so the
    # nightly A/B restores a ready baseline instead of paying the double build.
    branches: [main]

permissions:
  contents: read
  pull-requests: write

# Per-PR: a new push cancels the previous (up to 90-minute) A/B run instead of
# stacking them. Nightly schedule and manual dispatch get a unique group and
# always run to completion.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  # perf-3: on every push to main, build the release binary once and cache it
  # keyed by commit SHA (rustfs-baseline-<sha>). The nightly A/B (and, later, the
  # perf-7 PR gate) restore this instead of paying the ~32min-per-side source
  # build. That double build is what pushed the expanded 24-cell nightly past its
  # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
  # builds off the shared cargo cache keep each push cheap, and building on the
  # same sm-standard-2 runner the A/B measures on guarantees the cached binary is
  # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
  # are cancelled ~7/8 of the time and are not a reliable baseline.
  build-baseline-cache:
    name: Build + cache baseline binary
    if: github.event_name == 'push'
    runs-on: sm-standard-2
    # Latest-wins: consumers only ever restore the binary for the *current*
    # origin/main tip, so when pushes land faster than the ~65min build, a
    # superseded build's output is dead weight — cancel it instead of stacking
    # hour-long jobs on the shared runner pool. A skipped intermediate SHA at
    # most costs one same-commit self-heal in the A/B job.
    concurrency:
      group: perf-baseline-build-main
      cancel-in-progress: true
    # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
    # single release build past 60min on this runner — every cache build on
    # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
    # time of 1h0m0s") and the cache never populated. The measured binary must
    # keep the production profile, so the budget absorbs the build instead.
    timeout-minutes: 100
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Build release rustfs
        run: cargo build --release --bin rustfs

      - name: Stage binary for cache
        run: |
          set -euo pipefail
          mkdir -p baseline-bin
          cp target/release/rustfs baseline-bin/rustfs

      - name: Cache baseline binary by SHA
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: baseline-bin/rustfs
          key: rustfs-baseline-${{ github.sha }}

  warp-ab:
    name: Warp A/B budget gate
    # Always run on schedule / manual dispatch. Opt-in on PRs: only when the
    # `perf-ab` label is present, and for `labeled` events only when the label
    # being added is `perf-ab` itself (adding an unrelated label to an opted-in
    # PR must not re-run the gate). Never on push — that event only feeds
    # build-baseline-cache above.
    if: >-
      github.event_name == 'schedule' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' &&
       contains(github.event.pull_request.labels.*.name, 'perf-ab') &&
       (github.event.action != 'labeled' || github.event.label.name == 'perf-ab'))
    runs-on: sm-standard-2
    # With perf-3's cached baseline binary the common (cache-hit) nightly is
    # measurement-only and finishes well under 50min. This ceiling stays
    # generous only to absorb the same-commit cache-miss self-heal (~65min
    # single build with the post-#4806 LTO profile + measurement). A timeout
    # surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
    # not just failure). perf-6 recalibrates the budget once the noise study
    # lands.
    timeout-minutes: 120
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          fetch-depth: 0 # baseline is built from origin/main

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Install warp
        run: |
          set -euo pipefail
          WARP_VERSION="1.0.0"
          curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \
            | sudo tar -xz -C /usr/local/bin warp
          warp --version

      - name: Decide exemption
        id: exempt
        run: |
          allow="false"
          if [[ "${{ github.event_name }}" == "pull_request" ]] \
            && ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then
            allow="true"
          fi
          if [[ "${{ github.event.inputs.allow_regression }}" == "true" ]]; then
            allow="true"
          fi
          echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"

      # perf-3: resolve the commits so the cache can be keyed by SHA. The
      # baseline is origin/main; the candidate is the checked-out ref. On the
      # nightly (checkout == main) they are the same commit, so one cached binary
      # serves both phases and the run does zero source builds.
      - name: Resolve baseline / candidate commits
        id: commits
        run: |
          set -euo pipefail
          baseline_sha="$(git rev-parse origin/main)"
          candidate_sha="$(git rev-parse HEAD)"
          echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
          echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
          echo "baseline commit: $baseline_sha"
          echo "candidate commit: $candidate_sha"

      # Exact-key restore of the baseline binary built by build-baseline-cache
      # when origin/main last landed. A miss (binary evicted or not built yet)
      # leaves cache-hit unset and the rig falls back to a source build.
      - name: Restore cached baseline binary
        id: baseline_cache
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: baseline-bin/rustfs
          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}

      # Self-heal: on a nightly/dispatch run where the candidate commit IS the
      # baseline commit, a cache miss would make the rig build the same commit
      # twice (~65min per side with the post-#4806 LTO profile — no job budget
      # fits that). Build it once here, reuse it for both phases, and save it
      # back to the cache so the next run hits.
      - name: Build baseline on cache miss (same-commit self-heal)
        id: selfheal
        if: >-
          steps.baseline_cache.outputs.cache-hit != 'true' &&
          steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
        run: |
          set -euo pipefail
          cargo build --release --bin rustfs
          mkdir -p baseline-bin
          cp target/release/rustfs baseline-bin/rustfs
          echo "built=true" >> "$GITHUB_OUTPUT"

      - name: Save self-healed baseline to cache
        if: steps.selfheal.outputs.built == 'true'
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
        with:
          path: baseline-bin/rustfs
          key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}

      - name: Run warp A/B and gate
        id: ab
        run: |
          set -euo pipefail
          # Budget note: with perf-3's cached baseline the nightly does no source
          # build on a cache hit, so the wall-clock is dominated by the short warp
          # matrix — duration/rounds/cooldown are kept small to fit all 24 cells
          # (6 workloads x 2 phases x 2 drive-sync) rather than dropping cells.
          # --health-timeout 180 outlasts the server's own 120s startup-readiness
          # budget, which the rig's previous 60s health poll undershot (the first
          # two nightly failures). perf-6 recalibrates these once the noise study
          # lands.
          duration="${{ github.event.inputs.duration || '12s' }}"
          baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
          candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
          baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
          selfheal_built="${{ steps.selfheal.outputs.built }}"

          args=(--duration "$duration" --rounds 2 --cooldown 5 --health-timeout 180)

          if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" ]]; then
            chmod +x baseline-bin/rustfs
            base_bin="$PWD/baseline-bin/rustfs"
            args+=(--baseline-bin "$base_bin")
            if [[ "$baseline_hit" == "true" ]]; then
              base_src="actions-cache (rustfs-baseline-$baseline_sha)"
            else
              base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
            fi
            if [[ "$candidate_sha" == "$baseline_sha" ]]; then
              # Nightly on main: the candidate is the same commit as the baseline,
              # so reuse the one binary for both phases and skip all builds.
              args+=(--candidate-bin "$base_bin" --skip-build)
              cand_src="same binary as baseline (same commit)"
            else
              cand_src="source build of the checked-out ref"
            fi
          else
            # Cache miss with candidate != baseline (opt-in PR gate only): fall
            # back to the source double-build. With the post-#4806 LTO profile
            # this will overrun the job budget and alert; rerun once the push
            # cache build for origin/main has completed, or wait for perf-7's
            # merge-base caching.
            args+=(--baseline-ref origin/main)
            base_src="source build of origin/main (cache miss)"
            cand_src="source build of the checked-out ref"
          fi

          args+=(--provenance-note "baseline commit: $baseline_sha - $base_src")
          args+=(--provenance-note "candidate commit: $candidate_sha - $cand_src")

          if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
            args+=(--allow-regression --exemption-reason "labeled perf-deliberate-tradeoff / dispatch override")
          fi
          # Do not let a gate FAIL abort the job here; capture status and surface
          # it after the PR comment is posted.
          set +e
          bash scripts/run_hotpath_warp_ab.sh "${args[@]}"
          echo "status=$?" >> "$GITHUB_OUTPUT"
          set -e
          # Locate the newest run dir + gate.md for the summary/comment/artifact
          # steps. On a startup failure there is no gate.md, but the run dir still
          # holds server-logs/ for diagnosis.
          # Run dirs are UTC-timestamp names (no special chars); ls is safe here.
          # shellcheck disable=SC2012
          run_dir="$(ls -td target/hotpath-ab/*/ 2>/dev/null | head -n1 || true)"
          echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
          # shellcheck disable=SC2012
          gate_md="$(ls -t target/hotpath-ab/*/gate.md 2>/dev/null | head -n1 || true)"
          echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"

      - name: Upload A/B results
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: hotpath-warp-ab-${{ github.run_number }}
          # Includes per-cell median_summary.csv / baseline_compare.csv, gate.md,
          # and server-logs/ (rustfs.log + startup env per phase) so a failed run
          # is diagnosable. Short retention: this is churny nightly debug data.
          path: target/hotpath-ab/
          if-no-files-found: warn
          retention-days: 14

      - name: Write gate summary
        if: always()
        run: |
          set -euo pipefail
          status="${{ steps.ab.outputs.status }}"
          gate_md="${{ steps.ab.outputs.gate_md }}"
          run_dir="${{ steps.ab.outputs.run_dir }}"
          {
            echo "## Hotpath warp A/B — run ${{ github.run_number }}"
            echo
            if [[ "$status" == "0" ]]; then
              echo "Rig/gate exit: \`0\` (pass or warn)."
            else
              echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**."
            fi
            echo
            if [[ -n "$gate_md" && -f "$gate_md" ]]; then
              cat "$gate_md"
            else
              echo "No \`gate.md\` produced — the rig failed **before** the gate"
              echo "(most likely server startup / health). Failing phase(s) below;"
              echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact."
              if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then
                for f in "$run_dir"/server-logs/*.log; do
                  [[ -f "$f" ]] || continue
                  echo
                  echo "<details><summary>$(basename "$f")</summary>"
                  echo
                  echo '```'
                  tail -n 30 "$f"
                  echo '```'
                  echo "</details>"
                done
              fi
            fi
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Comment gate result on PR
        if: always() && github.event_name == 'pull_request' && steps.ab.outputs.gate_md != ''
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          gh pr comment "${{ github.event.pull_request.number }}" --body-file "${{ steps.ab.outputs.gate_md }}"

      # Scheduled failure alerting is handled by the alert-on-failure job below
      # (perf-2 consuming ci-8's schedule-failure-issue composite action).

      - name: Enforce gate
        if: always()
        run: |
          status="${{ steps.ab.outputs.status }}"
          if [[ "$status" != "0" ]]; then
            echo "::error::warp A/B budget gate failed (exit $status). See the step summary / PR comment / gate.md artifact." >&2
            exit "$status"
          fi
          echo "warp A/B budget gate passed."

  alert-on-failure:
    name: Alert on scheduled failure
    needs: [warp-ab]
    # `always()` is required: without it this job is skipped when a needed
    # job fails. Alerts only for scheduled (nightly) runs (backlog#1149
    # ci-8); PR and manual dispatch failures are already watched by a human.
    # `cancelled` is included alongside `failure` on purpose: a job that hits
    # timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
    # timeouts went silent precisely because the guard was failure-only. The
    # composite action already reports cancelled/timed-out jobs in the issue
    # body. (Scheduled runs get a unique concurrency group with
    # cancel-in-progress off, so a cancellation here means a timeout/manual
    # abort, never a superseding run.)
    if: >-
      always() && github.event_name == 'schedule' &&
      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
      - name: Open or update failure-tracking issue
        uses: ./.github/actions/schedule-failure-issue
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}