rustfs/.github/workflows/ci.yml

# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Continuous Integration

on:
  push:
    branches: [ main ]
    paths-ignore:
      - "**.md"
      - "docs/**"
      - "deploy/**"
      - "scripts/dev_*.sh"
      - "scripts/probe.sh"
      - "LICENSE*"
      - ".gitignore"
      - ".dockerignore"
      - "README*"
      - "**/*.png"
      - "**/*.jpg"
      - "**/*.svg"
      - ".github/workflows/build.yml"
      - ".github/workflows/docker.yml"
      - ".github/workflows/audit.yml"
      - "flake.lock"
  pull_request:
    types: [ opened, synchronize, reopened, closed ]
    # gateway/integration carries main's ruleset (rustfs/backlog#2734), so its
    # pull requests must produce the same required checks.
    branches: [ main, gateway/integration ]
  merge_group:
    types: [ checks_requested ]
  schedule:
    - cron: "11 0 * * *" # Nightly 00:11 UTC, after the 22:41 UTC cache-warm run
  workflow_dispatch:
    inputs:
      rehearse_workspace_shards:
        description: Compare one full execution with at most two resource-safe shards of the same archive
        type: boolean
        default: false

permissions:
  contents: read

# Concurrency groups are scoped per event so different triggers never cancel
# each other. PR pushes cancel superseded attempts; main and scheduled runs
# finish so a busy merge stream cannot starve the complete baseline. For main,
# GitHub retains one running run and replaces the pending run with the latest.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:

  # Compile-free lanes, and lanes that only run a prebuilt binary, use
  # GitHub-hosted runners: they are free for this public repository, and the
  # self-hosted pool is reserved for the lanes that need its CPU count and
  # disk (the workspace test build leaves a ~64GB target/, hosted runners
  # have 14GB). Placement and billing rules: README.md in this directory.
  cancel-closed-pr-runs:
    name: Cancel Closed PR Runs
    if: github.event_name == 'pull_request' && github.event.action == 'closed'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Explain cancellation run
        run: echo "PR closed; this run only cancels older runs in the same concurrency group."

  classify-changes:
    name: Select CI scope
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      mode: ${{ steps.scope.outputs.mode }}
      plan: ${{ steps.scope.outputs.plan }}
      connect: ${{ steps.scope.outputs.connect || 'true' }}
      packages: ${{ steps.scope.outputs.packages }}
      full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
      rehearse_shards: ${{ steps.scope.outputs.rehearse_shards || 'false' }}
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          fetch-depth: 2
          persist-credentials: false
      - name: Select scope using the base revision's policy
        id: scope
        env:
          CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
          CI_EVENT_ACTION: ${{ github.event.action }}
          CI_REHEARSE_SHARDS: ${{ inputs.rehearse_workspace_shards && 'true' || 'false' }}
        run: |
          if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
            python3 -I scripts/ci_gate.py select
          elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
            python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
          else
            printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
            echo "Base CI policy unavailable; running the full matrix."
          fi
          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
            printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
            echo "Base policy does not cover this workflow contract; retaining the full matrix."
          fi
      - name: Retain verification plan
        if: steps.scope.outputs.plan != ''
        env:
          CI_PLAN: ${{ steps.scope.outputs.plan }}
        run: |
          mkdir -p artifacts/ci-plan
          printf '%s\n' "$CI_PLAN" > artifacts/ci-plan/plan.json
      - name: Upload verification plan
        if: steps.scope.outputs.plan != ''
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: verification-plan-${{ github.run_attempt }}
          path: artifacts/ci-plan/plan.json
          if-no-files-found: error
          retention-days: 7

  typos:
    name: Typos
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - name: Typos check with custom config file
        uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2

  # Fail early with compile-free checks for every pull request.
  quick-checks:
    name: Quick Checks
    if: github.event_name != 'pull_request' || github.event.action != 'closed'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Run shared quick checks
        uses: ./.github/actions/quick-checks

  test-and-lint:
    name: Workspace Lint and Build
    if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
    needs: [ quick-checks, classify-changes ]
    runs-on: sm-standard-4
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          # Checkout otherwise writes the token into .git/config, where a PR's
          # own build.rs or proc-macro could read it back out.
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          # Every lane reads its cache; cache-warm.yml remains the sole writer
          # with the matching feature closure. Tests never compete to replace
          # shared caches, including when a PR attempt is cancelled.
          cache-shared-key: ci-dev
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      - name: Protect Connect test home
        run: chmod go-w "$(realpath "$HOME")"

      - name: Prepare test evidence
        run: |
          mkdir -p artifacts/test-and-lint
          {
            echo "run_id=${GITHUB_RUN_ID}"
            echo "job=${GITHUB_JOB}"
            echo "runner=${RUNNER_NAME}"
            echo "started_at=$(date --utc --iso-8601=seconds)"
          } > artifacts/test-and-lint/run-metadata.txt

      # Clippy gates every event, pull requests included. When it ran only after
      # merge, a lint regression merged freely and then failed every main run
      # until someone fixed it (10 of 12 main runs on 2026-10-17/18). cache-warm
      # writes the same invocation's check artifacts into ci-dev, so a pull
      # request re-checks only the crates its diff touches. Main keeps the step
      # because pull requests can merge without being up to date with main.
      - name: Run clippy lints
        run: |
          ./scripts/ci/resource_sampler.sh start clippy
          trap './scripts/ci/resource_sampler.sh stop' EXIT
          python3 scripts/ci_test_artifact.py cargo lint --configuration workspace

      - name: Archive workspace test binaries
        env:
          CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
        run: |
          ./scripts/ci/resource_sampler.sh start workspace-archive
          trap './scripts/ci/resource_sampler.sh stop' EXIT
          python3 scripts/ci_test_artifact.py build --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"

      - name: Upload workspace test binaries
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: workspace-test-binaries
          overwrite: true
          path: ${{ runner.temp }}/workspace-tests.tar.zst*
          compression-level: 0
          if-no-files-found: error
          retention-days: 1

      - name: Run documentation tests
        run: |
          mkdir -p artifacts/test-and-lint
          set +e
          timeout --verbose --signal=TERM --kill-after=30s 15m \
            python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
            2>&1 | tee artifacts/test-and-lint/doctest.log
          status=${PIPESTATUS[0]}
          {
            echo "command=cargo test --all --exclude e2e_test --doc"
            echo "exit_status=${status}"
            echo "finished_at=$(date --utc --iso-8601=seconds)"
            echo
            echo "Remaining test-related processes:"
            pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
          } > artifacts/test-and-lint/doctest-diagnostics.txt
          exit "${status}"

      - name: Upload test reports and diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            artifacts/test-and-lint
          retention-days: 3
          if-no-files-found: error

      # rustfs/backlog#1289: fail if a seed rule's log anchor no longer exists
      # verbatim in the source tree (log message drifted without updating the
      # rule). Placed here where the workspace — including the la-dump-anchors
      # bin — is already built by the clippy/test steps above.
      - name: Check log-analyzer rule anchors
        run: ./scripts/check_log_analyzer_rules.sh

  workspace-tests:
    name: Workspace tests
    needs: [ test-and-lint, classify-changes ]
    # The archive retains compile-time fixture paths and the large test set
    # requires the same runner image until the hosted-runtime pilot passes.
    runs-on: sm-standard-4
    timeout-minutes: 90
    outputs:
      workspace_shards: ${{ steps.shards.outputs.workspace_shards || '[]' }}
    env:
      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - uses: ./.github/actions/test-runtime
      - name: Protect Connect test home
        run: chmod go-w "$(realpath "$HOME")"
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: workspace-test-binaries
          path: ${{ runner.temp }}
      - name: Verify workspace archive identity
        run: python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"

      - name: Extract and list workspace tests
        run: |
          mkdir -p artifacts/test-and-lint
          cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
            > artifacts/test-and-lint/core-test-listing.json
          python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json

      - name: Run nextest tests
        run: |
          mkdir -p artifacts/test-and-lint
          rm -f target/nextest/ci/junit.xml
          ./scripts/ci/resource_sampler.sh start nextest
          trap './scripts/ci/resource_sampler.sh stop' EXIT
          set +e
          NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
            cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
            --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
            --status-level all --final-status-level all \
            2>&1 | tee artifacts/test-and-lint/nextest.log
          status=${PIPESTATUS[0]}
          {
            echo "command=workspace archive execution"
            echo "exit_status=${status}"
            echo "finished_at=$(date --utc --iso-8601=seconds)"
            echo
            echo "Remaining test-related processes:"
            pgrep -af 'cargo|nextest|target/.*/deps/' || true
            echo
            echo "Kernel OOM / kill events:"
            dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
          } > artifacts/test-and-lint/nextest-diagnostics.txt
          exit "${status}"

      - name: Verify workspace execution completeness
        run: |
          python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
          ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml

      - name: Run ignored ILM integration tests serially
        if: needs.classify-changes.outputs.mode == 'full'
        run: |
          mkdir -p artifacts/ilm-integration
          set +e
          NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 80m \
            cargo nextest run --profile default -j1 --run-ignored ignored-only \
            --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
            --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
            -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))' \
            --status-level all --final-status-level all \
            2>&1 | tee artifacts/ilm-integration/nextest.log
          status=${PIPESTATUS[0]}
          {
            echo "exit_status=${status}"
            echo "finished_at=$(date --utc --iso-8601=seconds)"
            echo
            echo "Remaining test-related processes:"
            pgrep -af 'cargo|nextest|target/.*/deps/' || true
          } > artifacts/ilm-integration/diagnostics.txt
          exit "${status}"

      - name: Upload workspace test evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: workspace-test-evidence-${{ github.run_attempt }}
          path: |
            target/nextest/ci/junit.xml
            artifacts/test-and-lint
            artifacts/ilm-integration
          retention-days: 3
          if-no-files-found: error

      - name: Prepare bounded shards from this candidate's observed durations
        id: shards
        if: needs.classify-changes.outputs.rehearse_shards == 'true'
        run: |
          python3 scripts/ci_test_shards.py prepare \
            --listing artifacts/test-and-lint/core-test-listing.json \
            --junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
      - name: Upload same-run shard plan
        if: needs.classify-changes.outputs.rehearse_shards == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: workspace-shard-plan-${{ github.run_attempt }}
          path: artifacts/workspace-shard-plan
          retention-days: 3
          if-no-files-found: error

  workspace-shards:
    name: Workspace shards (${{ matrix.shard }})
    if: needs.classify-changes.outputs.rehearse_shards == 'true'
    needs: [classify-changes, workspace-tests]
    # Reuse the producer image and checkout path for the full RustFS archive.
    runs-on: sm-standard-4
    timeout-minutes: 90
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        shard: ${{ fromJSON(needs.workspace-tests.outputs.workspace_shards || '[]') }}
    env:
      CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - uses: ./.github/actions/test-runtime
      - name: Protect Connect test home
        run: chmod go-w "$(realpath "$HOME")"
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: workspace-test-binaries-${{ github.run_attempt }}
          path: ${{ runner.temp }}
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: workspace-shard-plan-${{ github.run_attempt }}
          path: artifacts/workspace-shard-plan
      - name: Verify and execute this shard without compilation
        env:
          SHARD_INDEX: ${{ matrix.shard }}
        run: |
          python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
          python3 scripts/ci_test_shards.py execute --plan artifacts/workspace-shard-plan \
            --archive "$RUNNER_TEMP/workspace-tests.tar.zst" --index "$SHARD_INDEX" --output artifacts/workspace-shard-result
      - name: Retain shard evidence including failures
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: workspace-shard-result-${{ matrix.shard }}-${{ github.run_attempt }}
          path: artifacts/workspace-shard-result
          retention-days: 3
          if-no-files-found: error

  workspace-shard-evidence:
    name: Workspace shard completeness
    if: always() && needs.classify-changes.outputs.rehearse_shards == 'true'
    needs: [classify-changes, workspace-tests, workspace-shards]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: workspace-shard-plan-${{ github.run_attempt }}
          path: artifacts/workspace-shard-plan
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: workspace-shard-result-*-${{ github.run_attempt }}
          path: artifacts/workspace-shards
      - name: Require the exact union of successful shard executions
        run: |
          python3 scripts/ci_test_shards.py verify --plan artifacts/workspace-shard-plan --evidence artifacts/workspace-shards
          ./scripts/check_migration_gate_count.sh evidence artifacts/workspace-shard-plan/listing.json artifacts/workspace-shards/combined.xml

  connect-feature-boundaries:
    name: Connect Feature Boundaries
    if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
    needs: [ quick-checks, classify-changes ]
    runs-on: sm-standard-4
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-dev
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'
          install-test-tools: 'false'

      - name: Protect Connect test home
        run: chmod go-w "$(realpath "$HOME")"

      - name: Run short credential behavior tests
        env:
          CARGO_BUILD_JOBS: "2"
        run: |
          cargo test --locked -p rustfs --test connect_registration \
            --features connect-e2e-short-credentials -- --exact --test-threads=1 \
            registration_enforces_build_profile_credential_lifetime \
            rotation_waits_for_threshold_and_stops_on_revocation

      # The guard in rustfs/src/connect/registration.rs is cfg(not(debug_assertions)),
      # and it must be tripped by a profile that turns debug assertions off for
      # the whole graph. Handing `-C debug-assertions=off` to the rustfs lib alone
      # through `cargo rustc` leaves its dependencies on the dev-profile cfg, so
      # the lib also fails on items gated on one side only (E0599 on
      # Filenames::Embedded) and "fails solely at the compile_error" can never
      # hold. `cargo check --release` is metadata-only and needs no
      # link-concurrency limit; it is cold because ci-dev holds no release
      # artifacts.
      - name: Reject short credentials in release builds
        run: python3 scripts/check_connect_release_boundary.py

      - name: Check offline enrollment E2E root boundary
        run: ./scripts/check_offline_enrollment_e2e.sh

  test-and-lint-rio-v2:
    name: Test and Lint (rio-v2)
    # Alternate feature configurations run on the nightly schedule, merge
    # groups and manual dispatch. The default workspace remains the PR gate and
    # the per-push main baseline: repeating four feature builds on every merge
    # bought nothing the nightly run does not report within a day.
    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
    needs: [ quick-checks, classify-changes ]
    runs-on: sm-standard-4
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-feat-rio
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      - name: Protect Connect test home
        run: chmod go-w "$(realpath "$HOME")"

      - name: Run rio-v2 clippy lints
        run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings

      - name: Run rio-v2 feature tests
        shell: bash
        env:
          # Match the main nextest lane's #5394 link-I/O guard. A cold feature
          # cache otherwise fans out enough rust-lld processes to exhaust this
          # job's 90-minute budget before any test starts.
          CARGO_BUILD_JOBS: "2"
        run: |
          # --profile ci so the quarantine list (and its junit flaky markers)
          # covers this leg too; the default profile is the local no-retry
          # profile and silently ignored quarantined flakes here (rustfs#6703).
          mkdir -p artifacts/test-and-lint
          rm -f target/nextest/ci/junit.xml
          ./scripts/ci/resource_sampler.sh start nextest-rio-v2
          trap './scripts/ci/resource_sampler.sh stop' EXIT
          cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
            2>&1 | tee artifacts/test-and-lint/nextest.log
          cargo test -p rustfs --doc --features rio-v2

      - name: Upload rio-v2 test reports and diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            target/nextest/ci/junit.xml
            artifacts/test-and-lint
          retention-days: 3
          if-no-files-found: error

  test-and-lint-protocols:
    name: "Test and Lint (${{ matrix.features.name }})"
    # Same cadence as test-and-lint-rio-v2 above.
    if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
    needs: [ quick-checks, classify-changes ]
    runs-on: sm-standard-4
    timeout-minutes: 90
    strategy:
      # Preserve independent results when one protocol fails, so diagnosis
      # and a focused fix do not require rebuilding an interrupted sibling.
      fail-fast: false
      matrix:
        features:
          - name: swift
            flags: "--features swift"
          - name: sftp
            flags: "--features sftp"
          - name: http3
            flags: "--features http3,swift"
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-feat-proto
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      - name: Prepare protocol test state
        run: |
          rm -f target/nextest/ci/junit.xml
          chmod go-w "$(realpath "$HOME")"

      - name: Run clippy with ${{ matrix.features.name }}
        run: |
          ./scripts/ci/resource_sampler.sh start clippy
          trap './scripts/ci/resource_sampler.sh stop' EXIT
          cargo clippy -p rustfs -p rustfs-protocols --all-targets ${{ matrix.features.flags }} -- -D warnings

      - name: Run tests with ${{ matrix.features.name }}
        id: protocol-tests
        shell: bash
        env:
          # Keep feature-test linking under the same bounded concurrency as the
          # main nextest lane; Clippy is metadata-only and needs no such limit.
          CARGO_BUILD_JOBS: "2"
          SAMPLER_INTERVAL_SECS: "15"
        run: |
          # --profile ci so the quarantine list (and its junit flaky markers)
          # covers this leg too; the default profile is the local no-retry
          # profile and silently ignored quarantined flakes here (rustfs#6703).
          mkdir -p artifacts/protocol-tests
          ./scripts/ci/resource_sampler.sh start nextest
          trap './scripts/ci/resource_sampler.sh stop' EXIT
          cargo nextest run --profile ci -p rustfs -p rustfs-protocols ${{ matrix.features.flags }} \
            2>&1 | tee artifacts/protocol-tests/nextest.log

      - name: Upload protocol test reports and diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: junit-test-and-lint-${{ matrix.features.name }}-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            target/nextest/ci/junit.xml
            artifacts/protocol-tests
            artifacts/test-and-lint
          retention-days: 3
          if-no-files-found: warn

  build-rustfs-debug-binary:
    name: Build RustFS Debug Binary
    if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
    needs: [ quick-checks, classify-changes ]
    runs-on: sm-standard-4
    timeout-minutes: 60
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-dev
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      - name: Build debug binary
        run: |
          python3 - <<'PYBUILD'
          import hashlib
          import json
          import os
          import pathlib
          import subprocess

          def git(*args):
              return subprocess.check_output(["git", *args], text=True).strip()

          def sha256(path):
              digest = hashlib.sha256()
              with pathlib.Path(path).open("rb") as source:
                  for chunk in iter(lambda: source.read(1024 * 1024), b""):
                      digest.update(chunk)
              return digest.hexdigest()

          argv = ["python3", "scripts/e2e_binary.py", "build", "--bins", "--features", "e2e-test-hooks"]
          commit, tree = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^{tree}")
          clean_before = not git("status", "--porcelain", "--untracked-files=normal")
          if not clean_before:
              raise SystemExit("hooks binary requires a clean build checkout")
          lock_sha256 = sha256("Cargo.lock")
          lock_git_blob = git("hash-object", "Cargo.lock")
          rustc = subprocess.check_output(["rustc", "-vV"], text=True)
          host = next(line.removeprefix("host: ") for line in rustc.splitlines() if line.startswith("host: "))
          if os.environ.get("CARGO_BUILD_TARGET") or pathlib.Path(os.environ.get("CARGO_TARGET_DIR", "target")).resolve() != pathlib.Path("target").resolve():
              raise SystemExit("this artifact requires the native target/debug output")
          subprocess.run(argv, check=True)
          clean_after = not git("status", "--porcelain", "--untracked-files=normal")
          if not clean_after or commit != git("rev-parse", "HEAD") or tree != git("rev-parse", "HEAD^{tree}") or lock_sha256 != sha256("Cargo.lock"):
              raise SystemExit("hooks binary source changed while building")
          manifest = {
              "schema": 1, "commit": commit, "tree": tree,
              "clean_before": clean_before, "clean_after": clean_after,
              "lock_sha256": lock_sha256, "lock_git_blob": lock_git_blob,
              "argv": argv, "profile": "debug", "target": host,
              "features": ["e2e-test-hooks"],
              "rustc_verbose": rustc,
              "build_flags": {key: os.environ[key] for key in ("RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_TARGET_DIR", "RUSTUP_TOOLCHAIN") if key in os.environ},
              "binary_sha256": sha256("target/debug/rustfs"),
          }
          pathlib.Path("target/debug/rustfs.e2e-startup-cas-build.json").write_text(json.dumps(manifest, indent=2) + "\n")
          PYBUILD

      - name: Upload debug binary
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: rustfs-debug-binary
          overwrite: true
          path: |
            target/debug/rustfs
            target/debug/rustfs.e2e.json
            target/debug/rustfs.e2e-startup-cas-build.json
          if-no-files-found: error
          retention-days: 1

      - name: Archive E2E test binaries
        env:
          CARGO_BUILD_JOBS: "2"
        run: python3 scripts/ci_test_artifact.py build --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"

      - name: Run E2E documentation tests
        run: cargo test --locked -p e2e_test --doc

      - name: Upload E2E test binaries
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-test-binaries
          overwrite: true
          path: ${{ runner.temp }}/e2e-tests.tar.zst*
          compression-level: 0
          if-no-files-found: error
          retention-days: 1

  build-rustfs-debug-binary-rio-v2:
    name: Build RustFS Debug Binary (rio-v2)
    # Dormant rio-v2 variant (rustfs/backlog#1835): the feature ships in no
    # default build, so this full-suite lane runs only on the nightly schedule
    # and manual dispatch. Per-PR cfg-seam coverage stays with
    # test-and-lint-rio-v2. Lifecycle and the promote-or-delete condition:
    # docs/architecture/minio-file-format-compat.md ("rio-v2 variant lifecycle").
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    needs: [ quick-checks ]
    runs-on: sm-standard-4
    timeout-minutes: 30
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-feat-rio
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      - name: Build debug binary with rio-v2
        run: python3 scripts/e2e_binary.py build --bins --features rio-v2,e2e-test-hooks

      - name: Upload debug binary
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: rustfs-debug-binary-rio-v2
          overwrite: true
          path: |
            target/debug/rustfs
            target/debug/rustfs.e2e.json
          if-no-files-found: error
          retention-days: 1

  uring-integration:
    name: io_uring Integration (real)
    # The pull_request trigger includes `closed` purely so the concurrency
    # group cancels in-flight runs of a closed PR; every other job opts out of
    # that run with this guard (or is skipped through its `needs` chain). This
    # job had neither, so each closed/merged PR really ran the whole io_uring
    # suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
    # 30662728539) and kept the cancellation run in progress for minutes.
    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
    needs: [ quick-checks, classify-changes ]
    # GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
    # a container, applies no seccomp filter that would block io_uring_setup — so
    # the probe succeeds and the tests exercise the real UringBackend/FdCache/
    # latch paths instead of the StdBackend fallback (rustfs/backlog#1179). The
    # self-hosted sm-standard runners cannot guarantee this.
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          # Keeps its own key rather than joining ci-dev. rust-cache's key is
          # built from runner.os/arch plus rustc and lockfile fingerprints — it
          # does NOT include the runner label or image. ubuntu-latest and
          # sm-standard-4 are therefore indistinguishable to it, so sharing a key
          # would let two different system images overwrite each other's
          # artifacts, and would make a 2-core hosted runner unpack ci-dev's ~3GB
          # instead of this lane's ~1.3GB. cache-warm.yml warms this key on
          # ubuntu-latest for the same reason.
          cache-shared-key: ci-uring
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'

      # ext4 supports O_DIRECT; the runner's default TMPDIR may sit on tmpfs or
      # overlayfs, where open(O_DIRECT) returns EINVAL/EOPNOTSUPP and the native
      # read_at_direct path silently latches off to the aligned StdBackend
      # fallback. A dedicated ext4 loopback guarantees the native O_DIRECT read
      # path is actually exercised, not just the fallback (rustfs/backlog#1220).
      - name: Prepare ext4 loopback for O_DIRECT coverage
        run: |
          set -euo pipefail
          dd if=/dev/zero of=/tmp/rustfs-odirect.img bs=1M count=1024
          mkfs.ext4 -q -F /tmp/rustfs-odirect.img
          sudo mkdir -p /mnt/rustfs-odirect
          sudo mount -o loop /tmp/rustfs-odirect.img /mnt/rustfs-odirect
          sudo chmod 1777 /mnt/rustfs-odirect
          mount | grep rustfs-odirect

      # RUSTFS_URING_TESTS_MUST_RUN=1 makes the io_uring tests fail instead of
      # silently skipping if io_uring is unavailable, so this leg can never pass
      # vacuously. RUSTFS_IO_URING_READ_ENABLE=true selects the UringBackend.
      # TMPDIR points every tempfile-based test fixture at the ext4 loopback so
      # eligible reads keep O_DIRECT and the native path is covered end to end.
      - name: Run ecstore io_uring integration tests on real io_uring
        env:
          RUSTFS_IO_URING_READ_ENABLE: "true"
          RUSTFS_URING_TESTS_MUST_RUN: "1"
          TMPDIR: /mnt/rustfs-odirect
        # --lib narrows what gets compiled, not what gets run: every selected
        # test lives in the lib target. The 7 integration binaries under
        # crates/ecstore/tests/ each reported "running 0 tests" here, so they
        # were compiled and linked for nothing.
        #
        # The `uring_` filter must stay exactly as it is. libtest matches on
        # substring, so it also selects names containing `during_` — 6 of the 18
        # selected tests are such incidental matches. Narrowing the filter to
        # `io_uring` would silently drop them, which is a coverage change.
        # scripts/check_uring_lane_lib_only.sh guards the --lib precondition.
        run: cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture

  core-contract:
    name: Core contract
    needs: [ build-rustfs-debug-binary ]
    # Compile-time fixture paths must match the producer until the pilot passes.
    runs-on: sm-standard-4
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - uses: ./.github/actions/test-runtime
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: rustfs-debug-binary
          path: target/debug
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: e2e-test-binaries
          path: ${{ runner.temp }}
      - name: Run core storage and authorization contract
        env:
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/core-contract-logs
        run: |
          chmod +x target/debug/rustfs
          python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
          cargo nextest list --profile core-contract --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "$RUNNER_TEMP/core-contract-list.json"
          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile core-contract --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" --no-tests fail
          python3 scripts/ci_test_artifact.py evidence --core "$RUNNER_TEMP/core-contract-list.json" target/nextest/core-contract/junit.xml
      - name: Upload core contract evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: core-contract-${{ github.run_attempt }}
          path: |
            target/nextest/core-contract/junit.xml
            ${{ runner.temp }}/core-contract-list.json
            ${{ runner.temp }}/core-contract-logs
          if-no-files-found: error
          retention-days: 3

  e2e-tests:
    name: End-to-End Tests
    needs: [ build-rustfs-debug-binary ]
    # Retain the producer image and fixture paths until the runtime pilot passes.
    runs-on: sm-standard-4
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Install verification runtime
        uses: ./.github/actions/test-runtime

      - name: Download E2E test binaries
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: e2e-test-binaries
          path: ${{ runner.temp }}

      - name: Verify E2E archive identity
        run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install awscurl
        run: |
          python3 -m pip install --user --upgrade pip "awscurl==0.44"
          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"

      - name: Verify awscurl
        run: test -x "$AWSCURL_PATH"

      # Download after the cache restore so the freshly built binary from the
      # build job always wins over anything restored into target/debug.
      - name: Download debug binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: rustfs-debug-binary
          path: target/debug

      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs

      # Build the e2e test graph once. The archive is reused by the smoke
      # selection guard, security exact-count check, and run below, avoiding a
      # second compile of the same e2e_test target on cold runners (backlog#1645).
      - name: Verify e2e smoke test membership
        env:
          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
        run: |
          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
          ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"

      # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
      # profile.e2e-smoke default-filter in .config/nextest.toml is the single
      # wiring mechanism for e2e tests in CI — extend that filter instead of
      # adding new e2e jobs here. Each test spawns its own rustfs server on a
      # random port and reuses the downloaded debug binary above.
      - name: Run e2e smoke suite
        env:
          NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
          SAMPLER_INTERVAL_SECS: "15"
        run: |
          ./scripts/ci/resource_sampler.sh start e2e-smoke
          trap './scripts/ci/resource_sampler.sh stop' EXIT
          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
            --status-level all --final-status-level all --failure-output final

      - name: Verify e2e smoke execution completeness
        run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml

      - name: Upload e2e smoke diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-smoke-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            ${{ runner.temp }}/rustfs-e2e-smoke-logs/
            ${{ runner.temp }}/rustfs-e2e-smoke-list.json
            artifacts/test-and-lint
          if-no-files-found: warn

      - name: Upload e2e smoke JUnit report
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-smoke-junit-${{ github.run_number }}-${{ github.run_attempt }}
          path: target/nextest/e2e-smoke/junit.xml
          if-no-files-found: warn

      - name: Install tool build dependencies
        run: sudo apt-get install -y build-essential cmake pkg-config

      - name: Install s3s-e2e test tool
        uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
        with:
          tool: s3s-e2e
          git: https://github.com/s3s-project/s3s.git
          rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52

      - name: Run end-to-end tests
        run: |
          s3s-e2e --version
          RUN_ROOT="${RUNNER_TEMP}/rustfs-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
          mkdir -p "${RUN_ROOT}"
          RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
          RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
          RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"

      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-test-logs-${{ github.run_number }}-${{ github.run_attempt }}
          path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
          retention-days: 3

  e2e-full:
    name: End-to-End Tests (full merge gate)
    # Merge gate only (backlog#1149 ci-5): the never-automated user-visible
    # suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
    # security-boundary, ... — via the e2e-full nextest profile. Too heavy for
    # every PR, so it is gated to main pushes, the merge queue, and manual
    # dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
    # owned by other lanes (see .config/nextest.toml profile.e2e-full).
    if: >-
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'merge_group' ||
      (github.event_name == 'push' && github.ref == 'refs/heads/main')
    needs: [ build-rustfs-debug-binary ]
    # Retain the producer image and fixture paths until the runtime pilot passes.
    runs-on: sm-standard-4
    timeout-minutes: 55
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Install verification runtime
        uses: ./.github/actions/test-runtime

      - name: Download E2E test binaries
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: e2e-test-binaries
          path: ${{ runner.temp }}

      - name: Verify E2E archive identity
        run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"

      - name: Install network fault-injection tools
        run: |
          sudo apt-get install -y iptables
          sudo -n iptables --version
          # The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
          # runners can run iptables but not touch the rule set; the test then logs
          # a skip instead of failing, so surface that here where it is visible.
          if ! sudo -n iptables -w 5 -S OUTPUT >/dev/null 2>&1; then
            echo "::warning::iptables cannot read the OUTPUT chain on this runner (no CAP_NET_ADMIN); the endpoint-blackhole heal scenario will be skipped"
          fi

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install awscurl
        run: |
          python3 -m pip install --user --upgrade pip "awscurl==0.44"
          echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"

      - name: Verify awscurl
        run: test -x "$AWSCURL_PATH"

      - name: Install mc
        env:
          MC_VERSION: RELEASE.2025-08-13T08-35-41Z
          MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
        run: |
          MC_BINARY="mc.linux-amd64.${MC_VERSION}"
          curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
          echo "${MC_SHA256}  $RUNNER_TEMP/mc" | sha256sum --check --status
          chmod +x "$RUNNER_TEMP/mc"
          echo "$RUNNER_TEMP" >> "$GITHUB_PATH"

      - name: Verify mc
        run: mc --version

      - name: Install Vault
        run: |
          VAULT_VERSION="1.17.6"
          VAULT_ARCHIVE="vault_${VAULT_VERSION}_linux_amd64.zip"
          curl -fsSLo "$RUNNER_TEMP/$VAULT_ARCHIVE" "https://releases.hashicorp.com/vault/${VAULT_VERSION}/${VAULT_ARCHIVE}"
          echo "0cddc1fbbb88583b5ba5b845f9f8fae47c6fb39a6d48cd543c6ba6fd3ac1a669  $RUNNER_TEMP/$VAULT_ARCHIVE" | sha256sum --check --status
          unzip -q "$RUNNER_TEMP/$VAULT_ARCHIVE" -d "$RUNNER_TEMP/vault-bin"
          echo "RUSTFS_TEST_VAULT_BIN=$RUNNER_TEMP/vault-bin/vault" >> "$GITHUB_ENV"

      - name: Verify Vault
        run: |
          "$RUSTFS_TEST_VAULT_BIN" version

      # Download after the cache restore so the freshly built binary from the
      # build job always wins over anything restored into target/debug.
      - name: Download debug binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: rustfs-debug-binary
          path: target/debug

      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs

      - name: Preserve startup CAS binary input
        env:
          STARTUP_CAS_INPUT: ${{ runner.temp }}/rustfs-startup-cas-input
        run: |
          python3 - <<'PYINPUT'
          import hashlib
          import json
          import os
          import pathlib
          import shutil
          import subprocess

          source = pathlib.Path("target/debug/rustfs")
          manifest_path = source.with_name("rustfs.e2e-startup-cas-build.json")
          manifest = json.loads(manifest_path.read_text())
          target = pathlib.Path(os.environ["STARTUP_CAS_INPUT"])
          target.mkdir(parents=True, exist_ok=True)
          binary = target / "rustfs"
          shutil.copy2(source, binary)
          digest = hashlib.sha256()
          with binary.open("rb") as stream:
              for chunk in iter(lambda: stream.read(1024 * 1024), b""):
                  digest.update(chunk)
          commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
          if manifest["binary_sha256"] != digest.hexdigest() or manifest["commit"] != commit:
              raise SystemExit("downloaded hooks binary identity mismatch")
          shutil.copy2(manifest_path, target / manifest_path.name)
          shutil.copy2(source.with_name("rustfs.e2e.json"), target / "rustfs.e2e.json")
          binary.chmod(0o755)
          PYINPUT

      - name: Verify e2e full membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
        run: |
          cargo nextest list --profile e2e-full --archive-file "$RUNNER_TEMP/e2e-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"

      # Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
      # default-filter in .config/nextest.toml is the single wiring mechanism —
      # extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
      # debug binary; each test spawns its own rustfs server on a random port.
      - name: Run e2e full suite
        env:
          CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
          RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
          RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
          RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
          RUSTFS_HEAL_CHAOS_LOG_ROOT: ${{ runner.temp }}/rustfs-heal-chaos-logs
        run: python3 scripts/e2e_binary.py run --binary "$RUSTFS_E2E_STARTUP_CAS_BINARY" --features e2e-test-hooks -- cargo nextest run --profile e2e-full --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target"

      - name: Verify e2e full execution completeness
        run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-full-list.json" target/nextest/e2e-full/junit.xml

      - name: Upload junit
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-full-junit-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            target/nextest/e2e-full/junit.xml
            ${{ runner.temp }}/rustfs-e2e-full-list.json
          retention-days: 7

      - name: Upload startup CAS evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: fresh-startup-cas-evidence-${{ github.run_number }}-${{ github.run_attempt }}
          path: |
            ${{ runner.temp }}/rustfs-startup-cas-evidence
            ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
          if-no-files-found: warn
          retention-days: 7

      - name: Upload heal cluster logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-full-heal-logs-${{ github.run_number }}-${{ github.run_attempt }}
          path: ${{ runner.temp }}/rustfs-heal-chaos-logs
          if-no-files-found: warn
          retention-days: 7

  e2e-tests-rio-v2:
    name: End-to-End Tests (rio-v2)
    # Inherits the schedule/dispatch-only gate through needs: on every other
    # event build-rustfs-debug-binary-rio-v2 is skipped, so this job skips
    # with it (see the dormant-variant comment on that job).
    needs: [ build-rustfs-debug-binary-rio-v2 ]
    runs-on: sm-standard-2
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Clean up previous test run
        run: |
          rm -rf /tmp/rustfs
          rm -f /tmp/rustfs.log

      - name: Download debug binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: rustfs-debug-binary-rio-v2
          path: target/debug

      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs

      - name: Setup Rust toolchain for s3s-e2e installation
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

      # The sm-standard-* custom runner images (introduced in #4884) ship no C
      # toolchain, unlike GitHub-hosted ubuntu-latest. Installing s3s-e2e below
      # compiles it from source on a cache miss, and build scripts need cc.
      - name: Install build tools for s3s-e2e compilation
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential cmake pkg-config libssl-dev

      - name: Install s3s-e2e test tool
        uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
        with:
          tool: s3s-e2e
          git: https://github.com/s3s-project/s3s.git
          rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52

      - name: Run end-to-end tests
        run: |
          s3s-e2e --version
          python3 scripts/e2e_binary.py run --features rio-v2,e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs

      - name: Upload test logs
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: e2e-test-logs-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
          path: /tmp/rustfs.log
          retention-days: 3

  s3-implemented-tests:
    name: S3 Implemented Tests
    needs: [ build-rustfs-debug-binary ]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Download debug binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: rustfs-debug-binary
          path: target/debug

      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs

      - name: Run implemented s3-tests
        run: |
          RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
          mkdir -p "${RUN_ROOT}"
          S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
          DEPLOY_MODE=binary \
          RUSTFS_BINARY=./target/debug/rustfs \
          TEST_MODE=single \
          MAXFAIL=0 \
          XDIST=4 \
          S3_PORT="${S3_PORT}" \
          DATA_ROOT="${RUN_ROOT}" \
          S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh

      - name: Upload s3 test artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: s3tests-implemented-${{ github.run_number }}-${{ github.run_attempt }}
          path: artifacts/s3tests-single/**
          if-no-files-found: ignore
          retention-days: 3

  # Dedicated lifecycle behavior lane (backlog#1148 ilm-10, master plan #1155).
  #
  # These ceph/s3-tests cases assert that objects/versions/uploads are actually
  # removed by the background scanner and the stale-multipart cleanup loop, so
  # they need a debug-accelerated day plus an enabled scanner. That configuration
  # cannot be applied to the s3-implemented-tests lane above because a global
  # RUSTFS_ILM_DEBUG_DAY_SECS also shrinks the x-amz-expiration header those tests
  # assert on (test_lifecycle_expiration_header_*). Hence a separate server here.
  #
  # RUSTFS_ILM_DEBUG_DAY_SECS MUST stay equal to the s3-tests lc_debug_interval
  # default (10s). test_lifecycle_expiration polls the Days=1 rule with a
  # 4*lc_interval window while the Days=5 rule fires at 5*debug_day; observing
  # the intermediate 4-object plateau requires 4*lc_interval < 5*debug_day, i.e.
  # debug_day > 8. A smaller value lets the Days=5 rule fire inside the Days=1
  # poll window so the count collapses straight to 2 and the test fails (#4764).
  #
  # RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 is also required. A compacted directory
  # is only re-descended (and its objects re-evaluated for ILM) once every
  # DATA_USAGE_UPDATE_DIR_CYCLES scanner cycles (default 16). At the accelerated
  # RUSTFS_SCANNER_CYCLE=2 that is ~32s between evaluations, so a Days=1 object
  # that becomes due at debug_day (10s) is not actually expired until the next
  # ~32s boundary (~42s), landing just past the 4*lc_interval (40s) poll window
  # and making the count stall at 6 (#4767). Forcing every-cycle re-descent
  # evaluates ILM within ~2s of the due time, well inside the poll window.
  s3-lifecycle-behavior-tests:
    name: S3 Lifecycle Behavior Tests
    # Also gated on e2e-tests, matching s3-implemented-tests: when the e2e smoke
    # suite is already red this lane cannot tell us anything new, and it holds a
    # runner for up to 30 minutes doing so. Both lanes only download the
    # prebuilt debug binary (no cargo build), and s3-implemented-tests — which
    # already waits on e2e-tests — finishes later anyway, so a green PR's total
    # wall clock is unchanged.
    needs: [ build-rustfs-debug-binary ]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false

      - name: Download debug binary
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: rustfs-debug-binary
          path: target/debug

      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs

      - name: Run lifecycle behavior s3-tests
        run: |
          RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
          mkdir -p "${RUN_ROOT}"
          S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
          DEPLOY_MODE=binary \
          RUSTFS_BINARY=./target/debug/rustfs \
          TEST_MODE=single \
          MAXFAIL=0 \
          XDIST=0 \
          IMPLEMENTED_TESTS_FILE=scripts/s3-tests/lifecycle_behavior_tests.txt \
          RUSTFS_ILM_DEBUG_DAY_SECS=10 \
          RUSTFS_ILM_PROCESS_TIME=1 \
          RUSTFS_SCANNER_ENABLED=true \
          RUSTFS_SCANNER_CYCLE=2 \
          RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=1 \
          RUSTFS_SCANNER_START_DELAY_SECS=0 \
          RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s \
          S3_PORT="${S3_PORT}" \
          DATA_ROOT="${RUN_ROOT}" \
          S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
          python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh

      - name: Upload s3 test artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: s3tests-lifecycle-behavior-${{ github.run_number }}-${{ github.run_attempt }}
          path: artifacts/s3tests-single/**
          if-no-files-found: ignore
          retention-days: 3

  required-checks:
    name: Test and Lint
    if: always() && (github.event_name != 'pull_request' || github.event.action != 'closed')
    needs:
      - classify-changes
      - typos
      - quick-checks
      - test-and-lint
      - workspace-tests
      - workspace-shards
      - workspace-shard-evidence
      - core-contract
      - connect-feature-boundaries
      - test-and-lint-rio-v2
      - test-and-lint-protocols
      - build-rustfs-debug-binary
      - uring-integration
      - e2e-tests
      - s3-implemented-tests
      - s3-lifecycle-behavior-tests
      - build-rustfs-debug-binary-rio-v2
      - e2e-tests-rio-v2
      - e2e-full
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - name: Require the expected result of every CI lane
        env:
          CI_NEEDS: ${{ toJSON(needs) }}
        shell: bash
        run: python3 scripts/ci_gate.py verify

  alert-on-failure:
    name: Alert on scheduled failure
    needs:
      - classify-changes
      - connect-feature-boundaries
      - required-checks
      - typos
      - quick-checks
      - test-and-lint
      - workspace-tests
      - core-contract
      - test-and-lint-rio-v2
      - test-and-lint-protocols
      - build-rustfs-debug-binary
      - build-rustfs-debug-binary-rio-v2
      - uring-integration
      - e2e-tests
      - e2e-full
      - e2e-tests-rio-v2
      - s3-implemented-tests
      - s3-lifecycle-behavior-tests
    if: >-
      always() && github.event_name == 'schedule' &&
      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - name: Open or update failure-tracking issue
        uses: ./.github/actions/schedule-failure-issue
        with:
          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}