fix(ci): reduce duplicate work and preserve reliable test failures (#8233)

* fix(ci): reduce duplicate work and preserve reliable test failures

* fix(ci): retain protocol evidence and repair stale test fixtures

* test(connect): honor parent deadline during API fixture readiness

* fix(ci): reserve IO capacity for state writer proofs

* test(connect): align RPC fixtures with service capture contracts

* test(connect): cover pinned service capture failures

.github/actions/quick-checks/action.yml

@@ -100,12 +100,7 @@ runs:
 
     - name: Check test wiring
       shell: bash
-      run: |
-        python3 ./scripts/check_test_wiring.py --self-test
-        python3 ./scripts/check_scheduled_validation_freshness.py --self-test
-        python3 ./scripts/test_security_workflow.py
-        python3 ./scripts/test_nightly_candidate.py
-        python3 ./scripts/check_test_wiring.py
+      run: python3 ./scripts/check_test_wiring.py
 
     - name: Check no planning docs committed
       shell: bash

.github/actions/setup/action.yml

@@ -107,6 +107,8 @@ runs:
     - name: Install cargo-nextest
       if: inputs.install-test-tools == 'true'
       uses: taiki-e/install-action@96c7780c1d8a2b8723e12031def873a434d39d8d # nextest
+      with:
+        tool: cargo-nextest@0.9.138
 
     - name: Setup Rust cache
       uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2

.github/workflows/architecture-migration-rules.yml

@@ -1,61 +0,0 @@
-# Copyright 2024 RustFS Team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: Architecture Migration Rules
-
-on:
-  pull_request:
-    types: [ opened, synchronize, reopened, closed ]
-    branches: [ main ]
-    paths:
-      - "ARCHITECTURE.md"
-      - "docs/architecture/**"
-      - "scripts/check_architecture_migration_rules.sh"
-      - ".github/workflows/architecture-migration-rules.yml"
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  cancel-closed-pr-runs:
-    name: Cancel Closed PR Runs
-    if: github.event_name == 'pull_request' && github.event.action == 'closed'
-    runs-on: sm-standard-2
-    timeout-minutes: 10
-    steps:
-      - name: Explain cancellation run
-        run: echo "PR closed; this run only cancels older runs in the same concurrency group."
-
-  architecture-migration-rules:
-    name: Architecture Migration Rules
-    if: github.event_name != 'pull_request' || github.event.action != 'closed'
-    runs-on: sm-standard-2
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
-        with:
-          persist-credentials: false
-
-      - name: Install ripgrep
-        uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2
-        with:
-          tool: ripgrep@15.2.0
-
-      - name: Check architecture migration rules
-        run: ./scripts/check_architecture_migration_rules.sh

.github/workflows/cache-warm.yml

@@ -15,16 +15,16 @@
 # Sole writer of the Rust dependency caches that ci.yml restores.
 #
 # Why this is a separate workflow rather than steps inside ci.yml: ci.yml's
-# concurrency group cancels in-progress runs on main pushes, and merges land far
-# faster than its 70-minute pipeline. Measured over 15 consecutive main pushes:
+# concurrency group originally cancelled in-progress main runs, while merges
+# landed faster than its 70-minute pipeline. Over 15 consecutive main pushes:
 # 12 cancelled, 2 failed, 0 succeeded. A cancelled run never reaches
 # Swatinem/rust-cache's post step (cache-on-failure does not cover cancellation),
 # so the writer lanes were saving nothing and every PR paid a cold restore —
 # 11.8-20.9 minutes of "Setup Rust environment" against 0.7-3.4 warm.
 #
-# Splitting cache writing out of the test pipeline lets ci.yml keep cancelling
-# superseded runs (which is correct — nobody needs test results for a commit
-# that is already three merges behind) while the caches still get written.
+# Keep cache writing separate from validation: only these jobs publish the
+# complete feature closure for each key. Main validation now also finishes its
+# running baseline, while superseded PR attempts can still be cancelled.
 #
 # The group below deliberately does NOT cancel in progress; see the comment on
 # it for how that bounds concurrency and why it is scoped by event.
@@ -256,7 +256,7 @@ jobs:
   # belonged to it.
   warm-ci-uring:
     name: Warm ci-uring
-    runs-on: sm-standard-2
+    runs-on: ubuntu-latest
     timeout-minutes: 60
     steps:
       - name: Checkout repository

.github/workflows/ci.yml

@@ -47,12 +47,12 @@ permissions:
   contents: read
 
 # Concurrency groups are scoped per event so different triggers never cancel
-# each other: PR pushes cancel the previous run of that PR, main pushes keep
-# latest-wins semantics among themselves, and scheduled runs always complete
-# (a shared group used to let every merge kill the weekly scheduled run).
+# each other. PR pushes cancel superseded attempts; main and scheduled runs
+# finish so a busy merge stream cannot starve the complete baseline. For main,
+# GitHub retains one running run and replaces the pending run with the latest.
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
   CARGO_TERM_COLOR: always
@@ -128,7 +128,7 @@ jobs:
 
   test-and-lint:
     name: Workspace Test and Lint
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 90
@@ -146,11 +146,9 @@ jobs:
         uses: ./.github/actions/setup
         with:
           rust-version: stable
-          # Every lane in this workflow reads its cache and none writes it.
-          # cache-warm.yml is the sole writer for all four keys: this workflow
-          # cancels superseded runs on main, and a cancelled run never reaches
-          # rust-cache's post step, so writing from here saved nothing (12 of 15
-          # consecutive main-push runs were cancelled). See rustfs/backlog#1600.
+          # Every lane reads its cache; cache-warm.yml remains the sole writer
+          # with the matching feature closure. Tests never compete to replace
+          # shared caches, including when a PR attempt is cancelled.
           cache-shared-key: ci-dev
           cache-save-if: 'false'
           install-build-packaging-tools: 'false'
@@ -281,21 +279,10 @@ jobs:
       - name: Check log-analyzer rule anchors
         run: ./scripts/check_log_analyzer_rules.sh
 
-      # Explicit gate for migration-critical suites. These tests already ran in
-      # the full nextest pass above; a single filtered nextest invocation keeps
-      # the named gate without rebuilding or re-running them one package at a time.
-      #
-      # The gate selects tests by name substring (data_movement / rebalance /
-      # decommission / source_cleanup / delete_marker), so renames can silently
-      # thin it. The script owns the filter expression and first verifies the
-      # selected-test count against the committed floor in
-      # .config/migration-gate-floor.txt before running the gate; renames or
-      # removals must update that file consciously (see the script header).
-      # Kept on the default profile (no --profile ci): a second --profile ci run
-      # would clobber target/nextest/ci/junit.xml, and none of these tests are
-      # quarantined so they gain nothing from the ci profile's retry overrides.
-      - name: Run rebalance/decommission migration proofs
-        run: ./scripts/check_migration_gate_count.sh
+      # Preserve the migration floor and require successful execution evidence
+      # from the workspace run, without compiling or executing its subset again.
+      - name: Verify rebalance/decommission migration proofs
+        run: ./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
 
   # This gate builds into a fresh target directory to isolate the E2E root.
   # Give its cold build a separate budget from workspace tests and migration proofs.
@@ -525,13 +512,9 @@ jobs:
     runs-on: sm-standard-4
     timeout-minutes: 90
     strategy:
-      # On a PR, one failing protocol leg is enough to know the PR is not ready,
-      # so stop the sibling leg instead of paying another ~40 minutes for it.
-      # Everywhere else (main pushes, the merge queue, the weekly schedule) keep
-      # the full signal: there we want to know whether swift AND sftp are broken,
-      # not just whichever failed first. This is the only part of the early-stop
-      # work that also covers fork PRs, since it needs no token.
-      fail-fast: ${{ github.event_name == 'pull_request' }}
+      # Preserve both independent results when one protocol fails, so diagnosis
+      # and a focused fix do not require rebuilding an interrupted sibling.
+      fail-fast: false
       matrix:
         features:
           - name: swift
@@ -562,6 +545,8 @@ jobs:
           cargo clippy -p rustfs -p rustfs-protocols --all-targets ${{ matrix.features.flags }} -- -D warnings
 
       - name: Run tests with ${{ matrix.features.name }}
+        id: protocol-tests
+        shell: bash
         env:
           # Keep feature-test linking under the same bounded concurrency as the
           # main nextest lane; Clippy is metadata-only and needs no such limit.
@@ -570,11 +555,26 @@ jobs:
           # --profile ci so the quarantine list (and its junit flaky markers)
           # covers this leg too; the default profile is the local no-retry
           # profile and silently ignored quarantined flakes here (rustfs#6703).
-          cargo nextest run --profile ci -p rustfs -p rustfs-protocols ${{ matrix.features.flags }}
+          mkdir -p artifacts/protocol-tests
+          rm -f target/nextest/ci/junit.xml
+          cargo nextest run --profile ci -p rustfs -p rustfs-protocols ${{ matrix.features.flags }} \
+            2>&1 | tee artifacts/protocol-tests/nextest.log
+
+      - name: Upload protocol test reports and diagnostics
+        if: >-
+          always() && contains(fromJSON('["success", "failure", "cancelled"]'), steps.protocol-tests.outcome)
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: junit-test-and-lint-${{ matrix.features.name }}-${{ github.run_number }}-${{ github.run_attempt }}
+          path: |
+            target/nextest/ci/junit.xml
+            artifacts/protocol-tests
+          retention-days: 3
+          if-no-files-found: warn
 
   build-rustfs-debug-binary:
     name: Build RustFS Debug Binary
-    if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
+    if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
     needs: [ quick-checks, classify-changes ]
     runs-on: sm-standard-4
     timeout-minutes: 30

.github/workflows/e2e-distributed.yml

@@ -38,6 +38,7 @@ on:
       - "Cargo.lock"
       - "Cargo.toml"
       - ".config/nextest.toml"
+      - ".config/e2e-distributed-selection.txt"
       - ".github/workflows/e2e-distributed.yml"
       - "crates/audit/**"
       - "crates/common/**"

.github/workflows/e2e-upgrade.yml

@@ -51,8 +51,53 @@ env:
   UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
 
 jobs:
+  build:
+    name: Build upgrade candidate
+    runs-on: sm-standard-2
+    timeout-minutes: 60
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          cache-shared-key: e2e-upgrade-server
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: "false"
+          install-test-tools: "false"
+
+      - name: Download pinned previous release
+        run: |
+          mkdir -p target/debug
+          archive="target/debug/$UPGRADE_SOURCE_ASSET"
+          curl --fail --location --retry 3 --output "$archive" \
+            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
+
+      - name: Build current RustFS binary
+        run: python3 scripts/e2e_binary.py build
+
+      - name: Upload verified upgrade candidate
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: upgrade-candidate-${{ github.run_id }}
+          path: |
+            target/debug/rustfs
+            target/debug/rustfs.e2e.json
+            target/debug/${{ env.UPGRADE_SOURCE_ASSET }}
+          if-no-files-found: error
+          retention-days: 3
+          compression-level: 0
+          overwrite: true
+
   upgrade:
     name: ${{ matrix.name }}
+    needs: build
     strategy:
       fail-fast: false
       matrix:
@@ -63,31 +108,24 @@ jobs:
           # the CI required-check names. UPGRADE_SOURCE_VERSION above is the
           # single source of truth for which release they actually run against.
           - name: Direct upgrade from the previous release
-            cache_key: e2e-direct-upgrade
             test: direct_upgrade_from_rc2_preserves_object_contracts
             artifact: direct-upgrade
           - name: Mixed-version rolling upgrade from the previous release
-            cache_key: e2e-mixed-version-upgrade
             test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
             artifact: mixed-version-upgrade
           - name: Bucket configuration survives the upgrade
-            cache_key: e2e-bucket-config-upgrade
             test: direct_upgrade_from_previous_release_preserves_bucket_configuration
             artifact: bucket-config-upgrade
           - name: Rollback reads current bucket metadata
-            cache_key: e2e-bucket-config-rollback
             test: rollback_to_previous_release_reads_current_bucket_metadata
             artifact: bucket-config-rollback
           - name: ODM configuration recovery after rc.5 rollback
-            cache_key: e2e-odm-config-rollback
             test: rc5_rollback_requires_restoring_odm_configuration
             artifact: odm-config-rollback
           - name: Multipart layouts survive the rc.5 upgrade
-            cache_key: e2e-multipart-layout-upgrade
             test: direct_upgrade_from_rc5_preserves_multipart_layouts
             artifact: multipart-layout-upgrade
           - name: rc.5 multipart replication baseline
-            cache_key: e2e-multipart-layout-baseline
             test: rc5_baseline_replicates_multipart_layouts
             artifact: multipart-layout-baseline
     runs-on: sm-standard-2
@@ -103,19 +141,28 @@ jobs:
       - name: Setup Rust environment
         uses: ./.github/actions/setup
         with:
-          cache-shared-key: ${{ matrix.cache_key }}
-          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          cache-shared-key: e2e-upgrade-tests
+          # One main-branch consumer warms the shared test-harness dependencies.
+          cache-save-if: ${{ github.ref == 'refs/heads/main' && matrix.artifact == 'direct-upgrade' }}
           install-build-packaging-tools: "false"
+          install-test-tools: "false"
 
-      - name: Download pinned previous release
+      - name: Download verified upgrade candidate
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: upgrade-candidate-${{ github.run_id }}
+          path: target/debug
+
+      - name: Restore candidate executable bit
+        run: chmod +x target/debug/rustfs
+
+      - name: Verify and unpack pinned previous release
         env:
           SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
         run: |
           set -euo pipefail
           mkdir -p "$SOURCE_DIR"
-          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
-          curl --fail --location --retry 3 --output "$archive" \
-            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          archive="target/debug/$UPGRADE_SOURCE_ASSET"
           echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
           unzip -q "$archive" -d "$SOURCE_DIR"
           chmod +x "$SOURCE_DIR/rustfs"
@@ -123,14 +170,12 @@ jobs:
           echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
           echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
 
-      - name: Build current RustFS binary
-        run: |
-          python3 scripts/e2e_binary.py build
-
       - name: Run upgrade compatibility test
         env:
           RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
+          NO_PROXY: 127.0.0.1,localhost,::1
         run: |
+          export no_proxy="$NO_PROXY"
           python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
             "upgrade_compatibility_test::${{ matrix.test }}" \
             -- --ignored --exact --nocapture