ci: harden test selection and nightly coverage (#6341)

2026-08-22 20:36:38 +00:00 · 2026-08-21 23:04:08 +08:00
parent bce5922aef
commit bc07cfd115
31 changed files with 1146 additions and 259 deletions
@@ -0,0 +1,2 @@
 sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
 sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
@@ -0,0 +1 @@
 sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680
@@ -0,0 +1,2 @@
 sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315
 sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17
@@ -0,0 +1 @@
 sha256=655a3f3c1d042e694339d15caba7580518320322d1bac0f09450b37e6c09e2e7
@@ -0,0 +1 @@
 sha256=ec27cde6ce6400723c4b372bfbd2ac61709c744294e4810af765e8a808d8e31d
@@ -75,6 +75,11 @@ embedded-secrets-check: ## Check no private key material or credential literal i
 	@echo "🔑 Checking embedded secret material guard..."
 	./scripts/check_embedded_secrets.sh
 .PHONY: test-wiring-check
 test-wiring-check: ## Check tests stay registered and selected by their intended runners
 	@echo "🧪 Checking test wiring..."
 	python3 ./scripts/check_test_wiring.py
 .PHONY: log-analyzer-rules-check
 log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
 	@echo "🩺 Checking log-analyzer rule anchors..."
@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
 	./scripts/check_no_planning_docs.sh
 .PHONY: pre-commit
-pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
+pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
 	@echo "✅ All pre-commit checks passed!"
 .PHONY: pre-pr
-pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
+pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
 	@echo "✅ All pre-PR checks passed!"
 .PHONY: dev-check
-dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
+dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
 	@echo "✅ Fast development checks passed!"
@@ -35,6 +35,8 @@ script-tests: ## Run shell script tests
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
 	./scripts/validate_object_data_cache_cold_stampede.sh --self-test
@@ -38,10 +38,11 @@ e2e-vault = { max-threads = 1 }
 # replacement_privileged_e2e_test when explicitly run as root on Linux). They
 # are correct in isolation but resource-heavy; serialize them under nextest's
 # process boundary (serial_test's #[serial] does not cross it) so several 4-disk
-# servers never run at once. ci-7's nightly picks these up via the e2e suite;
+# servers never run at once. The e2e-full merge/main lane picks these up;
 # they are deliberately NOT in the fast PR `e2e-smoke` filter.
 e2e-reliability = { max-threads = 1 }
 e2e-inline-boundaries = { max-threads = 1 }
 e2e-cluster-nightly = { max-threads = 1 }
 # --- default profile (local): serialize the flaky groups, never retry --------
 [[profile.default.overrides]]
@@ -161,7 +162,7 @@ retries = 2
 # Serialize the 4-disk reliability / degraded-read e2e tests under the ci
 # profile too (see the e2e-reliability test-group note near the top). Not a
 # quarantine: no retries, just single-threaded so several 4-disk servers never
-# run concurrently when ci-7's nightly runs the full e2e suite.
+# run concurrently when e2e-full runs the suite.
 [[profile.ci.overrides]]
 filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
 test-group = 'e2e-reliability'
@@ -230,8 +231,8 @@ test-group = 'ecstore-serial-flaky'
 # the nightly profile derives its set as "the replication module MINUS this
 # allowlist", so any new replication test lands in nightly by default (never
 # silently unrun) until it is explicitly blessed as fast here. Keep the two
-# regexes byte-identical. Count invariant: 20 here + 49 nightly = 69 total
+# regexes byte-identical. The committed profile selection digests make changes
-# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
+# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md.
 # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
 # (#4724) because they set a loopback (127.0.0.1) replication target that the
 # SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
@@ -327,9 +328,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
 # the STS dual-node test actually exercises its path (it skips gracefully with
 # a visible log line when awscurl is absent), and routes scheduled failures
 # through .github/actions/schedule-failure-issue (ci-8). Explicit division of
-# labor with ci-5's future e2e-full merge gate: these tests run ONLY here, not
+# labor with e2e-full: these tests run only in the consolidated nightly
-# double-run there. TODO(ci-7): fold this interim repl-owned lane into the ci
+# workflow, not in the merge/main lane.
 # domain's consolidated scheduled e2e workflow once it exists.
 [profile.e2e-repl-nightly]
 default-filter = """
    package(e2e_test)
@@ -343,26 +343,60 @@ fail-fast = false
 # workflow as the failure-triage artifact.
 path = "junit.xml"
 # ---------------------------------------------------------------------------
 # e2e-nightly profile — destructive multi-process cluster fault domains
 # ---------------------------------------------------------------------------
 # These seven modules are deliberately outside e2e-full's merge budget. Each
 # starts a real multi-process or multi-disk topology and exercises node/disk
 # loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The
 # consolidated nightly workflow runs them serially to avoid resource
 # starvation; failures are never retried.
 [profile.e2e-nightly]
 default-filter = """
    package(e2e_test)
    & test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
 """
 fail-fast = false
 [profile.e2e-nightly.junit]
 path = "junit.xml"
 [[profile.e2e-nightly.overrides]]
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'
 # ---------------------------------------------------------------------------
 # e2e-protocols profile — serial protocol lane
 # ---------------------------------------------------------------------------
 # The suite owns fixed ports, so the nightly workflow runs this exact profile
 # with one nextest worker.
 [profile.e2e-protocols]
 default-filter = 'package(e2e_test) & test(/^protocols::/)'
 fail-fast = false
 [profile.e2e-protocols.junit]
 path = "junit.xml"
 # ---------------------------------------------------------------------------
 # e2e-full profile — merge-gate full single-node e2e lane (backlog#1149 ci-5)
 # ---------------------------------------------------------------------------
 # The merge gate (ci.yml `e2e-full` job: push main + merge_group +
-# workflow_dispatch). Runs the never-automated user-visible suites — KMS (40),
+# workflow_dispatch). Runs the user-visible KMS, object-lock, multipart-auth,
-# object_lock (33), multipart_auth (109), quota, checksum, encryption,
+# quota, checksum, encryption,
 # security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
 # skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
 # --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
 #
 # The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
-#   * protocols:: — FTPS/SFTP/WebDAV, still pinned to --test-threads=1 by fixed
+#   * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
-#     ports; they join a scheduled lane once ci-6 randomises the ports (ci-7).
+#     with one worker because the suite owns fixed ports.
 #   * the 7 cluster suites that spin up a RustFSTestClusterEnvironment
 #     (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
 #     namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
-#     object_lambda) — too heavy for the merge budget; they run in ci-7's
+#     object_lambda) — too heavy for the merge budget; they run in the
-#     nightly 4-node lane.
+#     e2e-nightly serial cluster-fault lane.
 #   * replication_extension_test — repl-1 already splits it into the PR
-#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (49 slow) lanes and reserves
+#     `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
 #     it for those, so e2e-full does not double-run it.
 #   * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
 #     manual-localhost:9000 reliant/policy tests are ci-13's migration.
@@ -46,10 +46,9 @@ lists when upstream changes.
  the PR.
 - **Weekly + manual**: `.github/workflows/e2e-s3tests.yml` runs the full
  upstream suite (`TEST_SCOPE=all`) against a Docker deployment (single node
-  or a 4-node distributed cluster behind HAProxy). It fails only on
+  or a 4-node distributed cluster behind HAProxy). The canonical gate policy
-  regressions in the implemented whitelist and publishes a classification
+  and compatibility-report behavior are documented in
-  report (`compat-report.md`, also shown in the job summary) listing promotion
+  [`scripts/s3-tests/README.md`](../../scripts/s3-tests/README.md).
  candidates and unclassified tests.
 ## Running Tests Locally
@@ -125,6 +125,9 @@ jobs:
      - name: Check no embedded secret material
        run: ./scripts/check_embedded_secrets.sh
      - name: Check test wiring
        run: python3 ./scripts/check_test_wiring.py
      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -160,6 +160,9 @@ jobs:
      - name: Check no embedded secret material
        run: ./scripts/check_embedded_secrets.sh
      - name: Check test wiring
        run: python3 ./scripts/check_test_wiring.py
      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -686,9 +689,9 @@ jobs:
      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs
-      # Build the e2e test graph once. The archive is reused by the security
+      # Build the e2e test graph once. The archive is reused by the smoke
-      # count-floor check and the smoke run below, avoiding a second compile of
+      # selection guard, security exact-count check, and run below, avoiding a
-      # the same e2e_test target on cold runners (backlog#1645).
+      # second compile of the same e2e_test target on cold runners (backlog#1645).
      - name: Archive e2e smoke test binaries
        env:
          NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
@@ -696,6 +699,7 @@ jobs:
        run: |
          cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
          cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
          ./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
      # PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
@@ -760,7 +764,7 @@ jobs:
    # suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
    # security-boundary, ... — via the e2e-full nextest profile. Too heavy for
    # every PR, so it is gated to main pushes, the merge queue, and manual
-    # dispatch. protocols / the 6 cluster suites / replication / #[ignore] are
+    # dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
    # owned by other lanes (see .config/nextest.toml profile.e2e-full).
    if: >-
      github.event_name == 'workflow_dispatch' ||
@@ -820,6 +824,13 @@ jobs:
      - name: Make binary executable
        run: chmod +x ./target/debug/rustfs
      - name: Verify e2e full membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
        run: |
          cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
      # Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
      # default-filter in .config/nextest.toml is the single wiring mechanism —
      # extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
@@ -832,7 +843,9 @@ jobs:
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: e2e-full-junit-${{ github.run_number }}
-          path: target/nextest/e2e-full/junit.xml
+          path: |
            target/nextest/e2e-full/junit.xml
            ${{ runner.temp }}/rustfs-e2e-full-list.json
          retention-days: 7
  e2e-tests-rio-v2:
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4).
+# Consolidated nightly e2e lane for replication, cluster faults, and protocols.
 #
 # The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the
 # FAST replication tests. This scheduled lane runs the remaining heavier
@@ -28,15 +28,12 @@
 # add ad-hoc cargo-test steps here; change the filterset instead. The
 # authoritative membership and count come from
 # `cargo nextest list -p e2e_test --profile e2e-repl-nightly`; the PR/nightly
-# count invariant is maintained next to the filtersets in .config/nextest.toml
+# selection digest is committed under .config/.
 # (deliberately not duplicated here).
 #
-# Explicit division of labor: the nightly subset runs ONLY here, never double-run
+# Explicit division of labor: these subsets run only here and never double-run
-# in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's
+# in the e2e-full merge gate.
 # consolidated scheduled e2e workflow exists, fold this interim repl-owned lane
 # into it rather than growing a second scheduled entrypoint.
-name: e2e-replication-nightly
+name: e2e-nightly
 on:
  workflow_dispatch:
@@ -50,6 +47,10 @@ on:
 permissions:
  contents: read
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false
 jobs:
  repl-nightly:
    name: Replication e2e (nightly)
@@ -97,9 +98,20 @@ jobs:
      # demand otherwise, but a single explicit build avoids several parallel
      # nextest test processes racing to build it at once.
      - name: Build rustfs binary
-        run: cargo build -p rustfs --bins
+        run: |
          cargo build -p rustfs --bins
          : > target/debug/rustfs.features
      - name: Verify replication e2e membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
        run: |
          cargo nextest list --profile e2e-repl-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-repl-nightly "${NEXTEST_LISTING}"
      - name: Run replication e2e nightly suite
        env:
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
        run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
      - name: Upload nextest junit report
@@ -107,13 +119,112 @@ jobs:
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: e2e-replication-nightly-junit-${{ github.run_number }}
-          path: target/nextest/e2e-repl-nightly/junit.xml
+          path: |
            target/nextest/e2e-repl-nightly/junit.xml
            ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
            ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs/
          retention-days: 7
          if-no-files-found: ignore
  cluster-nightly:
    name: Cluster fault e2e (nightly)
    runs-on: sm-standard-4
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-e2e-nightly
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'
      - name: Build rustfs binary
        run: |
          cargo build -p rustfs --bins --features e2e-test-hooks
          : > target/debug/rustfs.features
      - name: Verify cluster fault e2e membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
        run: |
          cargo nextest list --profile e2e-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-nightly "${NEXTEST_LISTING}"
      - name: Run cluster fault e2e nightly suite
        env:
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
        run: cargo nextest run --profile e2e-nightly -p e2e_test
      - name: Upload cluster fault diagnostics
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: e2e-cluster-nightly-${{ github.run_number }}
          path: |
            target/nextest/e2e-nightly/junit.xml
            ${{ runner.temp }}/rustfs-e2e-nightly-list.json
            ${{ runner.temp }}/rustfs-e2e-nightly-logs/
          retention-days: 7
          if-no-files-found: warn
  protocols-nightly:
    name: Protocol e2e (nightly)
    runs-on: sm-standard-4
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
    steps:
      - name: Checkout repository
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          rust-version: stable
          cache-shared-key: ci-e2e-protocols
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'
      # The suite owns fixed protocol ports and serializes its internal cases.
      - name: Verify protocol e2e membership
        env:
          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-protocols-list.json
        run: |
          cargo nextest list --profile e2e-protocols -p e2e_test --message-format json > "${NEXTEST_LISTING}"
          python3 ./scripts/check_test_wiring.py --check-profile e2e-protocols "${NEXTEST_LISTING}"
      - name: Run protocol e2e nightly suite
        env:
          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
        run: >-
          cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
      - name: Upload protocol diagnostics
        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: e2e-protocol-nightly-${{ github.run_number }}
          path: |
            target/nextest/e2e-protocols/junit.xml
            ${{ runner.temp }}/rustfs-e2e-protocols-list.json
            ${{ runner.temp }}/rustfs-protocol-e2e-logs/
          retention-days: 7
          if-no-files-found: warn
  alert-on-failure:
    name: Alert on scheduled failure
-    needs: [repl-nightly]
+    needs: [repl-nightly, cluster-nightly, protocols-nightly]
    # Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
    # manual workflow_dispatch runs stay quiet so a debugging run never files a
    # spurious alert.
@@ -18,10 +18,9 @@
 # runs only the implemented_tests.txt whitelist. This workflow complements it:
 #
 #   - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
-#     suite and reports promotion candidates (tests that newly pass) and
+#     suite and reports promotion candidates. Regressions, unclassified tests,
-#     unclassified tests. The job fails only on regressions in the implemented
+#     incomplete execution, and infrastructure errors fail the job; classified
-#     whitelist or on infrastructure errors — expected failures from
+#     failures for not-yet-implemented features remain informational.
 #     not-yet-implemented features do not turn the run red.
 #   - Manual runs (workflow_dispatch): same, with configurable mode/scope.
 #
 # All test execution is delegated to scripts/s3-tests/run.sh (single source of
@@ -45,13 +44,6 @@
 # The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
 # via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
 # DISABLED. This workflow is switched off in the repository's Actions settings
 # (state: disabled_manually) and does not run on any trigger, including its cron
 # and workflow_dispatch. That state lives in GitHub's UI and is invisible when
 # reading this file, which has already misled at least one audit — hence this
 # banner. Re-enabling is a UI action; anyone doing so should first check that the
 # workflow still matches the current CI layout. See rustfs/backlog#1603.
 #
 name: e2e-s3tests
 on:
@@ -81,6 +73,19 @@ on:
        description: "Stop after N failures. '0' to run everything."
        required: false
        default: "0"
      shard-count:
        description: "Exact-node-ID shard count for a targeted manual run"
        required: false
        default: "1"
        type: choice
        options:
          - "1"
          - "2"
          - "4"
      shard-index:
        description: "Zero-based shard index for a targeted manual run"
        required: false
        default: "0"
      markexpr:
        description: "Optional pytest -m expression"
        required: false
@@ -111,6 +116,8 @@ env:
  XDIST: ${{ github.event.inputs.xdist || '4' }}
  MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
  MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
  S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
  TEST_TIMEOUT: "300"
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
@@ -127,19 +134,22 @@ defaults:
 jobs:
  s3tests:
    name: s3tests (${{ matrix.test-mode }}, shard ${{ matrix.shard-index }})
    # GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
    # See the header note (ci-1) for why the self-hosted sm-standard-4 label
-    # was abandoned. TODO(ci-8): scheduled-failure alerting (auto-open issue)
+    # was abandoned. Scheduled failures are handled by alert-on-failure below.
    # is added by the ci-8 composite action; do not implement it here.
    runs-on: ubuntu-latest
    timeout-minutes: 180
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        # Scheduled sweeps cover both topologies; manual runs use the input.
        test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
        shard-index: ${{ github.event_name == 'schedule' && fromJSON('[0, 1, 2, 3]') || fromJSON(format('[{0}]', github.event.inputs.shard-index || '0')) }}
    env:
      TEST_MODE: ${{ matrix.test-mode }}
      S3_SHARD_INDEX: ${{ matrix.shard-index }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
@@ -181,6 +191,7 @@ jobs:
      - name: Start single RustFS
        if: env.TEST_MODE == 'single'
        run: |
          SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
          docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
          docker rm -f rustfs-single >/dev/null 2>&1 || true
          # The four disks share one physical device on the runner (a single
@@ -193,6 +204,7 @@ jobs:
            -e RUSTFS_ADDRESS=0.0.0.0:9000 \
            -e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
            -e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
            -e RUSTFS_SSE_S3_MASTER_KEY="${SSE_KEY}" \
            -e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
            -e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
            -v /tmp/rustfs-single:/data \
@@ -201,6 +213,7 @@ jobs:
      - name: Start 4-node distributed cluster
        if: env.TEST_MODE == 'multi'
        run: |
          SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
          # A real distributed deployment: every node lists all endpoints in
          # RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
          # node-local volume paths here — that would create four independent
@@ -213,6 +226,7 @@ jobs:
              RUSTFS_ADDRESS: "0.0.0.0:9000"
              RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
              RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
              RUSTFS_SSE_S3_MASTER_KEY: "${SSE_KEY}"
              RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
              # Each node's four disks share one physical device inside its
              # container, so bypass the local physical-disk-independence guard
@@ -294,7 +308,6 @@ jobs:
      - name: Run ceph s3-tests
        run: |
          set +e
          DEPLOY_MODE=existing \
          TEST_MODE="${TEST_MODE}" \
          TEST_SCOPE="${TEST_SCOPE}" \
@@ -302,26 +315,6 @@ jobs:
          MAXFAIL="${MAXFAIL}" \
          MARKEXPR="${MARKEXPR}" \
          ./scripts/s3-tests/run.sh
          RC=$?
          set -e
          if [ "${TEST_SCOPE}" = "implemented" ]; then
            # Whitelist run: every failure is a regression.
            exit "${RC}"
          fi
          # Full sweep: failures outside the implemented whitelist are
          # inventory (promotion candidates / unimplemented features), not a
          # gate. Fail only on whitelist regressions or infrastructure errors.
          JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
          if [ ! -f "${JUNIT}" ]; then
            echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
            exit "${RC}"
          fi
          python3 scripts/s3-tests/report_compat.py \
            --junit "${JUNIT}" \
            --lists-dir scripts/s3-tests \
            --fail-on-regression
      - name: Publish compatibility report
        if: always()
@@ -346,7 +339,7 @@ jobs:
        if: always() && env.ACT != 'true'
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: s3tests-${{ env.TEST_MODE }}
+          name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
          path: artifacts/**
  alert-on-failure:
@@ -12,27 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # DISABLED. This workflow is switched off in the repository's Actions settings
 # (state: disabled_manually) and does not run on any trigger, including its cron
 # and workflow_dispatch. That state lives in GitHub's UI and is invisible when
 # reading this file, which has already misled at least one audit — hence this
 # banner. Re-enabling is a UI action; anyone doing so should first check that the
 # workflow still matches the current CI layout. See rustfs/backlog#1603.
 #
 name: Fuzz
 on:
  pull_request:
    types: [ opened, synchronize, reopened, closed ]
-    # PR trigger is intentionally narrow: only changes to the fuzz harness
+    # Run when the harness or any directly fuzzed production crate changes.
    # itself gate a PR. Broad crate paths (ecstore/filemeta/utils/policy/…)
    # are covered by the nightly `schedule` run below, which fuzzes against
    # whatever landed on main. Widening these paths previously queued a
    # ~45min fuzz-build on nearly every PR and is why this workflow was
    # disabled; do not re-add crate paths here.
    paths:
      - "fuzz/**"
      - "scripts/fuzz/**"
      - "crates/ecstore/**"
      - "crates/filemeta/**"
      - "crates/policy/**"
      - "crates/security-governance/**"
      - "crates/utils/**"
      - "Cargo.toml"
      - "Cargo.lock"
      - ".github/workflows/fuzz.yml"
  schedule:
    - cron: "0 2 * * *"
@@ -81,7 +76,7 @@ jobs:
      github.event_name == 'schedule' ||
      github.event_name == 'workflow_dispatch'
    runs-on: sm-standard-4
-    timeout-minutes: 45
+    timeout-minutes: 60
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
@@ -121,12 +116,7 @@ jobs:
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: fuzz-prebuilt-binaries-${{ github.run_number }}
-          path: |
+          path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/
            fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/archive_extract
            fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/bucket_validation
            fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/local_metadata
            fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/path_containment
            fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/policy_ingress
          if-no-files-found: error
          retention-days: 1
          compression-level: 0
@@ -192,10 +182,7 @@ jobs:
  nightly-fuzz-corpus:
    name: "Nightly / ${{ matrix.target }}"
    needs: fuzz-build
-    # TODO(ci-8): when the schedule-failure-issue composite action lands,
+    # Scheduled failures are handled by alert-on-failure below.
    # add a step here (or a dependent job) that opens/updates a GitHub issue
    # on nightly failure. ci-8 is the single alerting mechanism for all
    # scheduled workflows; do not self-roll alerting in this workflow.
    if: >
      github.event_name == 'schedule' ||
      (github.event_name == 'workflow_dispatch' &&
@@ -48,16 +48,14 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
  -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
 # Protocols suite — fixed ports, MUST be single-threaded, gated by build features
 RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
  cargo test -p e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
 ```
 The protocols suite has its own contract (fixed bind ports 9022–9301,
-`--test-threads=1`, feature-gated scheduling) documented in
+single-worker execution, feature-gated scheduling) documented in
 [`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
 selects which features the spawned binary is built with; leave it unset to run
-every protocol entry.
+every protocol entry. Use the exact profile command under
 [Troubleshooting](#troubleshooting) for CI-equivalent execution.
 ### `#[ignore]` semantics
@@ -159,27 +157,26 @@ construction (random port + isolated temp dir) and need no serialization.
 ## CI map
 `e2e_test` is **excluded** from the main `cargo nextest run --profile ci --all`
-pass ([`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) line 158,
+pass (`--exclude e2e_test`) — the whole crate is too slow to gate every PR.
-`--exclude e2e_test`) — the whole crate is too slow to gate every PR. Subsets
+Subsets join CI through nextest profiles; the fixed-port protocol suite uses
-join CI through the nextest profile system only (never as ad-hoc jobs):
+the same profile for membership and execution with one nightly worker.
 | Suite | Runs where | Status |
 | --- | --- | --- |
 | Smoke subset (`e2e-smoke` profile) | `e2e-tests` job, every PR | **Active** (backlog#1149 ci-4) |
 | Full single-node suite (`e2e-full` profile) | `e2e-full` job, merge queue + main | **Active** (backlog#1149 ci-5) |
 | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
 | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
-| KMS suite | — | Not in CI yet (backlog#1149 ci-5) |
+| KMS suite | `e2e-full` job, merge queue + main | **Active** |
-| Protocols (FTPS/WebDAV/SFTP) | — | Not in CI yet (backlog#1149 ci-7) |
+| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
-| Replication (slow + dual-node) | `e2e-repl-nightly` profile, scheduled workflow | **Active** (backlog#1147 repl-1) |
+| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
-| `reliant/*` (pre-started server) | — | Manual only |
+| `reliant/*` | 19 tests in PR smoke; remaining default tests in `e2e-full` | **Active** except `#[ignore]` |
-Links: [`ci.yml`](../../.github/workflows/ci.yml) `e2e-tests` (line 347),
+The profile filters in [`.config/nextest.toml`](../../.config/nextest.toml) are
-`test-ilm-integration-serial` (line 196). The `e2e-smoke` `default-filter` in
+the wiring source of truth. Committed test-ID digests under
-[`.config/nextest.toml`](../../.config/nextest.toml) is the **single wiring
+`.config/e2e-*-selection.txt` make every membership change explicit.
 mechanism** — extend that filter (or add a sibling profile) to admit more
 tests; do not add e2e jobs to `ci.yml`. repl-1 / ilm-3 are landing in parallel
 and may add lanes; keep the table above easy to extend.
 ## Troubleshooting
@@ -188,9 +185,15 @@ and may add lanes; keep the table above easy to extend.
 ```bash
 # Smoke (e2e-tests job) — includes the 20 fast replication tests
 cargo nextest run --profile e2e-smoke -p e2e_test
-# Replication nightly lane (16 slow + dual-node tests; install awscurl for the
+# Full single-node merge/main lane
-# STS dual-node test, else it skips gracefully)
+cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
 # Replication nightly lane; install awscurl so STS paths do not skip
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
 RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
  cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
 # ILM serial lane
 cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
  -E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
@@ -273,4 +276,6 @@ current subset is.
 `docs/testing/e2e-suite-inventory.md` records the per-module test counts as
 listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
 moving e2e tests so acceptance numbers in the test-strategy issues
-(backlog#1147–#1155) stay auditable.
+(backlog#1147–#1155) stay auditable. When a profile membership change is
 intentional, review its JSON listing before updating the matching
 `.config/e2e-*-selection.txt` test-ID digest.
@@ -11,10 +11,17 @@ test process directly.
 ## Running Tests
 Use the canonical CI-equivalent protocol command in the parent
 [`e2e_test` README](../../README.md#troubleshooting).
 For targeted debugging of the core suite only:
 ```bash
 RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
 ```
 This targeted command does not cover the full `e2e-protocols` profile.
 `RUSTFS_BUILD_FEATURES` controls which features the test rustfs binary is
 built with. When this variable is set, the protocol test runner schedules
 only entries whose protocol is present in the requested feature list. Leave
@@ -133,4 +140,3 @@ property without consulting any external doc.
 Bind ports 9023 (SFTP) and 9100 (S3). Spawns rustfs with
 `RUSTFS_SFTP_IDLE_TIMEOUT=5`, sleeps 10 s past the timeout, then issues an
 SFTP request and asserts the server has closed the session.
@@ -922,14 +922,10 @@ mod prepared_get_object_metadata_tests {
            .expect("test should find an object whose initial fanout covers both data shards")
    }
-    #[allow(
+    fn bounded_initial_parity_disk_index(bucket: &str, object: &str) -> usize {
        dead_code,
        reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
    )]
    fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
        *bounded_metadata_fanout_order(bucket, object, 4, 2)
-            .get(3)
+            .get(2)
-            .expect("4-disk test geometry should leave one bounded spare disk")
+            .expect("4-disk test geometry should schedule one parity disk initially")
    }
    #[tokio::test]
@@ -1087,7 +1083,7 @@ mod prepared_get_object_metadata_tests {
                        ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
                    ],
                    async {
-                        let slow_parity_disk = bounded_spare_disk_index(bucket, &object);
+                        let slow_parity_disk = bounded_initial_parity_disk_index(bucket, &object);
                        let barrier =
                            rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
                        let calls = disk_call_counters::observe(&object);
@@ -622,7 +622,8 @@ mod test {
        let _resolver_lock = DNS_RESOLVER_TEST_LOCK.lock().unwrap();
        reset_dns_resolver_inner();
-        let err = resolve_domain("rustfs-resolver-provenance.invalid").unwrap_err();
+        // DNS labels are limited to 63 bytes, so the system resolver rejects this before lookup.
        let err = resolve_domain("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.invalid").unwrap_err();
        assert_ne!(err.kind(), std::io::ErrorKind::Other, "system resolver error was wrapped: {err}");
    }
@@ -5,12 +5,10 @@
 > ```bash
 > cargo nextest list -p e2e_test --message-format json | jq -r '.["rust-suites"][]?.testcases | to_entries[] | select(.value.ignored == false) | .key | split("::")[0]' | sort | uniq -c
 > ```
-> Modules marked ✅ are in the PR smoke profile `e2e-smoke`
+> Modules marked ✅ are in the PR smoke profile `e2e-smoke`; 🌙 marks the
-> (`.config/nextest.toml`); admission criteria: `crates/e2e_test/README.md`.
+> cluster, protocol, and replication subsets in the consolidated nightly
-> 🌙 marks tests in the scheduled `e2e-repl-nightly` profile (backlog#1147
+> workflow. The `e2e-full` merge/main profile covers the remaining default
-> repl-1): `replication_extension_test` splits 20 fast tests into the PR smoke
+> single-node tests. Committed test-ID digests are enforced before each run.
 > lane and 28 slow / `_real_dual_node` / `_real_three_node` / `_real_single_node` tests into the
 > nightly lane (`.github/workflows/e2e-replication-nightly.yml`).
 > Note: counts exclude `#[ignore]`d tests (nextest lists them separately).
 > Managed-SSE (SSE-S3/SSE-KMS) replication contracts assert successful
 > re-encryption on the target (backlog#1783); SSE-C replication still pins a
@@ -19,19 +17,21 @@
 | module | tests | PR smoke |
 |---|---|---|
 | admin_auth_test | 4 | ✅ |
-| admin_iam_crud_test | 2 | ✅ |
+| admin_iam_crud_test | 3 | ✅ |
 | admin_pools_test | 1 | ✅ |
-| admin_timeout_regression_test | 1 |  |
+| admin_timeout_regression_test | 1 | 🌙 |
 | anonymous_access_test | 4 | ✅ |
 | api_rate_limit_test | 3 |  |
 | archive_download_integrity_test | 13 |  |
 | bucket_logging_test | 3 |  |
 | bucket_policy_check_test | 1 | ✅ |
 | bucket_stats_regression_test | 3 |  |
 | chaos | 2 |  |
 | checksum_upload_test | 7 |  |
-| cluster_concurrency_test | 2 |  |
+| cluster_concurrency_test | 2 | 🌙 |
-| cluster_multidrive_pool_test | 2 |  |
+| cluster_multidrive_pool_test | 2 | 🌙 |
-| common | 12 |  |
+| common | 14 |  |
-| compression_test | 1 |  |
+| compression_test | 6 | ✅ |
 | connection_cap_test | 2 |  |
 | console_smoke_test | 1 | ✅ |
 | content_encoding_test | 3 | ✅ |
@@ -45,48 +45,58 @@
 | delete_marker_migration_semantics_test | 2 | ✅ |
 | delete_object_no_content_length_test | 1 |  |
 | delete_objects_versioning_test | 2 | ✅ |
 | delete_regression_test | 5 |  |
 | distributed_startup_regression_test | 3 |  |
 | existing_object_tag_policy_test | 4 |  |
-| fake_s3_target | 4 | ✅ |
+| fake_s3_target | 6 | ✅ |
 | fault_proxy | 7 |  |
 | get_codec_streaming_compat_test | 1 |  |
 | get_stream_failure_observability_test | 1 |  |
 | group_delete_test | 1 |  |
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
-| heal_erasure_disk_rebuild_test | 3 |  |
+| heal_erasure_disk_rebuild_test | 4 | 🌙 |
-| inline_fast_path_cluster_test | 14 |  |
+| inline_fast_path_cluster_test | 16 |  |
 | internode_rpc_signature_e2e_test | 5 |  |
-| kms | 41 |  |
+| kms | 48 |  |
 | leading_slash_key_test | 2 | ✅ |
 | lifecycle_regression_test | 4 |  |
 | list_buckets_auth_test | 1 | ✅ |
 | list_buckets_double_slash_test | 3 | ✅ |
 | list_buckets_iam_filter_test | 1 | ✅ |
 | list_object_versions_metadata_extension_test | 1 |  |
 | list_object_versions_regression_test | 2 | ✅ |
 | list_objects_duplicates_test | 3 | ✅ |
 | list_objects_v2_metadata_extension_test | 1 |  |
 | list_objects_v2_pagination_test | 12 | ✅ |
 | listing_regression_test | 4 |  |
 | mc_mirror_small_bucket_test | 1 |  |
 | multipart_auth_test | 75 |  |
 | multipart_storage_class_test | 3 | ✅ |
-| namespace_lock_quorum_test | 2 |  |
+| namespace_lock_quorum_test | 2 | 🌙 |
 | negative_sigv4_test | 6 | ✅ |
 | notification_startup_regression_test | 2 |  |
 | notification_webhook_test | 3 | ✅ |
-| object_lambda_test | 16 |  |
+| object_lambda_test | 16 | 🌙 |
-| object_lock | 33 |  |
+| object_lock | 34 |  |
 | overwrite_cleanup_regression_test | 1 |  |
 | presigned_negative_test | 7 | ✅ |
-| protocols | 16 |  |
+| protocols | 16 | 🌙 |
 | quota_test | 14 |  |
-| reliability_disk_fault_test | 3 |  |
+| reliability_disk_fault_test | 4 |  |
-| reliant | 24 | 18 ✅ |
+| reliant | 25 | 19 ✅ |
-| replication_extension_test | 50 | 20 ✅ +30 🌙 |
+| replication_extension_test | 75 | 20 ✅ +55 🌙 |
 | security_boundary_test | 4 |  |
 | ssec_copy_test | 2 | ✅ |
 | server_startup_failfast_test | 1 |  |
 | snowball_auto_extract_test | 6 |  |
 | special_chars_test | 14 | ✅ |
-| stale_multipart_cleanup_cluster_test | 1 |  |
+| ssec_copy_test | 2 | ✅ |
 | stale_multipart_cleanup_cluster_test | 1 | 🌙 |
 | storage_class_capability_test | 4 | ✅ |
-| sts_query_compat_test | 3 | ✅ |
+| sts_query_compat_test | 6 | ✅ |
 | tier_transition_regression_test | 3 |  |
 | tls_gen | 3 |  |
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
-**Total listed: 530 tests across 70 modules · PR smoke subset: 148 tests / 33 modules** (31 full modules + 18 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 30 tests** · updated 2026-08-09.
+**Total listed: 577 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 455 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-21.
@@ -48,11 +48,8 @@ fn materialize_case(path: String, prefix: Option<String>, flags: &[String]) -> (
        .fold((path, prefix), |(path, prefix), flag| apply_flag(path, prefix, flag))
 }
-fn has_dot_segments(path: &str) -> bool {
+fn has_parent_segments(path: &str) -> bool {
-    path.split(['/', '\\']).any(|segment| {
+    path.split(['/', '\\']).any(|segment| segment == "..")
        let trimmed = segment.trim();
        trimmed == "." || trimmed == ".."
    })
 }
 fuzz_target!(|data: &[u8]| {
@@ -66,8 +63,8 @@ fuzz_target!(|data: &[u8]| {
    if let Ok(key) = normalize_extract_entry_key(&path, prefix.as_deref(), is_dir) {
        assert!(
-            !has_dot_segments(&key),
+            !has_parent_segments(&key),
-            "accepted archive entry retained dot segments: path={:?} prefix={:?} key={:?}",
+            "accepted archive entry retained parent segments: path={:?} prefix={:?} key={:?}",
            path,
            prefix,
            key
@@ -69,7 +69,11 @@ fuzz_target!(|data: &[u8]| {
        && let Some(object) = value.as_object()
    {
        let mut legacy_doc = Map::new();
-        if let Some(policy) = object.get("Policy").or_else(|| object.get("policy")) {
+        if let Some(policy) = object
            .get("Policy")
            .or_else(|| object.get("policy"))
            .filter(|policy| serde_json::from_value::<Policy>((*policy).clone()).is_ok())
        {
            legacy_doc.insert("version".to_string(), json!(1));
            legacy_doc.insert("policy".to_string(), policy.clone());
            legacy_doc.insert("create_date".to_string(), json!("2025-03-07T12:00:00Z"));
@@ -0,0 +1,419 @@
 #!/usr/bin/env python3
 """Fail when committed tests silently fall out of their execution wiring."""
 from __future__ import annotations
 import hashlib
 import json
 import re
 import sys
 import tempfile
 import tomllib
 import unittest
 from unittest import mock
 from pathlib import Path
 ROOT = Path(__file__).resolve().parents[1]
 def words(value: str) -> set[str]:
    return {item.strip() for item in value.split(",") if item.strip()}
 def rust_code_only(source: str) -> str:
    """Blank Rust comments and literals while preserving byte positions."""
    def quoted_end(quote_index: int, delimiter: str) -> int:
        end = quote_index + 1
        while end < len(source):
            if source[end] == "\\":
                end += 2
            elif source[end] == delimiter:
                return end + 1
            else:
                end += 1
        return end
    code = list(source)
    index = 0
    while index < len(source):
        if source.startswith("//", index):
            end = source.find("\n", index)
            end = len(source) if end < 0 else end
        elif source.startswith("/*", index):
            depth = 1
            end = index + 2
            while end < len(source) and depth:
                if source.startswith("/*", end):
                    depth += 1
                    end += 2
                elif source.startswith("*/", end):
                    depth -= 1
                    end += 2
                else:
                    end += 1
        else:
            raw = re.match(r'(?:br|cr|r)(?P<hashes>#{0,})"', source[index:])
            if raw:
                marker = '"' + raw.group("hashes")
                end = source.find(marker, index + raw.end())
                end = len(source) if end < 0 else end + len(marker)
            elif source[index] == '"' or source.startswith(('b"', 'c"'), index):
                start_quote = index if source[index] == '"' else index + 1
                end = quoted_end(start_quote, '"')
            elif source[index] == "'" and index + 2 < len(source) and (
                source[index + 1] == "\\" or source[index + 2] == "'"
            ):
                end = quoted_end(index, "'")
            elif source.startswith("b'", index):
                end = quoted_end(index + 1, "'")
            else:
                index += 1
                continue
        for offset in range(index, end):
            if code[offset] != "\n":
                code[offset] = " "
        index = end
    return "".join(code)
 def declared(parent: Path, module: str) -> bool:
    source = parent.read_text()
    code = rust_code_only(source)
    pattern = re.compile(rf"^\s*(?:pub(?:\([^)]*\))?\s+)?mod\s+{re.escape(module)}\s*;", re.MULTILINE)
    allowed_preambles = {
        "#[cfg(test)]": "#[cfg(test)]",
        "#[cfg(all(test,target_os=))]": '#[cfg(all(test,target_os="linux"))]',
    }
    for match in pattern.finditer(code):
        prefix = code[: match.start()]
        depths = {"(": 0, "[": 0, "{": 0}
        pairs = {")": "(", "]": "[", "}": "{"}
        for char in prefix:
            if char in depths:
                depths[char] += 1
            elif char in pairs:
                depths[pairs[char]] -= 1
        if any(depths.values()):
            continue
        boundary = max(prefix.rfind(";"), prefix.rfind("{"), prefix.rfind("}"))
        code_preamble = re.sub(r"\s+", "", prefix[boundary + 1 :])
        if not code_preamble:
            return True
        if code_preamble in allowed_preambles:
            attr_start = prefix.rfind("#[cfg", boundary + 1)
            if attr_start >= 0 and re.sub(r"\s+", "", source[attr_start : match.start()]) == allowed_preambles[code_preamble]:
                return True
    return False
 def module_source(src: Path, directory: Path) -> Path | None:
    if not directory.parts:
        return src / "lib.rs"
    mod_file = src / directory / "mod.rs"
    if mod_file.is_file():
        return mod_file
    sibling = src.joinpath(*directory.parts[:-1], f"{directory.name}.rs")
    return sibling if sibling.is_file() else None
 def check_e2e_modules(root: Path) -> list[str]:
    src = root / "crates/e2e_test/src"
    errors: list[str] = []
    for test_file in sorted(src.rglob("*_test.rs")):
        relative = test_file.relative_to(root).as_posix()
        directory = test_file.relative_to(src).parent
        parent = module_source(src, directory)
        if parent is None:
            errors.append(f"{relative}: no canonical parent module")
            continue
        if not declared(parent, test_file.stem):
            errors.append(f"{relative}: not declared by {parent.relative_to(root).as_posix()}")
        while directory.parts:
            module = directory.name
            directory = directory.parent
            parent = module_source(src, directory)
            if parent is None:
                errors.append(f"{relative}: module {module} has no canonical parent")
                break
            if not declared(parent, module):
                errors.append(f"{relative}: module {module} not declared by {parent.relative_to(root).as_posix()}")
    return errors
 def check_fuzz_targets(root: Path) -> list[str]:
    manifest = tomllib.loads((root / "fuzz/Cargo.toml").read_text())
    expected = {item["name"] for item in manifest.get("bin", []) if "name" in item}
    errors: list[str] = []
    if not expected:
        return ["fuzz/Cargo.toml: no [[bin]] fuzz targets found"]
    runner = (root / "scripts/fuzz/run.sh").read_text()
    match = re.search(r'^targets="([^"]+)"', runner, re.MULTILINE)
    runner_targets = set(match.group(1).split()) if match else set()
    if runner_targets != expected:
        errors.append(f"scripts/fuzz/run.sh targets {sorted(runner_targets)} != manifest {sorted(expected)}")
    workflow = (root / ".github/workflows/fuzz.yml").read_text()
    matrices = [words(value) for value in re.findall(r"^\s*target:\s*\[([^]]+)]", workflow, re.MULTILINE)]
    if len(matrices) != 2:
        errors.append(f".github/workflows/fuzz.yml: expected smoke and nightly target matrices, found {len(matrices)}")
    for index, matrix in enumerate(matrices, start=1):
        if matrix != expected:
            errors.append(f".github/workflows/fuzz.yml matrix {index} {sorted(matrix)} != manifest {sorted(expected)}")
    runtime_targets = re.findall(r"^\s*FUZZ_TARGET:\s*(\S.*?)\s*$", workflow, re.MULTILINE)
    if runtime_targets != ["${{ matrix.target }}", "${{ matrix.target }}"]:
        errors.append(".github/workflows/fuzz.yml: smoke and nightly jobs must pass matrix.target to FUZZ_TARGET")
    dependency_paths = {
        f"{path.removeprefix('../')}/**"
        for dependency in manifest.get("dependencies", {}).values()
        if isinstance(dependency, dict)
        and isinstance(path := dependency.get("path"), str)
        and path.startswith("../crates/")
    }
    missing_paths = sorted(path for path in dependency_paths if f'"{path}"' not in workflow)
    if missing_paths:
        errors.append(f".github/workflows/fuzz.yml missing direct dependency paths: {', '.join(missing_paths)}")
    staged_matches = re.findall(r"^\s*for target in ([^;]+); do", workflow, re.MULTILINE)
    staged = set(staged_matches[0].split()) if staged_matches else set()
    if staged != expected:
        errors.append(f".github/workflows/fuzz.yml staged binaries {sorted(staged)} != manifest {sorted(expected)}")
    return errors
 def check_runner_selection(root: Path) -> list[str]:
    runner = (root / "scripts/run_e2e_tests.sh").read_text()
    errors: list[str] = []
    if "--include-ignored" not in runner:
        errors.append("scripts/run_e2e_tests.sh: runner must include default and ignored tests")
    if "--test-threads=1" not in runner:
        errors.append("scripts/run_e2e_tests.sh: runner must serialize fixed-port protocol tests")
    if re.search(r"(?<!include-)--ignored\b", runner):
        errors.append("scripts/run_e2e_tests.sh: bare --ignored silently excludes default tests")
    if "--exact" in runner:
        errors.append("scripts/run_e2e_tests.sh: --test is documented as a pattern and must not force exact matching")
    if 'eval "$test_cmd"' in runner:
        errors.append("scripts/run_e2e_tests.sh: command construction must not use eval")
    return errors
 def profile_selection(root: Path, profile: str) -> str:
    if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
        raise ValueError(f"invalid e2e profile name: {profile}")
    path = root / f".config/{profile}-selection.txt"
    lines = [line for line in path.read_text().splitlines() if line.strip()]
    values = dict(line.split("=", 1) for line in lines if "=" in line)
    if len(values) != len(lines) or any(not re.fullmatch(r"sha256(?:-[a-z0-9]+)?", key) for key in values):
        raise ValueError(f"{path.relative_to(root).as_posix()}: invalid sha256 entry")
    key = f"sha256-{sys.platform}"
    digest = values.get(key, values.get("sha256", ""))
    if not re.fullmatch(r"[0-9a-f]{64}", digest):
        raise ValueError(f"{path.relative_to(root).as_posix()}: missing sha256 for {sys.platform}")
    return digest
 def check_profile_definitions(root: Path) -> list[str]:
    config = tomllib.loads((root / ".config/nextest.toml").read_text())
    profiles = {
        profile
        for profile in config.get("profile", {})
        if profile.startswith("e2e-")
    }
    selection_profiles = {
        path.name.removesuffix("-selection.txt") for path in (root / ".config").glob("e2e-*-selection.txt")
    }
    errors: list[str] = []
    for profile in sorted(profiles | selection_profiles):
        if profile not in profiles:
            errors.append(f".config/nextest.toml: missing profile.{profile}")
        if profile not in selection_profiles:
            errors.append(f".config/{profile}-selection.txt: missing expected profile selection")
            continue
        try:
            profile_selection(root, profile)
        except (FileNotFoundError, ValueError) as error:
            errors.append(str(error))
    return errors
 def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
    try:
        expected_digest = profile_selection(root, profile)
        data = json.loads(listing.read_text())
        selected = sorted(
            f"{suite_id}::{test_name}"
            for suite_id, suite in data["rust-suites"].items()
            for test_name, testcase in suite["testcases"].items()
            if testcase.get("filter-match", {}).get("status") == "matches"
        )
        digest = hashlib.sha256(("\n".join(selected) + "\n").encode()).hexdigest()
    except (FileNotFoundError, KeyError, TypeError, ValueError, json.JSONDecodeError) as error:
        return [f"cannot read {profile} nextest listing: {error}"]
    if digest != expected_digest:
        return [
            f"{profile} selection changed: count={len(selected)} sha256={digest}; "
            f"expected sha256={expected_digest}"
        ]
    print(f"{profile} selection OK: {len(selected)} tests, sha256={digest}")
    return []
 def validate(root: Path) -> list[str]:
    errors: list[str] = []
    errors.extend(check_e2e_modules(root))
    errors.extend(check_fuzz_targets(root))
    errors.extend(check_runner_selection(root))
    errors.extend(check_profile_definitions(root))
    return errors
 class SelfTests(unittest.TestCase):
    def test_e2e_requires_registration(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            src = root / "crates/e2e_test/src"
            src.mkdir(parents=True)
            (src / "lib.rs").write_text("")
            test_file = src / "boundary_test.rs"
            test_file.write_text("#[test]\nfn boundary() {}\n")
            self.assertEqual(len(check_e2e_modules(root)), 1)
            (src / "lib.rs").write_text("mod boundary_test;\n")
            self.assertEqual(check_e2e_modules(root), [])
            nested = src / "protocols"
            nested.mkdir()
            (nested / "mod.rs").write_text("mod fixed_port_test;\n")
            (nested / "fixed_port_test.rs").write_text("#[test]\nfn fixed_port() {}\n")
            self.assertEqual(len(check_e2e_modules(root)), 1)
            (src / "lib.rs").write_text("mod boundary_test;\nmod protocols;\n")
            self.assertEqual(check_e2e_modules(root), [])
            (src / "lib.rs").write_text("#[cfg(any())]\nmod boundary_test;\nmod protocols;\n")
            self.assertEqual(len(check_e2e_modules(root)), 1)
            (src / "lib.rs").write_text(
                "#[cfg(any())]\n/// hidden module\nmod boundary_test;\n#[cfg_attr(test, cfg(any()))]\nmod protocols;\n"
            )
            self.assertEqual(len(check_e2e_modules(root)), 2)
            (src / "lib.rs").write_text(
                'const PHANTOM: &str = r#"{\nmod boundary_test;\n"#;\ndiscard! { mod protocols; }\n'
            )
            self.assertEqual(len(check_e2e_modules(root)), 2)
            (src / "lib.rs").write_text(
                '#[cfg(all(test, target_os = r"windows" /* target_os = "linux" */))]\n'
                "mod boundary_test;\nmod protocols;\n"
            )
            self.assertEqual(len(check_e2e_modules(root)), 1)
            (src / "lib.rs").write_text(
                '#[cfg(all(test, target_os = r"windows"))] // #[cfg(all(test, target_os = "linux"))]\n'
                "mod boundary_test;\nmod protocols;\n"
            )
            self.assertEqual(len(check_e2e_modules(root)), 1)
    def test_fuzz_runtime_uses_matrix_target(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "fuzz").mkdir()
            (root / "scripts/fuzz").mkdir(parents=True)
            (root / ".github/workflows").mkdir(parents=True)
            (root / "fuzz/Cargo.toml").write_text(
                'dep = { path = "../crates/dep" }\n[[bin]]\nname = "one"\n'
            )
            (root / "scripts/fuzz/run.sh").write_text('targets="one"\n')
            (root / ".github/workflows/fuzz.yml").write_text(
                'paths:\n  - "crates/dep/**"\n'
                "target: [one]\nFUZZ_TARGET: fixed\n"
                "target: [one]\nFUZZ_TARGET: ${{ matrix.target }}\n"
                "for target in one; do\n"
                "  fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/one\n"
            )
            self.assertEqual(len(check_fuzz_targets(root)), 1)
    def test_profile_listing_enforces_selection(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / ".config").mkdir()
            digest = hashlib.sha256(b"suite::two\n").hexdigest()
            (root / ".config/e2e-smoke-selection.txt").write_text(f"sha256={digest}\n")
            listing = root / "listing.json"
            listing.write_text(
                json.dumps(
                    {
                        "rust-suites": {
                            "suite": {
                                "testcases": {
                                    "one": {"filter-match": {"status": "matches"}},
                                    "two": {"filter-match": {"status": "mismatch"}},
                                }
                            }
                        }
                    }
                )
            )
            self.assertEqual(len(check_profile_listing(root, "e2e-smoke", listing)), 1)
    def test_profile_listing_binds_platform_digest(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / ".config").mkdir()
            darwin_digest = hashlib.sha256(b"suite::darwin\n").hexdigest()
            linux_digest = hashlib.sha256(b"suite::linux\n").hexdigest()
            (root / ".config/e2e-full-selection.txt").write_text(
                f"sha256-darwin={darwin_digest}\nsha256-linux={linux_digest}\n"
            )
            listing = root / "listing.json"
            listing.write_text(
                json.dumps(
                    {
                        "rust-suites": {
                            "suite": {
                                "testcases": {"darwin": {"filter-match": {"status": "matches"}}}
                            }
                        }
                    }
                )
            )
            with mock.patch.object(sys, "platform", "linux"):
                self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)
 def main() -> int:
    if sys.argv[1:] == ["--self-test"]:
        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
        return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
    if len(sys.argv) == 4 and sys.argv[1] == "--check-profile":
        errors = check_profile_listing(ROOT, sys.argv[2], Path(sys.argv[3]))
        if errors:
            for error in errors:
                print(f"ERROR: {error}", file=sys.stderr)
            return 1
        return 0
    if sys.argv[1:]:
        print(
            "usage: check_test_wiring.py [--self-test | --check-profile PROFILE LISTING]",
            file=sys.stderr,
        )
        return 2
    errors = validate(ROOT)
    if errors:
        for error in errors:
            print(f"ERROR: {error}", file=sys.stderr)
        return 1
    print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
    return 0
 if __name__ == "__main__":
    raise SystemExit(main())
@@ -20,6 +20,8 @@ DATA_DIR="$TARGET_DIR/rustfs_test_data"
 RUSTFS_PID=""
 TEST_FILTER=""
 TEST_TYPE="all"
 RUSTFS_BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-ftps,webdav,sftp}"
 export RUSTFS_BUILD_FEATURES
 # Function to print colored output
 print_info() {
@@ -92,7 +94,7 @@ build_rustfs() {
    print_info "Building RustFS..."
    cd "$PROJECT_ROOT"
-    if ! cargo build --bin rustfs; then
+    if ! cargo build --bin rustfs --features "$RUSTFS_BUILD_FEATURES"; then
        print_error "Failed to build RustFS"
        exit 1
    fi
@@ -220,26 +222,27 @@ run_tests() {
    print_info "Running e2e tests..."
    cd "$PROJECT_ROOT"
-    local test_cmd="cargo test --package e2e_test --lib"
+    local test_cmd=(cargo test --package e2e_test --lib)
    case "$TEST_TYPE" in
        "specific")
-            test_cmd="$test_cmd -- $TEST_FILTER --exact --show-output --ignored"
+            test_cmd+=(-- "$TEST_FILTER")
            print_info "Running specific test: $TEST_FILTER"
            ;;
        "file")
-            test_cmd="$test_cmd -- $TEST_FILTER --show-output --ignored"
+            test_cmd+=(-- "$TEST_FILTER")
            print_info "Running tests in file/module: $TEST_FILTER"
            ;;
        "all")
-            test_cmd="$test_cmd -- --show-output --ignored"
+            test_cmd+=(--)
            print_info "Running all e2e tests"
            ;;
    esac
    test_cmd+=(--show-output --include-ignored --test-threads=1)
-    print_info "Test command: $test_cmd"
+    print_info "Test command: ${test_cmd[*]}"
-    if eval "$test_cmd"; then
+    if "${test_cmd[@]}"; then
        print_success "All tests passed!"
        return 0
    else
@@ -253,6 +253,9 @@ Test results are saved in the `artifacts/s3tests-${TEST_MODE}/` directory (defau
 - `junit.xml`: Test results in JUnit format (compatible with CI/CD systems)
 - `pytest.log`: Detailed pytest logs with full test output
 - `all-collected-nodeids.txt`: Exact node IDs in the pinned upstream suite
 - `selected-nodeids.txt`: Exact node IDs expected in this run
 - `unsharded-selected-nodeids.txt`: Exact node IDs before deterministic sharding
 - `compat-report.md`: Classification report generated by `report_compat.py` —
  regressions against `implemented_tests.txt`, promotion candidates (tests
  that pass but are still listed as unimplemented/excluded), and tests missing
@@ -449,9 +452,11 @@ RustFS. Two GitHub Actions workflows delegate to it:
 - **Full sweep** (`.github/workflows/e2e-s3tests.yml`): weekly scheduled (and
  manually dispatchable) run of the ENTIRE upstream suite (`TEST_SCOPE=all`)
  against a Docker deployment — single node or a real 4-node distributed
-  cluster behind HAProxy. The sweep fails only on regressions in the
+  cluster behind HAProxy. Regressions, unclassified tests, incomplete
-  implemented whitelist; everything else is reported by `report_compat.py`
+  execution, and infrastructure errors fail the sweep; classified unsupported
-  as promotion candidates or unclassified tests.
+  behavior remains informational. Scheduled topology runs are split into four
  deterministic exact-node-ID shards, and every case has a five-minute timeout,
  so one stalled case cannot erase the entire sweep's evidence.
 Keeping both workflows on this script means local runs, the PR gate, and the
 scheduled sweep always execute tests the same way (same pinned s3-tests
@@ -466,8 +471,9 @@ pass/fail table in the job summary.
 ## Companion Tools
 - `report_compat.py` — diffs a junit.xml result against the classification
-  lists; run automatically at the end of `run.sh`, and used by the weekly
+  lists and the exact pytest collection; run before execution to reject stale
-  sweep to gate on whitelist regressions only (`--fail-on-regression`).
+  or missing classifications, then after execution to detect regressions and
  incomplete parameterized cases.
 - `api_coverage.py` — quantifies S3 API surface coverage by comparing the
  s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
  RustFS overrides in `impl S3 for FS`:
@@ -307,3 +307,10 @@ test_object_acl_write
 test_object_acl_writeacp
 test_put_bucket_acl_grant_group_read
 test_object_raw_get_bucket_acl
 # Require upstream cloud-storage or IAM account services
 test_bucket_logging_requester_assumed_role
 test_lifecycle_cloud_transition_target_by_bucket
 test_lifecycle_cloud_transition_target_by_bucket_multiple_buckets
 test_list_object_versions_restore_status
 test_list_objects_restore_status
@@ -521,9 +521,13 @@ test_atomic_dual_conditional_write_1mb
 test_atomic_write_bucket_gone
 test_bucket_acl_canned_private_to_private
 test_bucket_concurrent_set_canned_acl
 test_bucket_create_delete
 test_bucket_policy
 test_bucket_policy_acl
 test_bucket_policy_put_obj_acl
 test_bucketv2_policy_acl
 test_copy_enc
 test_copy_part_enc
 test_copy_object_ifmatch_failed
 test_copy_object_ifnonematch_good
 test_cors_presigned_put_object_tenant_with_acl
@@ -21,14 +21,17 @@ Classifies every executed test into:
  - unclassified passes:    passed but not present in any list (new upstream tests)
  - unclassified failures:  failed and not present in any list (new upstream tests)
-Writes a markdown report and prints a summary to stdout. Exit code is 0 unless
+Writes a markdown report and prints a summary to stdout. Optional gates reject
--fail-on-regression is given and at least one regression was found.
+regressions, unclassified tests, stale classifications, and incomplete node-ID
 execution.
 """
 from __future__ import annotations
 import argparse
 from collections import Counter
 import pathlib
 import re
 import sys
 import xml.etree.ElementTree as ET
@@ -45,33 +48,47 @@ LIST_FILES = {
 }
-def load_list(path: pathlib.Path) -> set[str]:
+def load_entries(path: pathlib.Path) -> list[str]:
-    names: set[str] = set()
+    names: list[str] = []
    if not path.is_file():
        return names
    for line in path.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if line and not line.startswith("#"):
-            names.add(line)
+            names.append(line)
    return names
 def classification_errors(entries: dict[str, list[str]]) -> list[str]:
    errors: list[str] = []
    lists = {key: set(names) for key, names in entries.items()}
    for key, names in entries.items():
        duplicates = sorted(name for name, count in Counter(names).items() if count > 1)
        if duplicates:
            errors.append(f"{LIST_FILES[key]} has duplicates: {', '.join(duplicates)}")
    keys = tuple(lists)
    for index, left in enumerate(keys):
        for right in keys[index + 1 :]:
            overlap = sorted(lists[left] & lists[right])
            if overlap:
                errors.append(f"{left}/{right} classifications overlap: {', '.join(overlap)}")
    return errors
 def base_name(testcase_name: str) -> str:
    """Strip pytest parametrization (test_foo[param]) to match list entries."""
    return testcase_name.split("[", 1)[0]
-def parse_junit(path: pathlib.Path) -> dict[str, str]:
+def parse_junit(path: pathlib.Path) -> tuple[dict[str, str], list[str], list[tuple[str, str, str, str]]]:
-    """Return {test name: status} with status in passed/failed/error/skipped.
+    """Return exact statuses, pytest-timeout cases, and failure summaries."""
    Parametrized cases collapse onto their base name; any failing variant marks
    the whole test failed.
    """
    results: dict[str, str] = {}
    timed_out: list[str] = []
    failures: list[tuple[str, str, str, str]] = []
    severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
    root = ET.parse(path).getroot()
    for case in root.iter("testcase"):
-        name = base_name(case.get("name", ""))
+        name = case.get("name", "")
        if not name:
            continue
        if case.find("failure") is not None:
@@ -85,7 +102,35 @@ def parse_junit(path: pathlib.Path) -> dict[str, str]:
        prev = results.get(name)
        if prev is None or severity[status] > severity[prev]:
            results[name] = status
-    return results
+        node = case.find("failure") if status == "failed" else case.find("error")
        if node is not None:
            details = " ".join(filter(None, [node.get("message", ""), node.text or ""]))
            message = node.get("message") or next(iter((node.text or "").strip().splitlines()), "")
            failures.append((case.get("classname", ""), name, case.get("time", "0"), message))
            if re.search(r"\bTimeout\s*(?:>|\()", details, re.IGNORECASE):
                timed_out.append(name)
    return results, timed_out, failures
 def collapse_results(results: dict[str, str]) -> dict[str, str]:
    """Collapse parametrized cases for classification-level reporting."""
    collapsed: dict[str, str] = {}
    severity = {"skipped": 0, "passed": 1, "failed": 2, "error": 2}
    for exact_name, status in results.items():
        name = base_name(exact_name)
        previous = collapsed.get(name)
        if previous is None or severity[status] > severity[previous]:
            collapsed[name] = status
    return collapsed
 def load_collected_nodeids(path: pathlib.Path) -> set[str]:
    names: set[str] = set()
    for line in path.read_text(encoding="utf-8").splitlines():
        nodeid = line.strip()
        if nodeid:
            names.add(nodeid.rsplit("::", 1)[-1])
    return names
 def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:
@@ -102,7 +147,7 @@ def render_section(title: str, rows: list[str], hint: str = "") -> list[str]:
 def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--junit", required=True, type=pathlib.Path, help="junit.xml produced by pytest")
+    parser.add_argument("--junit", type=pathlib.Path, help="junit.xml produced by pytest")
    parser.add_argument(
        "--lists-dir",
        type=pathlib.Path,
@@ -115,14 +160,60 @@ def main() -> int:
        action="store_true",
        help="exit non-zero when a test from implemented_tests.txt failed",
    )
    parser.add_argument(
        "--fail-on-unclassified",
        action="store_true",
        help="exit non-zero when an executed test is absent from every classification",
    )
    parser.add_argument(
        "--collected-nodeids",
        type=pathlib.Path,
        help="exact pytest node IDs from the pinned suite's collect-only pass",
    )
    parser.add_argument(
        "--check-classifications-only",
        action="store_true",
        help="validate classification names against collected node IDs without reading JUnit",
    )
    args = parser.parse_args()
-    if not args.junit.is_file():
+    entries = {key: load_entries(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
    lists = {key: set(names) for key, names in entries.items()}
    invalid_classifications = classification_errors(entries)
    collected: set[str] = set()
    if args.collected_nodeids:
        collected = load_collected_nodeids(args.collected_nodeids)
        collected_base = {base_name(name) for name in collected}
        classified = set().union(*lists.values())
        missing_classifications = sorted(collected_base - classified)
        stale_classifications = sorted(classified - collected_base)
    else:
        missing_classifications = []
        stale_classifications = []
    if args.check_classifications_only:
        if not args.collected_nodeids:
            parser.error("--check-classifications-only requires --collected-nodeids")
        for error in invalid_classifications:
            print(f"[INVALID] {error}")
        for name in missing_classifications:
            print(f"[UNCLASSIFIED] {name}")
        for name in stale_classifications:
            print(f"[STALE] {name}")
        return 1 if invalid_classifications or missing_classifications or stale_classifications else 0
    if invalid_classifications:
        for error in invalid_classifications:
            print(f"[ERROR] {error}", file=sys.stderr)
        return 2
    if not args.junit or not args.junit.is_file():
        print(f"[ERROR] junit file not found: {args.junit}", file=sys.stderr)
        return 2
-    lists = {key: load_list(args.lists_dir / fname) for key, fname in LIST_FILES.items()}
+    exact_results, timed_out, failures = parse_junit(args.junit)
-    results = parse_junit(args.junit)
+    results = collapse_results(exact_results)
    missing_results = sorted(collected - exact_results.keys()) if collected else []
    regressions: list[str] = []
    promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
@@ -155,7 +246,9 @@ def main() -> int:
    lines = [
        "# S3 compatibility report",
        "",
-        f"Executed: {len(results)} tests — "
+        f"Executed: {len(exact_results)} exact cases across {len(results)} classified tests.",
        "",
        "Classification status — "
        f"{counts['passed']} passed, {counts['failed']} failed, "
        f"{counts['error']} errored, {counts['skipped']} skipped.",
        "",
@@ -191,6 +284,16 @@ def main() -> int:
        unclassified_failed,
        "Failing and absent from every list — triage into `unimplemented_tests.txt` or `excluded_tests.txt`.",
    )
    lines += render_section(
        "Missing results",
        missing_results,
        "Present in the pinned upstream suite but absent from JUnit — the sweep was incomplete.",
    )
    lines += render_section(
        "Timed out",
        timed_out,
        "Per-test timeout is an infrastructure failure regardless of compatibility classification.",
    )
    report = "\n".join(lines)
    if args.output:
@@ -201,13 +304,28 @@ def main() -> int:
    print(
        f"[INFO] {len(regressions)} regression(s), "
        f"{len(promotions['unimplemented']) + len(promotions['excluded']) + len(unclassified_passed)} promotion candidate(s), "
-        f"{len(unclassified_failed)} unclassified failure(s)"
+        f"{len(unclassified_failed)} unclassified failure(s), "
        f"{len(missing_results)} missing result(s), "
        f"{len(timed_out)} timeout(s)"
    )
    for name in sorted(regressions):
        print(f"[REGRESSION] {name}")
    if failures:
        print("[ERROR] s3-tests failed testcase summary:")
        for classname, name, duration, message in failures[:20]:
            nodeid = f"{classname}::{name}" if classname else name
            print(f"[ERROR] - {nodeid} ({duration}s): {message}")
        if len(failures) > 20:
            print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
    if args.fail_on_regression and regressions:
        return 1
    if args.fail_on_unclassified and (unclassified_passed or unclassified_failed):
        return 1
    if args.collected_nodeids and missing_results:
        return 1
    if timed_out:
        return 1
    return 0
@@ -58,6 +58,19 @@ if [[ "${TEST_SCOPE}" != "implemented" && "${TEST_SCOPE}" != "all" ]]; then
    echo "[ERROR] Invalid TEST_SCOPE: ${TEST_SCOPE} (must be \"implemented\" or \"all\")" >&2
    exit 1
 fi
 S3_SHARD_COUNT="${S3_SHARD_COUNT:-1}"
 S3_SHARD_INDEX="${S3_SHARD_INDEX:-0}"
 TEST_TIMEOUT="${TEST_TIMEOUT:-300}"
 if [[ ! "${S3_SHARD_COUNT}" =~ ^[1-9][0-9]*$ ]] \
    || [[ ! "${S3_SHARD_INDEX}" =~ ^[0-9]+$ ]] \
    || (( S3_SHARD_INDEX >= S3_SHARD_COUNT )); then
    echo "[ERROR] Invalid S3 shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}" >&2
    exit 1
 fi
 if [[ ! "${TEST_TIMEOUT}" =~ ^[1-9][0-9]*$ ]]; then
    echo "[ERROR] Invalid TEST_TIMEOUT: ${TEST_TIMEOUT}" >&2
    exit 1
 fi
 # Upstream ceph/s3-tests suite, pinned for reproducible runs.
 # Bump S3TESTS_REV deliberately: upstream changes can rename tests or change
@@ -96,55 +109,6 @@ log_error() {
    echo -e "${RED}[ERROR]${NC} $*"
 }
 summarize_junit_failures() {
    local junit_path="$1"
    if [ ! -f "${junit_path}" ]; then
        log_warn "JUnit report not found: ${junit_path}"
        return 0
    fi
    python3 - "${junit_path}" <<'PY'
 import sys
 import xml.etree.ElementTree as ET
 junit_path = sys.argv[1]
 try:
    root = ET.parse(junit_path).getroot()
 except Exception as exc:
    print(f"[WARN] Failed to parse JUnit report {junit_path}: {exc}")
    raise SystemExit(0)
 failures = []
 for case in root.iter("testcase"):
    failure = case.find("failure")
    error = case.find("error")
    node = failure if failure is not None else error
    if node is None:
        continue
    classname = case.attrib.get("classname", "")
    name = case.attrib.get("name", "")
    duration = case.attrib.get("time", "0")
    message = node.attrib.get("message") or (node.text or "").strip().splitlines()[0:1]
    if isinstance(message, list):
        message = message[0] if message else ""
    failures.append((classname, name, duration, message))
 if not failures:
    print("[INFO] No failed testcases found in JUnit report")
    raise SystemExit(0)
 print("[ERROR] s3-tests failed testcase summary:")
 for classname, name, duration, message in failures[:20]:
    nodeid = f"{classname}::{name}" if classname else name
    print(f"[ERROR] - {nodeid} ({duration}s): {message}")
 if len(failures) > 20:
    print(f"[ERROR] - ... {len(failures) - 20} additional failed testcases omitted")
 PY
 }
 # =============================================================================
 # Test Classification Files
 # =============================================================================
@@ -322,6 +286,9 @@ Environment Variables:
  MAXFAIL                - Stop after N failures, 0 = never stop (default: 1)
  XDIST                  - Enable parallel execution with N workers (default: 0)
  TEST_SCOPE             - "implemented" (whitelist, default) or "all" (entire upstream suite)
  S3_SHARD_COUNT         - Number of deterministic exact-node-ID shards (default: 1)
  S3_SHARD_INDEX         - Zero-based shard index (default: 0)
  TEST_TIMEOUT           - Per-test timeout in seconds (default: 300)
  S3TESTS_REPO           - s3-tests repository URL (default: https://github.com/ceph/s3-tests.git)
  S3TESTS_REV            - Pinned s3-tests commit; bump deliberately and reclassify test lists
  MARKEXPR               - pytest marker expression (default: no marker filtering)
@@ -982,9 +949,10 @@ mkdir -p "${ARTIFACTS_DIR}"
 XDIST_ARGS=""
 if [ "${XDIST}" != "0" ]; then
    # Add pytest-xdist to requirements.txt so tox installs it inside its virtualenv
-    echo "pytest-xdist" >> requirements.txt
+    grep -qxF "pytest-xdist" requirements.txt || echo "pytest-xdist" >> requirements.txt
    XDIST_ARGS="-n ${XDIST} --dist=loadgroup"
 fi
 grep -qxF "pytest-timeout" requirements.txt || echo "pytest-timeout" >> requirements.txt
 # Resolve config path (absolute path for tox)
 if [[ "${S3TESTS_CONF}" = /* ]]; then
@@ -1003,12 +971,69 @@ else
    PYTEST_SELECTION_ARGS=("${S3_TEST_FILE}")
 fi
 collect_nodeids() {
    local output_path="$1"
    shift
    local collect_log="${output_path%.txt}.log"
    local collect_rc=0
    local node_prefix="${S3_TEST_FILE//./\\.}::"
    set +e
    S3TEST_CONF="${CONF_OUTPUT_PATH}" tox -- -q --collect-only "$@" 2>&1 | tee "${collect_log}"
    collect_rc=${PIPESTATUS[0]}
    set -e
    if [ "${collect_rc}" -ne 0 ]; then
        log_error "pytest collection failed with exit code ${collect_rc}"
        return "${collect_rc}"
    fi
    grep -E "^${node_prefix}" "${collect_log}" > "${output_path}" || true
    if [ ! -s "${output_path}" ]; then
        log_error "pytest collection produced no S3 test node IDs"
        return 1
    fi
 }
 ALL_COLLECTED_NODEIDS="${ARTIFACTS_DIR}/all-collected-nodeids.txt"
 UNSHARDED_SELECTED_NODEIDS="${ARTIFACTS_DIR}/unsharded-selected-nodeids.txt"
 SELECTED_NODEIDS="${ARTIFACTS_DIR}/selected-nodeids.txt"
 collect_nodeids "${ALL_COLLECTED_NODEIDS}" "${S3_TEST_FILE}" -m "not rustfs_never_marker"
 python3 "${SCRIPT_DIR}/report_compat.py" \
    --lists-dir "${SCRIPT_DIR}" \
    --collected-nodeids "${ALL_COLLECTED_NODEIDS}" \
    --check-classifications-only || {
    log_error "S3 test classifications do not match pinned revision ${S3TESTS_REV}"
    exit 1
 }
 if [[ "${TEST_SCOPE}" == "all" && -z "${TESTEXPR}" && "${MARKEXPR}" == "not rustfs_never_marker" ]]; then
    cp "${ALL_COLLECTED_NODEIDS}" "${UNSHARDED_SELECTED_NODEIDS}"
 else
    collect_nodeids "${UNSHARDED_SELECTED_NODEIDS}" "${PYTEST_SELECTION_ARGS[@]}" -m "${MARKEXPR}"
 fi
 if (( S3_SHARD_COUNT > 1 )); then
    awk -v count="${S3_SHARD_COUNT}" -v shard_index="${S3_SHARD_INDEX}" \
        '((NR - 1) % count) == shard_index' \
        "${UNSHARDED_SELECTED_NODEIDS}" > "${SELECTED_NODEIDS}"
    if [[ ! -s "${SELECTED_NODEIDS}" ]]; then
        log_error "Shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT} selected no tests"
        exit 1
    fi
    PYTEST_SELECTION_ARGS=()
    while IFS= read -r nodeid; do
        PYTEST_SELECTION_ARGS+=("${nodeid}")
    done < "${SELECTED_NODEIDS}"
    log_info "Selected shard ${S3_SHARD_INDEX}/${S3_SHARD_COUNT}: ${#PYTEST_SELECTION_ARGS[@]} exact cases"
 else
    cp "${UNSHARDED_SELECTED_NODEIDS}" "${SELECTED_NODEIDS}"
 fi
 # Run tests from s3tests/functional
 set +e
 S3TEST_CONF="${CONF_OUTPUT_PATH}" \
    tox -- \
    -vv -ra --showlocals --tb=long \
    --maxfail="${MAXFAIL}" \
    --timeout="${TEST_TIMEOUT}" \
    --junitxml="${ARTIFACTS_DIR}/junit.xml" \
    ${XDIST_ARGS} \
    "${PYTEST_SELECTION_ARGS[@]}" \
@@ -1033,19 +1058,22 @@ elif [ "${DEPLOY_MODE}" = "existing" ]; then
    echo "{\"host\": \"${S3_HOST}\", \"port\": ${S3_PORT}, \"mode\": \"existing\"}" > "${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/inspect.json" || true
 fi
-# Step 11: Classification report (informational, never fails the run)
+# Step 11: Classification report and gate
 REPORT_SCRIPT="${SCRIPT_DIR}/report_compat.py"
-if [ -f "${REPORT_SCRIPT}" ] && [ -f "${ARTIFACTS_DIR}/junit.xml" ]; then
+REPORT_ARGS=(
-    python3 "${REPORT_SCRIPT}" \
+    --junit "${ARTIFACTS_DIR}/junit.xml"
-        --junit "${ARTIFACTS_DIR}/junit.xml" \
+    --lists-dir "${SCRIPT_DIR}"
-        --lists-dir "${SCRIPT_DIR}" \
+    --collected-nodeids "${SELECTED_NODEIDS}"
-        --output "${ARTIFACTS_DIR}/compat-report.md" \
+    --output "${ARTIFACTS_DIR}/compat-report.md"
-        || log_warn "Compatibility report generation failed"
+    --fail-on-regression
-fi
+)
-
+if [[ "${TEST_SCOPE}" == "all" ]]; then
-if [ ${TEST_EXIT_CODE} -ne 0 ]; then
+    REPORT_ARGS+=(--fail-on-unclassified)
    summarize_junit_failures "${ARTIFACTS_DIR}/junit.xml"
 fi
 set +e
 python3 "${REPORT_SCRIPT}" "${REPORT_ARGS[@]}"
 REPORT_EXIT_CODE=$?
 set -e
 # Summary
 if [ ${TEST_EXIT_CODE} -eq 0 ]; then
@@ -1059,4 +1087,10 @@ else
    log_info "Check RustFS logs: ${ARTIFACTS_DIR}/rustfs-${TEST_MODE}/rustfs.log"
 fi
-exit ${TEST_EXIT_CODE}
+if [[ "${TEST_EXIT_CODE}" -ne 0 && "${TEST_EXIT_CODE}" -ne 1 ]]; then
    exit "${TEST_EXIT_CODE}"
 fi
 if [[ "${TEST_SCOPE}" == "implemented" && "${TEST_EXIT_CODE}" -ne 0 ]]; then
    exit "${TEST_EXIT_CODE}"
 fi
 exit "${REPORT_EXIT_CODE}"
@@ -0,0 +1,119 @@
 #!/usr/bin/env python3
 """Regression tests for the S3 compatibility report."""
 from __future__ import annotations
 import importlib.util
 import subprocess
 import sys
 import tempfile
 import unittest
 from pathlib import Path
 REPORT_PATH = Path(__file__).with_name("report_compat.py")
 SPEC = importlib.util.spec_from_file_location("report_compat", REPORT_PATH)
 assert SPEC and SPEC.loader
 REPORT = importlib.util.module_from_spec(SPEC)
 SPEC.loader.exec_module(REPORT)
 class ReportCompatTests(unittest.TestCase):
    def test_upstream_names_expose_incomplete_junit(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            directory = Path(tmp)
            collected = directory / "collected.txt"
            collected.write_text("s3tests/functional/test_s3.py::test_one[a]\ns3tests/functional/test_s3.py::test_one[b]\n")
            junit = directory / "junit.xml"
            junit.write_text('<testsuite><testcase name="test_one[a]" /></testsuite>')
            expected = REPORT.load_collected_nodeids(collected)
            results, _, _ = REPORT.parse_junit(junit)
            self.assertEqual(expected - results.keys(), {"test_one[b]"})
    def test_cli_fails_an_incomplete_sweep(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            directory = Path(tmp)
            collected = directory / "collected.txt"
            collected.write_text("s3tests/functional/test_s3.py::test_one\ns3tests/functional/test_s3.py::test_two\n")
            junit = directory / "junit.xml"
            junit.write_text('<testsuite><testcase name="test_one" /></testsuite>')
            for filename in REPORT.LIST_FILES.values():
                (directory / filename).write_text("")
            (directory / "implemented_tests.txt").write_text("test_one\n")
            result = subprocess.run(
                [
                    sys.executable,
                    str(REPORT_PATH),
                    "--junit",
                    str(junit),
                    "--lists-dir",
                    str(directory),
                    "--collected-nodeids",
                    str(collected),
                    "--fail-on-regression",
                    "--fail-on-unclassified",
                ],
                check=False,
                capture_output=True,
                text=True,
            )
            self.assertEqual(result.returncode, 1)
            self.assertIn("1 missing result(s)", result.stdout)
    def test_preflight_rejects_missing_and_stale_classifications(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            directory = Path(tmp)
            collected = directory / "collected.txt"
            collected.write_text("s3tests/functional/test_s3.py::test_known[a]\ntest_new\n")
            for filename in REPORT.LIST_FILES.values():
                (directory / filename).write_text("")
            (directory / "implemented_tests.txt").write_text("test_known\ntest_stale\ntest_stale\n")
            result = subprocess.run(
                [
                    sys.executable,
                    str(REPORT_PATH),
                    "--lists-dir",
                    str(directory),
                    "--collected-nodeids",
                    str(collected),
                    "--check-classifications-only",
                ],
                check=False,
                capture_output=True,
                text=True,
            )
            self.assertEqual(result.returncode, 1)
            self.assertIn("[UNCLASSIFIED] test_new", result.stdout)
            self.assertIn("[STALE] test_stale", result.stdout)
            self.assertIn("[INVALID] implemented_tests.txt has duplicates: test_stale", result.stdout)
    def test_timeout_fails_even_when_test_is_excluded(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            directory = Path(tmp)
            junit = directory / "junit.xml"
            junit.write_text(
                '<testsuite><testcase name="test_slow"><failure message="Failed: Timeout (&gt;300.0s)" /></testcase></testsuite>'
            )
            for filename in REPORT.LIST_FILES.values():
                (directory / filename).write_text("")
            (directory / "excluded_tests.txt").write_text("test_slow\n")
            result = subprocess.run(
                [sys.executable, str(REPORT_PATH), "--junit", str(junit), "--lists-dir", str(directory)],
                check=False,
                capture_output=True,
                text=True,
            )
            self.assertEqual(result.returncode, 1)
            self.assertIn("1 timeout(s)", result.stdout)
 if __name__ == "__main__":
    unittest.main()
@@ -11,8 +11,12 @@
 # Failed tests
 test_bucket_create_delete_bucket_ownership
 test_bucket_logging_request_id
 test_create_bucket_no_ownership_controls
 test_bucket_logging_owner
 test_head_object_404_with_policy_prefix
 test_multipart_reupload_checksum_and_etag
 test_multipart_upload_complete_without_create
 test_object_copy_not_owned_bucket
 test_bucket_policy_multipart
 test_post_object_upload_checksum
		`@@ -0,0 +1,2 @@`
							`sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b`
							`sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba`
		`@@ -0,0 +1 @@`
							`sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680`
		`@@ -0,0 +1,2 @@`
							`sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315`
							`sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17`
		`@@ -0,0 +1 @@`
							`sha256=655a3f3c1d042e694339d15caba7580518320322d1bac0f09450b37e6c09e2e7`
		`@@ -0,0 +1 @@`
							`sha256=ec27cde6ce6400723c4b372bfbd2ac61709c744294e4810af765e8a808d8e31d`