Merge branch 'test/distributed-e2e-hardening' into cursor/e2e-pool-tmpfs-fallback-f29d

test(e2e): prove operations overlap data movement
Merge branch 'main' into test/distributed-e2e-hardening
2026-09-05 19:55:37 +00:00 · 2026-09-05 23:56:21 +08:00 · 2026-09-05 22:50:00 +08:00 · 2026-09-05 22:21:31 +08:00 · 2026-09-05 12:58:22 +00:00 · 2026-09-05 11:57:33 +08:00
60 changed files with 4767 additions and 3277 deletions
@@ -0,0 +1,2 @@
+sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
+sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
@@ -31,7 +31,6 @@ script-tests: ## Run shell script tests
 	./scripts/test_object_batch_bench_enhanced.sh
 	./scripts/test_hotpath_warp_ab_gate.sh
 	./scripts/test_hotpath_warp_abba.sh
-	./scripts/test_scanner_validation_harness.sh
 	./scripts/test_exact_1mib_handoff_abba.sh
 	./scripts/test_pinned_paired_abba_bench.sh
 	./scripts/test_manual_transition_runbooks.sh
@@ -42,7 +41,6 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
@@ -183,6 +183,13 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'

+# 4-node 4-drive distributed Actions suite: each case starts four rustfs
+# processes and up to sixteen data directories. Serialize across nextest's
+# process boundary so several 4x4 clusters never overlap.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^distributed::/)'
+test-group = 'e2e-cluster-nightly'
+
 # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
 # does not cross nextest process boundaries, so keep every Vault-backed test in
 # one group.
@@ -526,6 +533,27 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'

+# ---------------------------------------------------------------------------
+# e2e-distributed profile — 4-node 4-disk Actions suite
+# ---------------------------------------------------------------------------
+# Storage-sensitive PR / nightly / dispatch lane owned by
+# .github/workflows/e2e-distributed.yml.
+# Each case starts four rustfs processes (and for site replication, two
+# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
+# Serialized via e2e-cluster-nightly with no retries.
+[profile.e2e-distributed]
+default-filter = 'package(e2e_test) & test(/^distributed::/)'
+fail-fast = false
+# Decommission / rebalance cases poll for up to 180s with little stdout.
+slow-timeout = { period = "120s", terminate-after = 6 }
+
+[profile.e2e-distributed.junit]
+path = "junit.xml"
+
+[[profile.e2e-distributed.overrides]]
+filter = 'package(e2e_test)'
+test-group = 'e2e-cluster-nightly'
+
 # ---------------------------------------------------------------------------
 # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
 # ---------------------------------------------------------------------------
@@ -586,6 +614,10 @@ path = "junit.xml"
 #     cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
 #     excluded here because backlog#2213 promotes core heal rebuild coverage to
 #     this merge/main lane while retaining nightly coverage.
+#   * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
+#     replication, quota, observability, expand/decommission/rebalance, site
+#     replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
+#     .github/workflows/e2e-distributed.yml.
 #   * on_demand_migration::interop_test — the ODM-20 provider interoperability
 #     cases, which are meaningless without a source: they run in the dedicated
 #     [profile.e2e-odm-interop] lane below, where the workflow points them at a
@@ -607,6 +639,7 @@ default-filter = """
    package(e2e_test)
    & !test(/^protocols::/)
    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^distributed::/)
    & !test(/^replication_extension_test::/)
    & !test(/^replication_target_matrix_test::/)
    & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
@@ -1,40 +0,0 @@
-{
-  "schema": 1,
-  "cases": {
-    "background-target-restart": {
-      "gate": "G14",
-      "task": "W21",
-      "lane": "e2e-nightly",
-      "suite": "e2e_test",
-      "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
-      "oracle": "background-target-restart.json",
-      "min_objects": 9,
-      "max_objects": 65,
-      "topology": {"nodes": 4, "drives_per_node": 1},
-      "scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
-    }
-  },
-  "release_pending": {
-    "G01": "W02/W04 complete root and quota authority coverage",
-    "G02": "W03 bounded checkpoint progress and independent version inventory",
-    "G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
-    "G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
-    "G05": "W06/W07 per-object outcomes and bounded terminal retention",
-    "G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
-    "G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
-    "G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
-    "G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
-    "G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
-    "G11": "W04/W19/W24 maintenance and complete producer coverage",
-    "G12": "W02/W15/W16 both quota paths during reset and settlement",
-    "G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
-    "G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
-    "P1": "W20 measured cold-walk share and foreground latency/throughput",
-    "P2": "W20/W24 measured post-stop convergence and cold segment reuse",
-    "P3": "W20 measured two-hour pressure/heal capacity and recovery window",
-    "P4": "W20 measured MRF scale and replay cost with retained responsibility",
-    "R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
-    "R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
-    "R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
-  }
-}
@@ -94,17 +94,12 @@ runs:
      shell: bash
      run: ./scripts/check_embedded_secrets.sh

-    - name: Run script contract tests
-      shell: bash
-      run: make script-tests
-
    - name: Check test wiring
      shell: bash
      run: |
        python3 ./scripts/check_test_wiring.py --self-test
        python3 ./scripts/check_scheduled_validation_freshness.py --self-test
        python3 ./scripts/test_security_workflow.py
-        python3 ./scripts/test_nightly_candidate.py
        python3 ./scripts/check_test_wiring.py

    - name: Check no planning docs committed
@@ -4,6 +4,11 @@
  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  {
+    "workflow": ".github/workflows/e2e-distributed.yml",
+    "max_age_hours": 36,
+    "never_ran_grace_until": "2026-09-18T00:00:00Z"
+  },
  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
@@ -0,0 +1,206 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 4-node 4-disk distributed e2e lane.
+#
+# Each selected test starts a real localhost cluster via
+# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
+# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
+#
+# Isolated pool filesystems: expand/decommission/rebalance cases require
+# independent `statfs` capacity. `sm-standard-4` is an ARC pod
+# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no
+# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed:
+# No such file or directory"). The prepare step therefore mounts four 1 GiB
+# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`.
+
+name: e2e-distributed
+
+on:
+  pull_request:
+    paths:
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - ".config/nextest.toml"
+      - ".github/workflows/e2e-distributed.yml"
+      - "crates/audit/**"
+      - "crates/common/**"
+      - "crates/config/**"
+      - "crates/e2e_test/**"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/iam/**"
+      - "crates/lock/**"
+      - "crates/madmin/**"
+      - "crates/notify/**"
+      - "crates/replication/**"
+      - "crates/s3-client/**"
+      - "crates/s3-ops/**"
+      - "crates/s3-types/**"
+      - "crates/scanner/**"
+      - "crates/storage-api/**"
+      - "crates/utils/**"
+      - "rustfs/**"
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
+        required: false
+        default: ""
+  schedule:
+    # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
+    - cron: "53 5 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
+jobs:
+  distributed:
+    name: Distributed 4-node 4-disk e2e
+    runs-on: sm-standard-4
+    timeout-minutes: 180
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      NO_PROXY: 127.0.0.1,localhost
+      HTTP_PROXY: ""
+      HTTPS_PROXY: ""
+      # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
+      UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
+      UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
+      UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-e2e-distributed
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
+
+      - name: Prepare isolated filesystems for pool movement
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          mkdir -p "${mount_base}"
+          roots=()
+          for pool in 0 1 2 3; do
+            mountpoint="${mount_base}/pool-${pool}"
+            mkdir -p "${mountpoint}"
+            # sm-standard-4 is an ARC pod without usable loop devices, so
+            # `mount -o loop` fails with ENOENT. Sized tmpfs still reports a
+            # distinct st_dev and independent 1G statfs capacity.
+            sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"
+            sudo chmod 1777 "${mountpoint}"
+            roots+=("${mountpoint}")
+          done
+          printf -v joined_roots '%s:' "${roots[@]}"
+          echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}"
+
+      - name: Download pinned previous release
+        env:
+          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
+        run: |
+          set -euo pipefail
+          mkdir -p "$SOURCE_DIR"
+          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
+          curl --fail --location --retry 3 --output "$archive" \
+            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
+          unzip -q "$archive" -d "$SOURCE_DIR"
+          chmod +x "$SOURCE_DIR/rustfs"
+          test -x "$SOURCE_DIR/rustfs"
+          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
+
+      - name: Build rustfs binary
+        run: |
+          cargo build -p rustfs --bins
+          : > target/debug/rustfs.features
+
+      - name: Verify distributed e2e membership
+        env:
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+        run: |
+          cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
+
+      - name: Run distributed 4-node e2e suite
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
+          FILTER: ${{ inputs.filter }}
+        run: |
+          set -euo pipefail
+          if [ -n "${FILTER}" ]; then
+            cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
+          else
+            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+          fi
+
+      - name: Upload distributed e2e diagnostics
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-distributed-${{ github.run_number }}
+          path: |
+            target/nextest/e2e-distributed/junit.xml
+            ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+            ${{ runner.temp }}/rustfs-e2e-distributed-logs/
+          retention-days: 7
+          if-no-files-found: warn
+
+      - name: Unmount isolated pool filesystems
+        if: always()
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          for pool in 0 1 2 3; do
+            mountpoint="${mount_base}/pool-${pool}"
+            if mountpoint --quiet "${mountpoint}"; then
+              sudo umount "${mountpoint}"
+            fi
+          done
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [distributed]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -166,9 +166,8 @@ jobs:
      # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
      # Skipped when the R2 secrets are not configured (artifact-only mode).
      - name: Upload DEB to Cloudflare R2
-        id: publish
+        if: env.R2_ACCESS_KEY_ID != ''
        env:
-          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -183,70 +182,28 @@ jobs:
            exit 0
          fi

+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y -qq awscli
+          fi
+
          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
          export AWS_DEFAULT_REGION="auto"

-          SOURCE_SHA="$(git rev-parse HEAD)"
-          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
-            echo "Checkout SHA does not match the nightly build run" >&2
-            exit 1
-          fi
-          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
-          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
-          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
-
-          # Old AWS CLI models lack conditional PutObject support. Never fall
-          # back to an overwriting upload for a candidate.
-          AWS_CLI=aws
-          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
-            sudo apt-get update
-            sudo apt-get install -y -qq python3-venv
-            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
-            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
-            python3 -m venv "${AWS_CLI_DIR}"
-            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
-            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
-          fi
-          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
-          "${AWS_CLI}" --version
-          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
-            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
-          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
-          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
-            echo "Published candidate checksum does not match the built package" >&2
-            exit 1
-          fi
-
+          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
          R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"

          echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
-          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors

          # Stable "latest" alias so tests can fetch the newest nightly
          # without knowing today's date.
          echo "📤 Uploading latest alias"
-          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
            --endpoint-url "$R2_ENDPOINT" --only-show-errors

          echo "✅ R2 upload complete"

-          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --arg source_sha "${SOURCE_SHA}" \
-            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
-            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
-            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
-            > "${CANDIDATE_FILE}"
-          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload nightly candidate manifest
-        if: ${{ steps.publish.outputs.candidate_file != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ steps.publish.outputs.candidate_file }}
-          if-no-files-found: error
-
  # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
  #
  # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
@@ -14,8 +14,8 @@

 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication -> performance). Each suite attempts the next handoff even
-# when its tests fail.
+# replication, with performance on its own runner in parallel) and guarantees
+# the chain keeps moving even when individual suites fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,3 +59,16 @@ jobs:
          gh api --method POST repos/rustfs/rustfs/dispatches \
            -f event_type='rustfs-chain-upgrade' \
            -F 'client_payload[from_suite]=nightly-build'
+
+      - name: Dispatch performance suite (parallel, own runner)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=nightly-build'
@@ -59,21 +59,6 @@ jobs:
    # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -129,7 +114,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"

      - name: Preflight checks
        run: |
@@ -139,7 +124,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"

      - name: Run heal test (write -> outage -> heal -> verify)
        id: test
@@ -149,10 +134,13 @@ jobs:
            --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file "${LOG_FILE}"
+            --log-file /tmp/rustfs-heal-test.log

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-heal-test.log
+          REPORT_FILE: /tmp/rustfs-heal-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -161,9 +149,8 @@ jobs:
          else
            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
          fi
-          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
-          CASE_RESULT=success
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
+          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
          import re
          import sys

@@ -175,7 +162,6 @@ jobs:

          steps = {}
          order = []
-          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
          version = None
          version_node = None
          verdict = None
@@ -189,15 +175,14 @@ jobs:
                          n, desc, status = m.group(1), m.group(2), m.group(3)
                          if n not in steps:
                              order.append(n)
-                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
-                              steps[n] = (desc, status)
+                          steps[n] = (desc, status)  # later lines win (fail after pass)
                          continue
                      m = ver_re.match(line)
                      if m:
                          version, version_node = m.group(1), m.group(2)
                          continue
                      m = result_re.match(line)
-                      if m and verdict != 'FAIL':
+                      if m:
                          verdict, verdict_detail = m.group(1), m.group(2)
          except FileNotFoundError:
              pass
@@ -217,43 +202,30 @@ jobs:
                  out.write(f'| {n} | {desc} | {status} |\n')
              if not order:
                  out.write('| - | - | NOT RUN (no step result lines found) |\n')
-          complete = set(steps) == {str(n) for n in range(1, 8)}
-          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
          PY
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
          {
            echo "# RustFS heal test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${STEPS_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
-            fi
+            cat "${STEPS_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-heal-report.md
          SUITE: heal
        run: |
          set -euo pipefail
@@ -263,32 +235,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'heal'
          SUITE_LABEL: 'Heal'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-heal-report.md'
+          LOG_FILE: '/tmp/rustfs-heal-test.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -316,16 +284,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -341,16 +307,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload test logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-heal-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
-          if-no-files-found: error
+            /tmp/rustfs-heal-test*.log
+            /tmp/rustfs-warp.*.log
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: ${{ always() && inputs.cleanup_after != 'false' }}
@@ -52,25 +52,6 @@ jobs:
    timeout-minutes: 420
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -127,6 +108,8 @@ jobs:

      - name: Run KMS suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-kms-test.sh
@@ -156,7 +139,10 @@ jobs:
          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+          REPORT_FILE: /tmp/rustfs-kms-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -168,43 +154,79 @@ jobs:
          else
            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-kms-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS KMS test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-kms-report.md
          SUITE: kms
        run: |
          set -euo pipefail
@@ -214,32 +236,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'kms'
          SUITE_LABEL: 'KMS'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-kms-report.md'
+          LOG_FILE: '/tmp/rustfs-kms.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -267,16 +285,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -292,15 +308,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-kms-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-kms.log
+            /tmp/rustfs-kms-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -49,16 +49,17 @@ on:
        type: boolean
        default: true
  repository_dispatch:
-    # Chain handoff: dispatched when the replication suite finishes.
+    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
+    # pf-testing runner, in parallel with the shared-VM chain).
    types: [rustfs-chain-performance]

 permissions:
  contents: read

-# The default performance nodes overlap the other suites' remote VMs, even
-# though the runner differs. Hold the shared lock through cleanup as well.
+# Dedicated pf-testing runner/environment: own concurrency group so perf runs
+# never block (or are blocked by) the pool-expansion / heal tests.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-performance-test
  cancel-in-progress: false

 defaults:
@@ -75,6 +76,8 @@ env:
  # Package used by the nightly run (workflow_dispatch inputs are empty for
  # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  # Fixed benchmark result directory so later steps can read summary.md
+  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
  # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}

@@ -86,22 +89,6 @@ jobs:
    # Skipped when nightly failed.
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -133,7 +120,7 @@ jobs:
        if: ${{ inputs.cleanup_before != 'false' }}
        run: |
          chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y

      - name: Install RustFS package & start cluster (4x4)
        run: |
@@ -143,7 +130,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"

      - name: Preflight checks
        run: |
@@ -153,7 +140,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"

      - name: Run benchmark (GET/PUT/MIXED)
        id: benchmark
@@ -166,15 +153,17 @@ jobs:
            --step 5 -y \
            --warp-duration "${{ inputs.warp_duration || '5m' }}" \
            --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file "${LOG_FILE}"
+            --log-file /tmp/rustfs-perf-test.log

      - name: Analyze results
        if: ${{ steps.benchmark.conclusion == 'success' }}
        run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y

      - name: Collect RustFS version info
        if: ${{ steps.benchmark.conclusion == 'success' }}
+        env:
+          VERSION_FILE: /tmp/rustfs-version.txt
        run: |
          set -euo pipefail
          read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -194,6 +183,7 @@ jobs:
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
          RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
+          VERSION_FILE: /tmp/rustfs-version.txt
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -201,7 +191,7 @@ jobs:
            exit 0
          fi
          SUMMARY="${RESULT_DIR}/summary.md"
-          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="reports/${DATE}.md"
          {
@@ -209,8 +199,6 @@ jobs:
            echo ""
            echo "- **Date**: ${DATE}"
            echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
-            echo "- **Workflow Commit**: ${GITHUB_SHA}"
            echo "- **Trigger**: ${{ github.event_name }}"
            echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
            echo ""
@@ -220,8 +208,8 @@ jobs:
            echo '```text'
            cat "${VERSION_FILE}"
            echo '```'
-          } > "${REPORT_FILE}"
-          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          } > /tmp/rustfs-perf-report.md
+          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
            jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -240,10 +228,11 @@ jobs:
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'performance'
          SUITE_LABEL: 'Performance'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-perf-report.md'
+          LOG_FILE: '/tmp/rustfs-perf-test.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -271,16 +260,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -296,26 +283,20 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload test logs & results
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-perf-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
-          if-no-files-found: error
+            /tmp/rustfs-perf-test*.log
+            /tmp/rustfs-perf-results/**
+            /tmp/rustfs-version.txt
+          if-no-files-found: warn

      - name: Reset test environment (after)
        if: ${{ always() && inputs.cleanup_after != 'false' }}
        run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y

      - name: Notify on failure
        if: failure()
@@ -539,22 +539,17 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
@@ -34,7 +34,8 @@ on:
          - site
        default: all
  repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes.
+    # Chain handoff: dispatched when the security suite finishes. This is the
+    # last link of the functional chain.
    types: [rustfs-chain-replication]

 permissions:
@@ -64,25 +65,6 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -131,6 +113,8 @@ jobs:

      - name: Run replication suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-replication.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-replication-test.sh
@@ -153,7 +137,10 @@ jobs:
          ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-replication.log
+          REPORT_FILE: /tmp/rustfs-replication-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -175,44 +162,80 @@ jobs:
              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
            fi
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-replication-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+                  for raw in fh:
+                      line = ansi.sub('', raw).strip()
+                      m = start_re.match(line)
+                      if m:
+                          case_id, name = m.group(1), m.group(2)
+                          if case_id not in index:
+                              index[case_id] = len(rows)
+                              rows.append([case_id, name, 'RUNNING'])
+                          continue
+                      m = done_re.match(line)
+                      if m:
+                          status, case_id = m.group(1), m.group(2)
+                          if case_id in index:
+                              rows[index[case_id]][2] = status
+                          else:
+                              rows.append([case_id, case_id, status])
+                              index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+              rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+              counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+              out.write('## Case Summary\n\n')
+              out.write(f"- Total: {len(rows)}\\n")
+              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+              out.write('\\n')
+              out.write('| Case | Name | Status |\\n')
+              out.write('| --- | --- | --- |\\n')
+              for case_id, name, status in rows:
+                  out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS replication test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-replication-report.md
          SUITE: replication
        run: |
          set -euo pipefail
@@ -222,32 +245,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'replication'
          SUITE_LABEL: 'Replication (bucket + site)'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-replication-report.md'
+          LOG_FILE: '/tmp/rustfs-replication.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -275,16 +294,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -300,15 +317,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-replication-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-replication.log
+            /tmp/rustfs-replication-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -329,50 +345,13 @@ jobs:
            '
          done

-      - name: "Continue functional chain (next: Performance)"
+      - name: Chain complete
+        # Replication is the last link of the functional chain: nothing to
+        # dispatch after it. This step just records that the chain finished.
        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
-              -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            trap 'rm -f "${BODY_FILE}"' EXIT
-            {
-              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          echo "Functional chain complete: replication (final suite) finished."
+          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"

      - name: Notify on failure
        if: failure()
@@ -40,25 +40,6 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -106,6 +87,8 @@ jobs:

      - name: Run S3 compatibility suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -122,7 +105,10 @@ jobs:
          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -144,44 +130,83 @@ jobs:
              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
            fi
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          current = None
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  current = case_id
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+                  current = None
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS S3 compatibility test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
          SUITE: s3
        run: |
          set -euo pipefail
@@ -191,32 +216,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 's3'
          SUITE_LABEL: 'S3 compatibility'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
+          LOG_FILE: '/tmp/rustfs-s3-compat.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -244,16 +265,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -269,15 +288,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-s3-compat-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-s3-compat.log
+            /tmp/rustfs-s3-compat-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -77,14 +77,10 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      # Checkout the repository into its own subdirectory. Checking out at
-      # the workspace root would wipe the auto-testing clone above (that is
-      # exactly how run 33934141181 lost rustfs-security-test.sh).
      - name: Checkout repository (for the OIDC live gate script)
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
-          path: rustfs-repo

      - name: Initialize security evidence
        id: evidence
@@ -92,7 +88,7 @@ jobs:
          set -euo pipefail
          umask 077
          SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
+          mkdir -- "${SECURITY_ARTIFACTS_DIR}"
          printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"

      # auto-testing is private: clone it with the dedicated PF token (not
@@ -148,8 +144,8 @@ jobs:
        continue-on-error: true
        env:
          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
-          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
-          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
+          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
+          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-security-test.sh
@@ -172,7 +168,7 @@ jobs:
          else
            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
          fi
-          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
+          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"

      - name: Generate report
        id: report
@@ -223,22 +219,17 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
@@ -305,10 +296,7 @@ jobs:
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
-            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
-            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
+          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
          if-no-files-found: error
          retention-days: 3

@@ -49,25 +49,6 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -115,6 +96,8 @@ jobs:

      - name: Run storage engine suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-storage.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-storage-test.sh
@@ -137,7 +120,10 @@ jobs:
          ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-storage.log
+          REPORT_FILE: /tmp/rustfs-storage-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -159,44 +145,83 @@ jobs:
              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
            fi
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-storage-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          current = None
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  current = case_id
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+                  current = None
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS storage engine test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-storage-report.md
          SUITE: storage
        run: |
          set -euo pipefail
@@ -206,32 +231,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'storage'
          SUITE_LABEL: 'Storage engine'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-storage-report.md'
+          LOG_FILE: '/tmp/rustfs-storage.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -259,16 +280,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -284,15 +303,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-storage-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-storage.log
+            /tmp/rustfs-storage-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -377,22 +377,17 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: Verify required tier evidence
        id: evidence_verify
@@ -82,25 +82,6 @@ jobs:
    timeout-minutes: 420
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -161,6 +142,7 @@ jobs:
      - name: Run upgrade compatibility suite
        id: test
        env:
+          LOG_FILE: /tmp/rustfs-upgrade.log
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
        run: |
          set -euo pipefail
@@ -218,7 +200,10 @@ jobs:
          ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-upgrade.log
+          REPORT_FILE: /tmp/rustfs-upgrade-report.md
        run: |
          set -euo pipefail
          FROM_URL='${{ inputs.from_url }}'
@@ -239,47 +224,103 @@ jobs:
          else
            TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
+          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+          topo_re = re.compile(
+              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
+
+          rows = []
+          index = {}
+          topo_rows = []
+          try:
+              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+                  for raw in fh:
+                      line = ansi.sub('', raw).strip()
+                      m = topo_re.match(line)
+                      if m:
+                          topo_rows.append(m.groups())
+                          continue
+                      m = start_re.match(line)
+                      if m:
+                          case_id, name = m.group(1), m.group(2)
+                          if case_id not in index:
+                              index[case_id] = len(rows)
+                              rows.append([case_id, name, 'RUNNING'])
+                          continue
+                      m = done_re.match(line)
+                      if m:
+                          status, case_id = m.group(1), m.group(2)
+                          if case_id in index:
+                              rows[index[case_id]][2] = status
+                          else:
+                              rows.append([case_id, case_id, status])
+                              index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+              rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+              counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+              out.write('## Case Summary\n\n')
+              out.write(f"- Total: {len(rows)}\\n")
+              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+              out.write('\\n')
+              out.write('| Case | Name | Status |\\n')
+              out.write('| --- | --- | --- |\\n')
+              for case_id, name, status in rows:
+                  out.write(f'| {case_id} | {name} | {status} |\\n')
+
+          # Upgrade matrix: one row per topology/backend with the versions
+          # captured on the nodes (rustfs --version) and the aggregated
+          # result. The dashboard renders this table directly.
+          with open(matrix_file, 'w', encoding='utf-8') as out:
+              out.write('## Upgrade Matrix\n\n')
+              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
+              out.write('| --- | --- | --- | --- | --- |\n')
+              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
+                  result = 'PASS' if nfail == '0' else 'FAIL'
+                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
+              if not topo_rows:
+                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
+          PY
          {
            echo "# RustFS upgrade compatibility report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- From: ${FROM_SOURCE}"
            echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${MATRIX_TABLE}"
-              echo ""
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${MATRIX_TABLE}" || true
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-upgrade-report.md
          SUITE: upgrade
        run: |
          set -euo pipefail
@@ -289,32 +330,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'upgrade'
          SUITE_LABEL: 'Upgrade compatibility'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
+          LOG_FILE: '/tmp/rustfs-upgrade.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -342,16 +379,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -367,16 +402,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-upgrade-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
-          if-no-files-found: error
+            /tmp/rustfs-upgrade-report.md
+            /tmp/rustfs-upgrade.*/*
+          if-no-files-found: ignore
          retention-days: 3

      - name: Cleanup environment (after)
@@ -22,6 +22,7 @@ on:
      - "Continuous Integration"
      - "coverage"
      - "e2e-nightly"
+      - "e2e-distributed"
      - "e2e-s3tests"
      - "Fuzz"
      - "mint"
@@ -1679,7 +1679,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]

 [[package]]
@@ -1698,7 +1698,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]

 [[package]]
@@ -2110,7 +2110,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
- "crypto-common 0.1.6",
+ "crypto-common 0.1.7",
 "inout 0.1.4",
 ]

@@ -2580,7 +2580,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "rand_core 0.6.4",
 "subtle",
 "zeroize",
@@ -2605,11 +2605,11 @@ dependencies = [

 [[package]]
 name = "crypto-common"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "typenum",
 ]

@@ -3901,7 +3901,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
 "block-buffer 0.10.4",
 "const-oid 0.9.6",
- "crypto-common 0.1.6",
+ "crypto-common 0.1.7",
 "subtle",
 ]

@@ -4166,7 +4166,7 @@ dependencies = [
 "crypto-bigint 0.5.5",
 "digest 0.10.7",
 "ff 0.13.1",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "group 0.13.0",
 "hkdf 0.12.4",
 "pem-rfc7468 0.7.0",
@@ -4499,7 +4499,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
 dependencies = [
 "io-lifetimes 2.0.4",
 "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.52.0",
 ]

 [[package]]
@@ -4622,9 +4622,9 @@ dependencies = [

 [[package]]
 name = "generic-array"
-version = "0.14.9"
+version = "0.14.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
 "typenum",
 "version_check",
@@ -4637,7 +4637,7 @@ version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "rustversion",
 "typenum",
 ]
@@ -5319,9 +5319,9 @@ dependencies = [

 [[package]]
 name = "hotpath-macros"
-version = "0.25.1"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "846bde0d9600d98434e1aac376977d7718bfe3d2f5312a041b7c59a6a466c51a"
+checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5660,7 +5660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
 dependencies = [
 "block-padding 0.3.3",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]

 [[package]]
@@ -5693,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
 dependencies = [
 "io-lifetimes 3.0.1",
- "windows-sys 0.60.2",
+ "windows-sys 0.52.0",
 ]

 [[package]]
@@ -7115,7 +7115,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.21.7",
 "chrono",
 "getrandom 0.2.17",
 "http 1.5.0",
@@ -11400,7 +11400,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
 dependencies = [
 "base16ct 0.2.0",
 "der 0.7.10",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "pkcs8 0.10.2",
 "subtle",
 "zeroize",
@@ -12400,7 +12400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
 "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
 "once_cell",
 "rustix",
 "windows-sys 0.61.2",
@@ -13652,15 +13652,6 @@ dependencies = [
 "windows-targets 0.52.6",
 ]

-[[package]]
-name = "windows-sys"
-version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.60.2"
@@ -13833,7 +13824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
 dependencies = [
 "bitflags 2.13.1",
- "windows-sys 0.59.0",
+ "windows-sys 0.52.0",
 ]

 [[package]]
@@ -371,8 +371,7 @@ dav-server = "0.11.0"

 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.3" }
-# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "=0.25.0", default-features = false }
+hotpath = { version = "0.25.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }

@@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |

@@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker.
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
 | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
+# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
+# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
+cargo nextest run --profile e2e-distributed -p e2e_test
 # Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
@@ -1,74 +0,0 @@
-// Copyright 2024 RustFS Team
-// Licensed under the Apache License, Version 2.0.
-
-use std::path::Path;
-use std::process::Command;
-
-fn git(root: &Path, args: &[&str]) -> Option<String> {
-    let output = Command::new("git").args(args).current_dir(root).output().ok()?;
-    output
-        .status
-        .success()
-        .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
-}
-
-fn emit(name: &str, value: &str) {
-    let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
-    println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
-}
-
-fn main() {
-    let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
-    let root = Path::new(&manifest).join("../..");
-    // Cover dependency/common sources as well as this crate. HEAD/ref/index
-    // changes must refresh identity even when no Rust source mtime changes.
-    for path in [
-        "crates",
-        "rustfs",
-        "Cargo.toml",
-        "Cargo.lock",
-        "rust-toolchain.toml",
-        ".cargo",
-        ".config",
-    ] {
-        println!("cargo:rerun-if-changed={}", root.join(path).display());
-    }
-    let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
-    if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
-        git_paths.push(reference);
-    }
-    for path in git_paths {
-        if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
-            let path = Path::new(&path);
-            let path = if path.is_absolute() {
-                path.to_owned()
-            } else {
-                root.join(path)
-            };
-            if path.exists() {
-                println!("cargo:rerun-if-changed={}", path.display());
-            }
-        }
-    }
-    let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
-    let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
-    let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
-    let mut features = std::env::vars()
-        .filter_map(|(key, _)| {
-            key.strip_prefix("CARGO_FEATURE_")
-                .map(|name| name.to_ascii_lowercase().replace('_', "-"))
-        })
-        .collect::<Vec<_>>();
-    features.sort();
-    emit("COMMIT", &revision);
-    emit("DIRTY", if dirty { "true" } else { "false" });
-    emit("LOCK", &lock);
-    emit("FEATURES", &features.join(","));
-    for name in ["TARGET", "PROFILE"] {
-        emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
-    }
-    println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
-    let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
-    let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
-    emit("RUSTFLAGS_HEX", &flags);
-}
@@ -55,20 +55,18 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
 /// A successful S3 GET only proves that a quorum can serve an object. Replacement
 /// tests need this lower-level record to prove that the rebuilt target holds the
 /// `xl.meta` selected for a specific version and every `part.N` it declares.
-#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub(crate) struct VersionShardCensus {
    pub version_id: Option<String>,
    pub has_xl_meta: bool,
    pub data_dir: Option<String>,
    pub erasure_index: Option<usize>,
-    pub data_blocks: Option<usize>,
-    pub parity_blocks: Option<usize>,
    pub expected_part_numbers: BTreeSet<usize>,
    pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
    pub inline_data_fingerprint: Option<PartShardFingerprint>,
 }

-#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub(crate) struct PartShardFingerprint {
    pub size: u64,
    pub sha256: String,
@@ -90,15 +88,13 @@ impl VersionShardCensus {
            && manifest.is_complete()
            && self.data_dir == manifest.data_dir
            && self.erasure_index == manifest.erasure_index
-            && self.data_blocks == manifest.data_blocks
-            && self.parity_blocks == manifest.parity_blocks
            && self.expected_part_numbers == manifest.expected_part_numbers
            && self.present_part_fingerprints == manifest.present_part_fingerprints
            && self.inline_data_fingerprint == manifest.inline_data_fingerprint
    }
 }

-pub(crate) fn sha256_hex(data: &[u8]) -> String {
+fn sha256_hex(data: &[u8]) -> String {
    let digest = Sha256::digest(data);
    digest.iter().map(|byte| format!("{byte:02x}")).collect()
 }
@@ -317,8 +313,6 @@ pub(crate) fn census_object_version_on_disk(
            has_xl_meta: false,
            data_dir: None,
            erasure_index: None,
-            data_blocks: None,
-            parity_blocks: None,
            expected_part_numbers: BTreeSet::new(),
            present_part_fingerprints: BTreeMap::new(),
            inline_data_fingerprint: None,
@@ -366,8 +360,6 @@ pub(crate) fn census_object_version_on_disk(
        has_xl_meta: true,
        data_dir,
        erasure_index,
-        data_blocks: Some(file_info.erasure.data_blocks),
-        parity_blocks: Some(file_info.erasure.parity_blocks),
        expected_part_numbers,
        present_part_fingerprints,
        inline_data_fingerprint,
@@ -421,8 +413,6 @@ mod tests {
            has_xl_meta: true,
            data_dir: Some("data-dir".to_string()),
            erasure_index: Some(3),
-            data_blocks: Some(2),
-            parity_blocks: Some(2),
            expected_part_numbers: BTreeSet::from([1]),
            present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
            inline_data_fingerprint: None,
@@ -1700,6 +1700,69 @@ impl RustFSTestClusterEnvironment {
        Ok(())
    }

+    /// Append a new single-node erasure pool to a stopped multi-pool cluster.
+    ///
+    /// Used to simulate pool expansion on localhost: every pool already owns
+    /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
+    /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
+    /// allocated a fresh port and empty drive directories; callers must
+    /// [`Self::start`] afterwards so every process picks up the extended
+    /// volumes argument. Existing data directories are left untouched.
+    pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+        if self.nodes.iter().any(|node| node.process.is_some()) {
+            return Err("stop the cluster before appending a pool".into());
+        }
+        if self.topology.drives_per_node < 2 {
+            return Err(
+                "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
+                    .into(),
+            );
+        }
+
+        let mut pools = self.topology.normalized_pools();
+        for (pool_idx, nodes) in pools.iter().enumerate() {
+            if nodes.len() != 1 {
+                return Err(format!(
+                    "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
+                    nodes.len()
+                )
+                .into());
+            }
+        }
+
+        let new_idx = self.nodes.len();
+        let port = RustFSTestEnvironment::find_available_port().await?;
+        let address = format!("127.0.0.1:{port}");
+        let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
+            .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
+            .collect();
+        for dir in &data_dirs {
+            fs::create_dir_all(dir).await?;
+        }
+
+        self.nodes.push(ClusterNode {
+            url: format!("http://{address}"),
+            address,
+            data_dir: data_dirs[0].clone(),
+            data_dirs,
+            pool_idx: pools.len(),
+            process: None,
+        });
+        pools.push(vec![new_idx]);
+        self.topology.node_count = self.nodes.len();
+        self.topology.pools = pools;
+        self.node_extra_env.push(Vec::new());
+        self.node_capture_log_paths.push(None);
+        self.volume_proxy_addresses.push(None);
+
+        if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
+            self.extra_env
+                .push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
+        }
+
+        Ok(new_idx)
+    }
+
    /// Gracefully stop one cluster node and wait for its process to exit.
    ///
    /// This is intentionally separate from [`Self::stop_node`]: the latter is
@@ -0,0 +1,222 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
+    wait_for_ready, wait_until,
+};
+use crate::chaos::{census_object_version_on_disk, signed_admin_post};
+use crate::common::{build_test_s3_config, init_logging};
+use crate::fault_proxy::FaultMode;
+use aws_sdk_s3::Client;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{Barrier, mpsc};
+use tokio::time::timeout;
+
+#[tokio::test]
+async fn kill_and_restart_node_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("killnode");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x11u8; 128 * 1024];
+    put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
+
+    dist.cluster.stop_node(3)?;
+    retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn full_cluster_restart_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("pwr");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x44u8; 64 * 1024];
+    put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
+    let bucket = unique_bucket("baddrive");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
+
+    let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
+    let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(
+        baseline.is_complete(),
+        "replacement target did not hold a complete baseline shard: {baseline:?}"
+    );
+    assert!(
+        !baseline.expected_part_numbers.is_empty(),
+        "replacement witness must use physical part shards: {baseline:?}"
+    );
+
+    dist.cluster.stop_node(0)?;
+    let format_path = replaced_drive.join(".rustfs.sys/format.json");
+    let format = std::fs::read(&format_path)?;
+    let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
+    std::fs::rename(&replaced_drive, &retired_drive)?;
+    std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
+    std::fs::write(&format_path, format)?;
+    let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
+
+    dist.cluster.start_node(0).await?;
+    wait_for_ready(&dist.cluster).await?;
+    let heal_body =
+        r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+    let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
+    signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+    wait_until(
+        Duration::from_secs(90),
+        || async {
+            let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+            Ok(healed.matches_manifest(&baseline))
+        },
+        "fresh replacement contains the original complete shard manifest",
+    )
+    .await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("getkill");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
+
+    let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
+    let worker_count = 12;
+    let release = Arc::new(Barrier::new(worker_count + 1));
+    let (started_tx, mut started_rx) = mpsc::unbounded_channel();
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let client = live[idx % live.len()].clone();
+        let bucket = bucket.clone();
+        let body = body.clone();
+        let release = release.clone();
+        let started_tx = started_tx.clone();
+        handles.push(tokio::spawn(async move {
+            let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
+            if response.content_length() != Some(body.len() as i64) {
+                return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
+                    format!("worker {idx} received a wrong content length").into(),
+                );
+            }
+            started_tx.send(idx)?;
+            release.wait().await;
+            let actual = response.body.collect().await?.into_bytes();
+            if actual.as_ref() != body.as_slice() {
+                return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
+            }
+            Ok(())
+        }));
+    }
+    drop(started_tx);
+    for _ in 0..worker_count {
+        timeout(Duration::from_secs(30), started_rx.recv())
+            .await?
+            .ok_or("a streaming GET exited before reaching the kill barrier")?;
+    }
+
+    dist.cluster.stop_node(3)?;
+    release.wait().await;
+    for handle in handles {
+        handle.await??;
+    }
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
+    let proxied_url = format!("http://{}", proxy.local_addr());
+    let proxied_client = Client::from_conf(build_test_s3_config(
+        &proxied_url,
+        &dist.cluster.access_key,
+        &dist.cluster.secret_key,
+        None,
+        "distributed-network-chaos",
+    ));
+
+    let result: TestResult = async {
+        let bucket = unique_bucket("netfault");
+        dist.create_bucket(&bucket).await?;
+        let baseline = payload_for("network/baseline.bin", 1024 * 1024);
+        put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
+        assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
+
+        proxy.set_mode(FaultMode::Blackhole);
+        assert_eq!(proxy.mode(), FaultMode::Blackhole);
+        if let Ok(Ok(_)) = timeout(
+            Duration::from_secs(5),
+            proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
+        )
+        .await
+        {
+            return Err("blackholed node endpoint unexpectedly completed a GET".into());
+        }
+
+        let during = payload_for("network/during.bin", 1024 * 1024);
+        timeout(Duration::from_secs(30), async {
+            put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
+            assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
+            assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
+        })
+        .await??;
+
+        proxy.set_mode(FaultMode::Pass);
+        retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
+        Ok(())
+    }
+    .await;
+
+    proxy.set_mode(FaultMode::Pass);
+    proxy.shutdown().await;
+    result
+}
@@ -0,0 +1,98 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use crate::common::init_logging;
+use std::collections::BTreeSet;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("conc");
+    dist.create_bucket(&bucket).await?;
+    let clients = Arc::new(dist.clients()?);
+    let worker_count = 24;
+    let rounds = 4;
+    let barrier = Arc::new(Barrier::new(worker_count));
+
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let writer = &clients[idx % clients.len()];
+            let reader = &clients[(idx + 1) % clients.len()];
+            let copier = &clients[(idx + 2) % clients.len()];
+            let mut retained = Vec::with_capacity(rounds);
+            for round in 0..rounds {
+                let key = format!("source/worker-{idx:02}-round-{round}.bin");
+                let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
+                let body = payload_for(&key, 64 * 1024);
+                put_object(writer, &bucket, &key, body.clone()).await?;
+
+                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                if head.content_length() != Some(body.len() as i64) {
+                    return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
+                }
+                assert_object_bytes(reader, &bucket, &key, &body).await?;
+
+                copier
+                    .copy_object()
+                    .bucket(&bucket)
+                    .key(&copy_key)
+                    .copy_source(format!("{bucket}/{key}"))
+                    .send()
+                    .await?;
+                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+
+                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                let missing = reader
+                    .head_object()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .send()
+                    .await
+                    .expect_err("deleted source key must not remain visible");
+                if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
+                    return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
+                }
+                retained.push((copy_key, body));
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
+        }));
+    }
+
+    let mut inventory = Vec::new();
+    for handle in handles {
+        inventory.extend(handle.await??);
+    }
+
+    let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
+    for (node_idx, client) in clients.iter().enumerate() {
+        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+        assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
+        for (key, body) in &inventory {
+            assert_object_bytes(client, &bucket, key, body)
+                .await
+                .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
+        }
+    }
+    Ok(())
+}
@@ -0,0 +1,74 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
+    unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("concdecom");
+    dist.create_bucket(&bucket).await?;
+    let baseline_client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(17));
+    let mut handles = Vec::new();
+    for idx in 0..16 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("live/{idx:02}.bin");
+            let body = payload_for(&key, 8 * 1024);
+            retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    barrier.wait().await;
+
+    let mut live_objects = Vec::new();
+    for handle in handles {
+        live_objects.push(handle.await??);
+    }
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let checker = dist.client(2)?;
+    assert_inventory(&checker, &bucket, &inventory).await?;
+    for (key, body) in live_objects {
+        retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,156 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
+    sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("integrity");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    enable_versioning(&client, &bucket).await?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
+
+    let versioned_key = "history/versioned.bin";
+    let version_one = b"historical bytes before data movement".to_vec();
+    let version_two = b"current bytes before data movement".to_vec();
+    let version_one_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_one.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("historical PUT omitted version ID")?
+        .to_string();
+    let version_two_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_two.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("current PUT omitted version ID")?
+        .to_string();
+
+    let multipart_key = "multipart/moved.bin";
+    let first_part = vec![0x31; 5 * 1024 * 1024];
+    let second_part = vec![0x72; 1024 * 1024];
+    let upload = client
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
+    let uploaded_one = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(first_part.clone()))
+        .send()
+        .await?;
+    let uploaded_two = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(2)
+        .body(ByteStream::from(second_part.clone()))
+        .send()
+        .await?;
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after_client = dist.client(2)?;
+    assert_inventory(&after_client, &bucket, &inventory).await?;
+    for (key, expected_hash) in before {
+        let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
+        let body = got.body.collect().await?.into_bytes();
+        assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
+    }
+    for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
+        let got = after_client
+            .get_object()
+            .bucket(&bucket)
+            .key(versioned_key)
+            .version_id(version_id)
+            .send()
+            .await?
+            .body
+            .collect()
+            .await?
+            .into_bytes();
+        assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
+    }
+    let mut expected_multipart = first_part;
+    expected_multipart.extend_from_slice(&second_part);
+    let got_multipart = after_client
+        .get_object()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?
+        .body
+        .collect()
+        .await?
+        .into_bytes();
+    assert_eq!(
+        sha256_hex(got_multipart.as_ref()),
+        sha256_hex(&expected_multipart),
+        "multipart checksum changed after decommission"
+    );
+    Ok(())
+}
@@ -0,0 +1,81 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
+    put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
+    wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("expand");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
+    assert_inventory(&client, &bucket, &inventory).await?;
+
+    for expected_nodes in 2..=4 {
+        let new_node = dist.append_pool_and_restart().await?;
+        assert_eq!(new_node + 1, expected_nodes);
+        assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
+    }
+    assert_eq!(dist.cluster.nodes.len(), 4);
+
+    // Prove that the expanded pool map is durable, and clear any recovery
+    // latch raised while the newly-added pool replicas converged.
+    dist.restart_current_binary_gracefully().await?;
+
+    let after_expand = dist.client(0)?;
+    assert_inventory(&after_expand, &bucket, &inventory).await?;
+    let peer = dist.client(3)?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("decom");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    let pools_before = list_pools_json(&dist.cluster).await?;
+    let pool_count = pools_before
+        .as_array()
+        .map(Vec::len)
+        .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
+        .ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
+    assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after = dist.client(2)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,149 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("extra");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+
+    let key = "multipart.bin";
+    let part1 = vec![0x41u8; 5 * 1024 * 1024];
+    let part2 = vec![0x42u8; 5 * 1024 * 1024];
+    let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
+
+    let uploaded1 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(part1.clone()))
+        .send()
+        .await?;
+    let uploaded2 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(2)
+        .body(ByteStream::from(part2.clone()))
+        .send()
+        .await?;
+
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded1.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded2.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let mut expected = part1;
+    expected.extend_from_slice(&part2);
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
+    }
+
+    put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
+    put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
+    let mut seen = Vec::new();
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let listed = dist
+            .client(node_idx)?
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("list/")
+            .send()
+            .await?;
+        let keys: Vec<String> = listed
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        seen.push(keys);
+    }
+    for keys in &seen[1..] {
+        assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
+    }
+
+    let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
+    assert_eq!(got, b"a");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("listed");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node_idx)?;
+        let name = bucket.clone();
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = client.clone();
+                let name = name.clone();
+                async move {
+                    let listed = client.list_buckets().send().await?;
+                    Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
+                }
+            },
+            &format!("node {node_idx} lists {bucket}"),
+        )
+        .await?;
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = dist.client(node_idx).expect("client");
+                let name = bucket.clone();
+                async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
+            },
+            &format!("node {node_idx} reads seed.bin"),
+        )
+        .await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,35 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node 4-drive distributed e2e coverage.
+//!
+//! Selected by `[profile.e2e-distributed]` and run from
+//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
+//! each case starts four real `rustfs` processes.
+
+mod chaos_test;
+mod concurrency_stability_test;
+mod concurrent_data_movement_test;
+mod data_integrity_movement_test;
+mod expand_decommission_rebalance_test;
+mod extra_test;
+mod harness;
+mod object_lock_test;
+mod observability_test;
+mod replication_quota_test;
+mod s3_basic_test;
+mod s3_during_data_movement_test;
+mod site_replication_test;
+mod upgrade_test;
+mod versioning_test;
@@ -0,0 +1,219 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
+use crate::common::init_logging;
+use crate::object_lock::common::{
+    delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::operation::delete_object::DeleteObjectError;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{
+    DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
+    ObjectLockRule,
+};
+use chrono::{Duration as ChronoDuration, Utc};
+
+fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
+    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+    if code == Some("AccessDenied") {
+        Ok(())
+    } else {
+        Err(format!("{context}: expected AccessDenied, got {error:?}").into())
+    }
+}
+
+async fn expect_versioned_delete_denied(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    bypass: bool,
+    context: &str,
+) -> TestResult {
+    match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
+        Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
+        Err(error) => delete_denied(error.as_ref(), context),
+    }
+}
+
+#[tokio::test]
+async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let client = dist.client(0)?;
+    let peer = dist.client(2)?;
+    let bucket = unique_bucket("objlock");
+
+    client
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    let retain_until = Utc::now() + ChronoDuration::days(1);
+
+    let compliance_key = "compliance.bin";
+    let compliance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        compliance_key,
+        b"locked-compliance",
+        ObjectLockRetentionMode::Compliance,
+        retain_until,
+    )
+    .await?;
+
+    // Unversioned DELETE is allowed: it only creates a delete marker. WORM
+    // applies to a specific version id.
+    let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
+    assert_eq!(
+        marker.delete_marker(),
+        Some(true),
+        "unversioned DELETE on a locked object must create a delete marker"
+    );
+
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
+        .await?;
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
+
+    let governance_key = "governance.bin";
+    let governance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        governance_key,
+        b"locked-governance",
+        ObjectLockRetentionMode::Governance,
+        retain_until,
+    )
+    .await?;
+
+    expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
+        .await?;
+    delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
+    let deleted_governance = peer
+        .head_object()
+        .bucket(&bucket)
+        .key(governance_key)
+        .version_id(&governance_version)
+        .send()
+        .await
+        .expect_err("GOVERNANCE bypass must remove the retained version");
+    assert_eq!(
+        deleted_governance.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
+    );
+
+    let hold_key = "legal-hold.bin";
+    let hold_version =
+        put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let bucket = unique_bucket("default-lock");
+
+    writer
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+    put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
+
+    let key = "default-governance.bin";
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"default retention payload"))
+        .send()
+        .await?;
+    let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
+
+    let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
+    let default_retention = config
+        .object_lock_configuration()
+        .and_then(|configuration| configuration.rule())
+        .and_then(|rule| rule.default_retention())
+        .ok_or("GetObjectLockConfiguration omitted default retention")?;
+    assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    assert_eq!(default_retention.days(), Some(1));
+
+    let retention = reader
+        .get_object_retention()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
+    assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    let retain_until = retention
+        .retain_until_date()
+        .ok_or("default retention omitted retain-until date")?;
+    assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
+
+    let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
+    assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
+    expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
+        .await?;
+
+    let plain_bucket = unique_bucket("no-lock");
+    dist.create_bucket(&plain_bucket).await?;
+    let configuration = ObjectLockConfiguration::builder()
+        .object_lock_enabled(ObjectLockEnabled::Enabled)
+        .rule(
+            ObjectLockRule::builder()
+                .default_retention(
+                    DefaultRetention::builder()
+                        .mode(ObjectLockRetentionMode::Governance)
+                        .days(1)
+                        .build(),
+                )
+                .build(),
+        )
+        .build();
+    let error = writer
+        .put_object_lock_configuration()
+        .bucket(&plain_bucket)
+        .object_lock_configuration(configuration)
+        .send()
+        .await
+        .expect_err("an unversioned bucket must reject Object Lock enablement");
+    let service_error = error
+        .as_service_error()
+        .ok_or("non-lock bucket rejection was not an S3 service error")?;
+    assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
+    assert_eq!(
+        service_error.message(),
+        Some("Object Lock configuration cannot be enabled on existing buckets"),
+        "unexpected error: {error:?}"
+    );
+
+    Ok(())
+}
@@ -0,0 +1,236 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
+use crate::common::{admin_request, init_logging, local_http_client};
+use aws_sdk_s3::operation::RequestId;
+use aws_sdk_s3::primitives::ByteStream;
+use bytes::Bytes;
+use http::Method;
+use http_body_util::{BodyExt, Empty};
+use hyper::body::Incoming;
+use hyper::service::service_fn;
+use hyper::{Request, Response};
+use hyper_util::rt::TokioIo;
+use local_ip_address::local_ip;
+use rustfs_madmin::metrics::RealtimeMetrics;
+use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
+use serde_json::Value;
+use std::convert::Infallible;
+use std::time::Duration;
+use tokio::net::TcpListener;
+use tokio::sync::mpsc;
+use tokio::task::JoinHandle;
+use tokio::time::{Instant, timeout};
+
+async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
+    let listener = TcpListener::bind("0.0.0.0:0").await?;
+    let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
+    let (tx, rx) = mpsc::unbounded_channel();
+    let handle = tokio::spawn(async move {
+        loop {
+            let Ok((stream, _)) = listener.accept().await else {
+                return;
+            };
+            let tx = tx.clone();
+            tokio::spawn(async move {
+                let service = service_fn(move |request: Request<Incoming>| {
+                    let tx = tx.clone();
+                    async move {
+                        let method = request.method().clone();
+                        if let Ok(body) = request.into_body().collect().await
+                            && method == Method::POST
+                            && let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
+                        {
+                            if let Some(records) = payload["Records"].as_array() {
+                                for entry in records {
+                                    let _ = tx.send(entry.clone());
+                                }
+                            } else {
+                                let _ = tx.send(payload);
+                            }
+                        }
+                        Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
+                    }
+                });
+                let _ = hyper::server::conn::http1::Builder::new()
+                    .serve_connection(TokioIo::new(stream), service)
+                    .await;
+            });
+        }
+    });
+    Ok((endpoint, rx, handle))
+}
+
+async fn wait_for_audit_entry(
+    rx: &mut mpsc::UnboundedReceiver<Value>,
+    bucket: &str,
+    key: &str,
+    request_id: &str,
+) -> TestResult<Value> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    let mut seen = Vec::new();
+    loop {
+        let remaining = deadline.saturating_duration_since(Instant::now());
+        if remaining.is_zero() {
+            return Err(format!(
+                "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                seen.len()
+            )
+            .into());
+        }
+        let entry = match timeout(remaining, rx.recv()).await {
+            Ok(Some(entry)) => entry,
+            Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
+            Err(_) => {
+                return Err(format!(
+                    "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                    seen.len()
+                )
+                .into());
+            }
+        };
+        if entry["api"]["name"].as_str() == Some("s3:PutObject")
+            && entry["api"]["bucket"].as_str() == Some(bucket)
+            && entry["api"]["object"].as_str() == Some(key)
+            && entry["requestID"].as_str() == Some(request_id)
+        {
+            return Ok(entry);
+        }
+        if seen.len() < 8 {
+            seen.push(format!(
+                "api={:?} bucket={:?} object={:?} requestID={:?}",
+                entry["api"]["name"].as_str(),
+                entry["api"]["bucket"].as_str(),
+                entry["api"]["object"].as_str(),
+                entry["requestID"].as_str()
+            ));
+        }
+    }
+}
+
+#[tokio::test]
+async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
+    init_logging();
+    let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
+    let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
+    let audit_env = [
+        ("RUSTFS_AUDIT_ENABLE", "true"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
+        (ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
+    ];
+    let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
+        tokio::fs::create_dir_all(&queue_dir).await?;
+        dist.cluster
+            .set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
+    }
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+
+    let http = local_http_client();
+    for node in &dist.cluster.nodes {
+        for probe in ["ready", "live"] {
+            let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
+            assert!(
+                response.status().is_success(),
+                "node {} {probe} probe failed: {}",
+                node.address,
+                response.status()
+            );
+        }
+    }
+
+    let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    let info: Value = serde_json::from_str(&info_body)?;
+    let servers = info["info"]["servers"]
+        .as_array()
+        .ok_or_else(|| format!("admin info omitted servers: {info}"))?;
+    assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
+
+    let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    let storage: Value = serde_json::from_str(&storage_body)?;
+    let disks = storage["info"]["disks"]
+        .as_array()
+        .ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
+    assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
+    assert!(
+        disks.iter().all(|disk| {
+            disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
+                && disk["runtimeState"]
+                    .as_str()
+                    .is_some_and(|state| state.eq_ignore_ascii_case("online"))
+        }),
+        "storageinfo reported a drive that was not healthy and online: {storage}"
+    );
+
+    for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
+        let (status, metrics_body) = admin_request(
+            &node.url,
+            Method::GET,
+            "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
+            None,
+            &dist.cluster.access_key,
+            &dist.cluster.secret_key,
+        )
+        .await?;
+        assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
+        let sample: RealtimeMetrics = serde_json::from_str(
+            metrics_body
+                .lines()
+                .next()
+                .ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
+        )?;
+        assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
+        assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
+        assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
+    }
+
+    let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
+    let targets: Value = serde_json::from_str(&targets_body)?;
+    let configured = targets["audit_endpoints"]
+        .as_array()
+        .ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
+        .iter()
+        .any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
+    assert!(configured, "configured audit webhook was missing: {targets}");
+
+    let bucket = unique_bucket("audit");
+    dist.create_bucket(&bucket).await?;
+    let key = "correlated/audit-object.bin";
+    let put = dist
+        .client(2)?
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"distributed audit payload"))
+        .send()
+        .await?;
+    let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
+    let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
+    assert_eq!(
+        audit["api"]["status_code"].as_i64(),
+        Some(200),
+        "audit entry did not report success: {audit}"
+    );
+    assert!(
+        !audit.to_string().contains(&dist.cluster.secret_key),
+        "audit entry leaked the root secret key"
+    );
+
+    collector.abort();
+    Ok(())
+}
@@ -0,0 +1,191 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
+    set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
+};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use http::Method;
+use std::time::Duration;
+
+async fn wait_for_replication_status(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    expected: &[&str],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            let head = client.head_object().bucket(bucket).key(key).send().await?;
+            Ok(head
+                .replication_status()
+                .is_some_and(|status| expected.contains(&status.as_str())))
+        },
+        &format!("replication status for {bucket}/{key} in {expected:?}"),
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
+    init_logging();
+    let (source, mut target) = DistCluster::start_replication_pair().await?;
+    let source_bucket = unique_bucket("replsrc");
+    let target_bucket = unique_bucket("repldst");
+    source.create_bucket(&source_bucket).await?;
+    target.create_bucket(&target_bucket).await?;
+
+    let source_client = source.client(0)?;
+    let target_client = target.client(0)?;
+    enable_versioning(&source_client, &source_bucket).await?;
+    enable_versioning(&target_client, &target_bucket).await?;
+
+    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    let key = "replicated/metadata-and-tags.bin";
+    let body = b"distributed-bucket-replication".to_vec();
+    source_client
+        .put_object()
+        .bucket(&source_bucket)
+        .key(key)
+        .metadata("origin", "four-node-source")
+        .tagging("suite=distributed&shape=metadata")
+        .body(ByteStream::from(body.clone()))
+        .send()
+        .await?;
+    wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
+
+    let peer_read = target.client(3)?;
+    wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
+    let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
+    assert_eq!(
+        replica_head
+            .metadata()
+            .and_then(|metadata| metadata.get("origin"))
+            .map(String::as_str),
+        Some("four-node-source")
+    );
+    assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
+    let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
+    let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
+    assert_eq!(tags.get("suite"), Some(&"distributed"));
+    assert_eq!(tags.get("shape"), Some(&"metadata"));
+
+    target.cluster.stop();
+    let outage_key = "replicated/queued-during-target-outage.bin";
+    let outage_body = b"retry-after-target-restart".to_vec();
+    put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
+    wait_for_replication_status(
+        &source_client,
+        &source_bucket,
+        outage_key,
+        &["PENDING", "FAILED"],
+        Duration::from_secs(30),
+    )
+    .await?;
+
+    target.cluster.start().await?;
+    wait_for_ready(&target.cluster).await?;
+    wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
+    let bucket = unique_bucket("quota");
+    dist.create_bucket(&bucket).await?;
+    set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
+
+    let client = dist.client(1)?;
+    retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, body) = super::harness::cluster_admin(
+                &dist.cluster,
+                Method::GET,
+                &format!("/rustfs/admin/v3/quota-stats/{bucket}"),
+                None,
+            )
+            .await?;
+            if !status.is_success() {
+                return Ok(false);
+            }
+            let stats: serde_json::Value =
+                serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
+            let usage = stats
+                .get("current_usage")
+                .and_then(serde_json::Value::as_u64)
+                .ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
+            Ok(usage >= 1024)
+        },
+        "quota stats observe small object",
+    )
+    .await?;
+
+    let oversized_key = "too-big.bin";
+    let error = client
+        .put_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .body(vec![0u8; 16 * 1024].into())
+        .send()
+        .await
+        .expect_err("hard quota must reject the oversized PUT");
+    let service_error = error
+        .as_service_error()
+        .ok_or("quota rejection was not an S3 service error")?;
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(400),
+        "quota rejection must be HTTP 400: {error:?}"
+    );
+    assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
+    assert!(
+        service_error
+            .message()
+            .is_some_and(|message| message.starts_with("Bucket quota exceeded")),
+        "PUT must fail specifically at quota admission: {error:?}"
+    );
+
+    let missing = client
+        .head_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .send()
+        .await
+        .expect_err("an object rejected by quota must not become visible");
+    assert_eq!(
+        missing.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "quota-rejected object returned an unexpected HEAD result: {missing:?}"
+    );
+
+    let listed = client.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
+        "quota-rejected key leaked into ListObjectsV2"
+    );
+    Ok(())
+}
@@ -0,0 +1,258 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
+use crate::common::{init_logging, local_http_client};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::presigning::PresigningConfig;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3basic");
+    dist.create_bucket(&bucket).await?;
+
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let key = "dir/object.bin";
+    let body = vec![0xA5u8; 256 * 1024];
+    put_object(&writer, &bucket, key, body.clone()).await?;
+
+    let head = reader.head_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(head.content_length(), Some(body.len() as i64));
+    assert_object_bytes(&reader, &bucket, key, &body).await?;
+
+    let ranged = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .range("bytes=0-15")
+        .send()
+        .await?;
+    let ranged_body = ranged.body.collect().await?.into_bytes();
+    assert_eq!(ranged_body.as_ref(), &body[..16]);
+
+    let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
+    let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+    assert_eq!(keys, vec![key]);
+
+    let copy_key = "dir/object-copy.bin";
+    reader
+        .copy_object()
+        .bucket(&bucket)
+        .key(copy_key)
+        .copy_source(format!("{bucket}/{key}"))
+        .metadata_directive(MetadataDirective::Copy)
+        .send()
+        .await?;
+    assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
+
+    let moved_key = "dir/object-moved.bin";
+    writer
+        .copy_object()
+        .bucket(&bucket)
+        .key(moved_key)
+        .copy_source(format!("{bucket}/{copy_key}"))
+        .send()
+        .await?;
+    writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
+    match writer.head_object().bucket(&bucket).key(copy_key).send().await {
+        Ok(_) => return Err("copied source still present after rename delete".into()),
+        Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
+        Err(error) => return Err(error.into()),
+    }
+    assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
+
+    let presigned = writer
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
+        .await?;
+    let response = local_http_client().get(presigned.uri().to_string()).send().await?;
+    assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
+    let presigned_body = response.bytes().await?;
+    assert_eq!(presigned_body.as_ref(), body.as_slice());
+
+    let empty_key = "empty";
+    put_object(&writer, &bucket, empty_key, Vec::new()).await?;
+    let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
+    assert!(empty.is_empty());
+
+    let deleted = writer
+        .delete_objects()
+        .bucket(&bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key(key).build()?)
+                .objects(ObjectIdentifier::builder().key(moved_key).build()?)
+                .objects(ObjectIdentifier::builder().key(empty_key).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
+    assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
+
+    let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3matrix");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+
+    let special_key = "unicode/测试 space+percent%25.txt";
+    let special_body = b"metadata and tagging survive distributed routing".to_vec();
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .metadata("test-meta", "distributed")
+        .tagging("purpose=compatibility&scope=four-by-four")
+        .body(ByteStream::from(special_body.clone()))
+        .send()
+        .await?;
+    let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
+
+    let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
+    assert_eq!(
+        head.metadata()
+            .and_then(|metadata| metadata.get("test-meta"))
+            .map(String::as_str),
+        Some("distributed")
+    );
+    assert_eq!(head.e_tag(), Some(etag.as_str()));
+    let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
+    let actual_tags: std::collections::BTreeMap<_, _> = tags
+        .tag_set()
+        .iter()
+        .map(|tag| (tag.key().to_string(), tag.value().to_string()))
+        .collect();
+    assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
+    assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
+
+    let conditional = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .if_match(&etag)
+        .send()
+        .await?;
+    assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
+    let invalid_range = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .range("bytes=999999-1000000")
+        .send()
+        .await
+        .expect_err("an unsatisfiable range must fail");
+    assert_eq!(
+        invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("InvalidRange"),
+        "unexpected invalid-range error: {invalid_range:?}"
+    );
+
+    let upload_key = "multipart/aborted.bin";
+    let upload = writer
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
+    writer
+        .upload_part()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
+        .send()
+        .await?;
+    let pending = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
+    writer
+        .abort_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    let after_abort = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
+    let aborted_head = reader
+        .head_object()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await
+        .expect_err("aborted multipart upload must not create an object");
+    assert_eq!(
+        aborted_head.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
+    );
+
+    for index in 0..113 {
+        let key = format!("page/{index:04}.txt");
+        put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
+    }
+    let mut token = None;
+    let mut paged_keys = Vec::new();
+    loop {
+        let page = reader
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("page/")
+            .max_keys(37)
+            .set_continuation_token(token.take())
+            .send()
+            .await?;
+        paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
+        if page.is_truncated() != Some(true) {
+            break;
+        }
+        token = Some(
+            page.next_continuation_token()
+                .ok_or("truncated ListObjectsV2 page omitted next continuation token")?
+                .to_string(),
+        );
+    }
+    assert_eq!(paged_keys.len(), 113);
+    let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
+    assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
+    Ok(())
+}
@@ -0,0 +1,94 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
+    retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
+    wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("s3move");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    let live = dist.client(2)?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-decommission.bin")),
+        "list during decommission missed the newly written key"
+    );
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+    assert_inventory(&live, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let status = rebalance_status_json(&dist.cluster).await?;
+    if !rebalance_running_with_progress(&status, &rebalance_id)? {
+        return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
+    }
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,128 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
+    wait_until,
+};
+use crate::common::{init_logging, signed_request};
+use http::{Method, StatusCode};
+use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
+use std::time::Duration;
+
+async fn site_replication_add(
+    cluster: &crate::common::RustFSTestClusterEnvironment,
+    sites: &[PeerSite],
+) -> TestResult<ReplicateAddStatus> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &cluster.access_key,
+        &cluster.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let info = site_replication_info(cluster).await?;
+            Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
+        },
+        "site replication enabled with two synchronized sites",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+    let bucket = unique_bucket("siterepl");
+    site_a.create_bucket(&bucket).await?;
+    site_b.create_bucket(&bucket).await?;
+
+    let client_a = site_a.client(0)?;
+    let client_b = site_b.client(0)?;
+    enable_versioning(&client_a, &bucket).await?;
+    enable_versioning(&client_b, &bucket).await?;
+
+    let sites = vec![
+        PeerSite {
+            name: "site-a".to_string(),
+            endpoint: site_a.cluster.nodes[0].url.clone(),
+            access_key: site_a.cluster.access_key.clone(),
+            secret_key: site_a.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+        PeerSite {
+            name: "site-b".to_string(),
+            endpoint: site_b.cluster.nodes[0].url.clone(),
+            access_key: site_b.cluster.access_key.clone(),
+            secret_key: site_b.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+    ];
+    let add_status = site_replication_add(&site_a.cluster, &sites).await?;
+    assert!(
+        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
+        "site replication add reported failure: {add_status:?}"
+    );
+    wait_for_site_replication_enabled(&site_a.cluster).await?;
+    wait_for_site_replication_enabled(&site_b.cluster).await?;
+
+    let info_a = site_replication_info(&site_a.cluster).await?;
+    let remote = info_a
+        .sites
+        .iter()
+        .find(|site| site.name == "site-b")
+        .ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
+    assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
+    let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
+    assert!(
+        deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
+        "site peers must have two distinct non-empty deployment IDs: {info_a:?}"
+    );
+    assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
+    assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
+
+    let key = "site-object.bin";
+    let body = b"four-node-site-replication".to_vec();
+    put_object(&client_a, &bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
+
+    let peer_b = site_b.client(3)?;
+    wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
+
+    let reverse_key = "reverse/site-object.bin";
+    let reverse_body = b"site-b-to-site-a".to_vec();
+    put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
+    wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
+    Ok(())
+}
@@ -0,0 +1,345 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node upgrade coverage for historical objects and IAM AK/SK.
+//!
+//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
+//! mixed-version listing). This module pins the distributed contract the
+//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
+//! written on the previous release still read back, and IAM user credentials
+//! created before the upgrade still authenticate.
+//!
+//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
+//! release. The `e2e-distributed` workflow downloads that binary; a local run
+//! without it fails closed rather than skipping.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
+    unique_bucket, wait_until,
+};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use uuid::Uuid;
+
+const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const IAM_SECRET: &str = "UpgradeTestSecretKey1";
+const WRONG_SECRET: &str = "WrongSecretKey000000";
+const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct UpgradeSeed {
+    history_bucket: String,
+    history_key: &'static str,
+    history_body: Vec<u8>,
+    versioned_bucket: String,
+    versioned_key: &'static str,
+    version1: String,
+    version1_body: Vec<u8>,
+    version2: String,
+    version2_body: Vec<u8>,
+    iam_bucket: String,
+    iam_key: &'static str,
+    iam_body: Vec<u8>,
+    iam_user: String,
+    iam_secret: &'static str,
+}
+
+fn source_binary() -> TestResult<PathBuf> {
+    let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
+        format!(
+            "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
+        )
+    })?;
+    if !path.is_file() {
+        return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
+    }
+    Ok(path)
+}
+
+fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
+        cluster
+            .cluster
+            .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+fn iam_rw_policy(bucket: &str) -> String {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}"),
+                format!("arn:aws:s3:::{bucket}/*")
+            ]
+        }]
+    })
+    .to_string()
+}
+
+async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
+    let url = &dist.cluster.nodes[0].url;
+    let access = &dist.cluster.access_key;
+    let admin_secret = &dist.cluster.secret_key;
+    admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
+    admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
+    admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
+    Ok(())
+}
+
+async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                put_object(&client, &bucket, &key, body).await?;
+                Ok(true)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let expected = expected.to_vec();
+            async move {
+                let got = get_object_bytes(&client, &bucket, &key).await?;
+                Ok(got == expected)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
+    let history_bucket = unique_bucket("upg-hist");
+    let versioned_bucket = unique_bucket("upg-ver");
+    let iam_bucket = unique_bucket("upg-iam");
+    dist.create_bucket(&history_bucket).await?;
+    dist.create_bucket(&versioned_bucket).await?;
+    dist.create_bucket(&iam_bucket).await?;
+
+    let root = dist.client(0)?;
+    enable_versioning(&root, &versioned_bucket).await?;
+
+    let history_key = "plain-history.bin";
+    let history_body = b"written by the previous 4-node release".to_vec();
+    put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
+
+    let versioned_key = "versioned-history.txt";
+    let version1_body = b"version-one-before-upgrade".to_vec();
+    let version1 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("first versioned PUT omitted version ID")?
+        .to_string();
+    let version2_body = b"version-two-before-upgrade".to_vec();
+    let version2 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("second versioned PUT omitted version ID")?
+        .to_string();
+
+    let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
+
+    let iam_key = "iam-history.bin";
+    let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
+    let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
+    wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
+
+    Ok(UpgradeSeed {
+        history_bucket,
+        history_key,
+        history_body,
+        versioned_bucket,
+        versioned_key,
+        version1,
+        version1_body,
+        version2,
+        version2_body,
+        iam_bucket,
+        iam_key,
+        iam_body,
+        iam_user,
+        iam_secret: IAM_SECRET,
+    })
+}
+
+async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
+    let root_a = dist.client(0)?;
+    let root_b = dist.client(3)?;
+    wait_for_bytes(
+        &root_b,
+        &seed.history_bucket,
+        seed.history_key,
+        &seed.history_body,
+        &format!("{context}: root GET historical object"),
+    )
+    .await?;
+    assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
+
+    let v1 = root_b
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version1)
+        .send()
+        .await?;
+    let v1_body = v1.body.collect().await?.into_bytes();
+    if v1_body.as_ref() != seed.version1_body.as_slice() {
+        return Err(format!("{context}: version 1 bytes changed after upgrade").into());
+    }
+    let v2 = root_a
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version2)
+        .send()
+        .await?;
+    let v2_body = v2.body.collect().await?.into_bytes();
+    if v2_body.as_ref() != seed.version2_body.as_slice() {
+        return Err(format!("{context}: version 2 bytes changed after upgrade").into());
+    }
+
+    let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    if !users.contains(&seed.iam_user) {
+        return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
+    }
+
+    let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
+    let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
+    wait_for_bytes(
+        &iam_on_upgraded,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 0"),
+    )
+    .await?;
+    wait_for_bytes(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 3"),
+    )
+    .await?;
+
+    let post_key = format!("after-upgrade-{context}.txt");
+    let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
+    wait_for_put(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        &post_key,
+        post_body.clone(),
+        &format!("{context}: IAM PUT after upgrade"),
+    )
+    .await?;
+    assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
+
+    let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
+    match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
+        Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
+        Err(error) => {
+            let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+            let rejected = code == Some("SignatureDoesNotMatch")
+                || code == Some("InvalidAccessKeyId")
+                || code == Some("AccessDenied")
+                || code == Some("InvalidArgument")
+                || error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
+            if !rejected {
+                return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
+            }
+        }
+    }
+
+    let post_root_key = format!("root-after-{context}.bin");
+    let post_root_body = format!("{context}: root write after upgrade").into_bytes();
+    put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
+    assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "direct-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+    dist.restart_with_current_binary().await?;
+    assert_history_and_iam(&dist, &seed, "direct").await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+
+    dist.replace_node_with_current_binary(0).await?;
+    assert_history_and_iam(&dist, &seed, "one-current-node").await?;
+
+    for node_idx in [1, 2] {
+        dist.replace_node_with_current_binary(node_idx).await?;
+    }
+    assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
+
+    dist.replace_node_with_current_binary(3).await?;
+    assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
+    Ok(())
+}
@@ -0,0 +1,188 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+
+#[tokio::test]
+async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("version");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "versioned.txt";
+    let v1_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v1".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v1 PUT omitted version ID")?
+        .to_string();
+    let v2_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v2".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v2 PUT omitted version ID")?
+        .to_string();
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_versions: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
+    assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
+        "v1 was not the historical version: {versions:?}"
+    );
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
+        "v2 was not the latest version: {versions:?}"
+    );
+
+    let latest = get_object_bytes(&reader, &bucket, key).await?;
+    assert_eq!(latest, b"v2");
+
+    let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let older_body = older.body.collect().await?.into_bytes();
+    assert_eq!(older_body.as_ref(), b"v1");
+
+    let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(deleted.delete_marker(), Some(true));
+    let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
+    let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_markers: Vec<_> = after_delete
+        .delete_markers()
+        .iter()
+        .filter(|marker| marker.key() == Some(key))
+        .collect();
+    assert_eq!(
+        matching_markers.len(),
+        1,
+        "delete marker missing or duplicated after current-version delete: {after_delete:?}"
+    );
+    assert!(
+        matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
+        "DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
+    );
+
+    let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
+    match latest_after_delete {
+        Ok(_) => return Err("current version should be a delete marker".into()),
+        Err(error)
+            if error
+                .as_service_error()
+                .and_then(ProvideErrorMetadata::code)
+                .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
+        Err(error) => return Err(error.into()),
+    }
+
+    let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let restored_body = restored.body.collect().await?.into_bytes();
+    assert_eq!(restored_body.as_ref(), b"v1");
+
+    writer
+        .delete_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(marker_id)
+        .send()
+        .await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("suspend");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "suspended.txt";
+    let original = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"enabled-history".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("enabled PUT omitted version ID")?
+        .to_string();
+    writer
+        .put_bucket_versioning()
+        .bucket(&bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Suspended)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
+    put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
+    let null_version_count = matching
+        .iter()
+        .filter(|version| {
+            matches!(
+                version.version_id(),
+                None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
+            )
+        })
+        .count();
+    assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
+
+    let historical = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(&original)
+        .send()
+        .await?;
+    assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
+    Ok(())
+}
@@ -16,18 +16,15 @@

 #[cfg(test)]
 mod tests {
-    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
+    use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
    use crate::common::{
        FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
-        rustfs_binary_path,
    };
    use crate::storage_api::RUSTFS_META_BUCKET;
    use aws_sdk_s3::primitives::ByteStream;
    use http::Method;
-    use sha2::{Digest, Sha256};
    use std::collections::HashSet;
    use std::error::Error;
-    use std::io::{Read, Write};
    use std::net::SocketAddr;
    use std::path::{Path, PathBuf};
    use std::process::Command;
@@ -37,76 +34,6 @@ mod tests {

    const POOL_METADATA_OBJECT: &str = "pool.bin";

-    #[derive(serde::Deserialize)]
-    struct EvidenceBuild {
-        sha256: String,
-    }
-
-    #[derive(serde::Deserialize)]
-    struct RestartEvidenceRun {
-        schema: u32,
-        run_id: String,
-        source_revision: String,
-        test_build: serde_json::Value,
-        binary: EvidenceBuild,
-        test_binary: EvidenceBuild,
-    }
-
-    fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
-        let mut file = std::fs::File::open(path)?;
-        let mut digest = Sha256::new();
-        let mut buffer = [0_u8; 64 * 1024];
-        loop {
-            let read = file.read(&mut buffer)?;
-            if read == 0 {
-                break;
-            }
-            digest.update(&buffer[..read]);
-        }
-        Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
-    }
-
-    fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
-        let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
-            return Ok(None);
-        };
-        let directory = PathBuf::from(directory);
-        let receipt = directory.join("run.json");
-        if receipt.metadata()?.len() > 1024 * 1024 {
-            return Err("oversized scanner/heal execution receipt".into());
-        }
-        let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
-        if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
-            return Err("invalid scanner/heal execution identity".into());
-        }
-        let built = compiled_test_identity();
-        for key in ["source_revision", "dirty", "lock_blob", "features"] {
-            assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
-        }
-        assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
-        assert_eq!(
-            file_sha256(&std::env::current_exe()?)?,
-            run.test_binary.sha256,
-            "test executable must match the run receipt"
-        );
-        if directory.join("background-target-restart.json").exists() {
-            return Err("scanner/heal oracle already exists; create a new execution receipt".into());
-        }
-        Ok(Some((directory, run)))
-    }
-
-    fn compiled_test_identity() -> serde_json::Value {
-        serde_json::json!({
-            "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
-            "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
-            "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
-            "features": env!("RUSTFS_E2E_BUILD_FEATURES"),
-            "target": env!("RUSTFS_E2E_BUILD_TARGET"),
-            "profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
-            "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
-        })
-    }
-
    struct TcpPortBlackhole {
        port: u16,
        comment: String,
@@ -268,9 +195,8 @@ mod tests {
        clients: &[aws_sdk_s3::Client],
        bucket: &str,
        expected_keys: &HashSet<String>,
-    ) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
+    ) -> Result<(), Box<dyn Error + Send + Sync>> {
        const PAGE_SIZE: i32 = 10;
-        let mut node_listings = Vec::with_capacity(clients.len());
        for (node_index, client) in clients.iter().enumerate() {
            let mut listed_keys = Vec::new();
            let mut continuation_token = None;
@@ -317,10 +243,8 @@ mod tests {
                &listed_key_set, expected_keys,
                "node {node_index} did not expose the complete recovered namespace"
            );
-            listed_keys.sort();
-            node_listings.push(listed_keys);
        }
-        Ok(node_listings)
+        Ok(())
    }

    fn heal_task_status_diagnostic(body: &str) -> String {
@@ -884,13 +808,6 @@ mod tests {
    }

    async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
-        let server_binary = rustfs_binary_path();
-        let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
-            restart_evidence_run(&server_binary)?
-        } else {
-            None
-        };
-        let mut evidence_objects = Vec::new();
        let (background_enabled, interruption_node, interruption_kind) = match scenario {
            InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
            InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
@@ -938,7 +855,7 @@ mod tests {
        for node_index in 0..cluster.nodes.len() {
            cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
        }
-        cluster.start_with_binary(&server_binary).await?;
+        cluster.start().await?;
        let clients = cluster.create_all_clients()?;

        let bucket = "heal-restart-during-rebuild";
@@ -1079,7 +996,7 @@ mod tests {
            }
        }

-        cluster.start_node_from_binary(1, &server_binary).await?;
+        cluster.start_node(1).await?;

        let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
        let recovery_deadline = Instant::now() + Duration::from_secs(60);
@@ -1357,7 +1274,7 @@ mod tests {
                    }
                }
            }
-            cluster.start_node_from_binary(interruption_node, &server_binary).await?;
+            cluster.start_node(interruption_node).await?;
            if interruption_node == 0 {
                let target = cluster.nodes[1]
                    .process
@@ -1456,7 +1373,7 @@ mod tests {
            .map(|manifest| manifest.key.clone())
            .collect::<HashSet<_>>();
        assert!(expected_keys.insert(outage_key.to_string()));
-        let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
+        assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;

        let target_client = cluster.create_s3_client(1)?;
        for expected in &expected_manifests {
@@ -1464,31 +1381,11 @@ mod tests {
            let actual = response.body.collect().await?.into_bytes();
            let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
            assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
-            if evidence_run.is_some() {
-                evidence_objects.push(serde_json::json!({
-                    "key": expected.key, "version_id": expected.shard_census.version_id,
-                    "expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
-                    "expected_sha256": sha256_hex(&expected_body),
-                    "actual_sha256": sha256_hex(&actual),
-                    "expected_physical": expected.shard_census,
-                    "physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
-                }));
-            }
        }
        let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
        let actual = response.body.collect().await?.into_bytes();
        let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
        assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
-        if evidence_run.is_some() {
-            evidence_objects.push(serde_json::json!({
-                "key": outage_key, "version_id": null,
-                "expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
-                "expected_sha256": sha256_hex(&expected_outage_body),
-                "actual_sha256": sha256_hex(&actual),
-                "expected_physical": null,
-                "physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
-            }));
-        }

        let terminal_deadline = Instant::now() + Duration::from_secs(30);
        loop {
@@ -1535,31 +1432,6 @@ mod tests {
            return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
        }

-        if let Some((directory, run)) = evidence_run {
-            let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
-            assert_ne!(target_pid, restarted_pid, "target must be a new process");
-            assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
-            let evidence = serde_json::json!({
-                "schema": 1, "case": "background-target-restart", "evidence": "process-restart",
-                "run_id": run.run_id, "source_revision": run.source_revision,
-                "test_build": compiled_test_identity(),
-                "binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
-                "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
-                "pid_before": target_pid, "pid_after": restarted_pid,
-                "objects": evidence_objects, "node_listings": node_listings,
-            });
-            let data = serde_json::to_vec(&evidence)?;
-            if data.len() > 1024 * 1024 {
-                return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
-            }
-            let mut output = std::fs::OpenOptions::new()
-                .write(true)
-                .create_new(true)
-                .open(directory.join("background-target-restart.json"))?;
-            output.write_all(&data)?;
-            output.sync_all()?;
-        }
-
        Ok(())
    }

@@ -378,6 +378,11 @@ mod bucket_stats_regression_test;
 #[cfg(test)]
 mod distributed_startup_regression_test;

+// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
+// quota, observability, expand/decommission/rebalance, site replication, chaos).
+#[cfg(test)]
+mod distributed;
+
 // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
 #[cfg(test)]
 mod tier_transition_regression_test;
@@ -34,149 +34,6 @@ The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/

 ## Test Matrix

-### Formal Scanner/Heal ABBA
-
-The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`,
-`multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least
-three A1/B1/B2/A2 groups for both baseline/candidate with background work on,
-and candidate-only background off/on. A measured leg lasts at least 900
-seconds; the minimum matrix contains 120 legs (30 hours before setup/oracles).
-The existing `performance-ab.yml` supplies the pattern for immutable build
-provenance and failure propagation, but its short Warp workload is not this
-scanner gate. No scheduled workflow starts this matrix automatically.
-
-```bash
-scripts/run_scanner_validation_harness.sh --abba \
-  --manifest scanner-abba.json --adapter /path/to/isolated-deployment-adapter \
-  --out-dir /path/to/new-artifacts --data-root /path/to/new-test-data
-```
-
-Both roots must be new and non-overlapping. Every leg receives a unique data
-directory. The runner checks disk capacity before each leg, never removes data,
-and stops the adapter after success or failure. Retain raw artifacts and inspect
-task ownership before removing any test data. The operator must reserve the
-target machines and map the assigned directory to separate data paths on every
-node; the runner cannot prove remote isolation from local path names.
-
-The manifest has the following JSON contract (all fields are required):
-
-| Field | Value |
-|---|---|
-| `schema`, `evidence` | `1`, and `measured` or `synthetic`. |
-| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. |
-| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. |
-| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. |
-| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. |
-| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. |
-
-Record exact build flags and effective durability settings, not just defaults.
-Use deterministic workload seeds so every isolated leg has the same expected
-object/version/content result. Fix the foreground arrival rate (offered load),
-cache preparation procedure, configuration, and hardware across every leg.
-Do not include credentials in the manifest, adapter output, or saved commands;
-the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment.
-
-#### Deployment Adapter Contract
-
-The runner invokes an executable as `adapter ACTION request.json response.json`
-with no shell evaluation. Actions are separate processes: `prepare`, `measure`,
-`oracle`, and `stop`. Every action must return zero and write a JSON object of
-at most 1 MiB. Logs are kept separately and require an operator-managed disk
-quota. Missing output, timeout, nonzero exit, unknown/missing metrics, zero
-samples, and request errors fail the run. Adapters must terminate their own
-children on failure and `stop` must be idempotent even after partial preparation.
-The runner keeps its session leader unreaped while stopping a failed command
-or collector: it sends TERM, allows the existing ten-second grace period, then
-kills the remaining process group before reaping. This prevents a parent exit
-from hiding live descendants or allowing the group ID to be reused before its
-last signal. A successful `prepare` preserves adapter-owned services until
-`stop`; services that leave the command's process group remain the adapter's
-cleanup responsibility.
-
-The request contains the fixed manifest fields, selected build, scenario, round,
-leg, comparison (`build` or `background`), background mode (`on` or `off`),
-duration, unique `data_dir`, expected object oracle, and expected repair count.
-Adapter responsibilities:
-
-1. `prepare` deploys the selected binary into an authorized isolated topology,
-   checks actual binary/config/durability, initializes deterministic scenario
-   data and the requested cache state, and returns `{"ready": true}`. For measured
-   runs it also returns `collector` with exactly `alias`, `endpoint`, and
-   comma-separated `metrics_endpoints`; the runner starts the existing scanner
-   collector at 60-second cadence while `measure` runs.
-2. `measure` maintains the fixed offered load for the entire requested duration.
-   `cold-hot` retains cold buckets while mutating a hot bucket; `fresh-hot`
-   creates a bucket after scanner startup; `multi-hot-new` combines several hot
-   buckets with a newly created bucket; `running-heal` applies foreground load
-   during active repair; `mrf-replay` replays independently seeded durable repair
-   work. Capture same-window status for bucket-freshness issue #7108. Actual
-   fault injection and dataset generation belong to the reviewed adapter.
-3. `oracle` independently enumerates all objects and versions, reads and checks
-   their complete bytes, and verifies repairs. Return `complete: true`, integer
-   `errors: 0`, and `actual` matching the manifest's expected oracle. Never copy
-   expected values into a measured oracle or infer completion from empty queues.
-4. `stop` stops task-owned workload/server processes and returns `stopped: true`.
-   Preserve data and artifacts for diagnosis. An adapter may restore previous
-   settings but must not delete arbitrary paths or stop unrelated deployments.
-
-The `measure` response echoes the observed `evidence`, `fixed`, `build`,
-`data_dir`, and `background`, plus `sample_count` (1..3600), `elapsed_seconds`,
-and `metrics`. All metrics must be finite nonnegative numbers: `p99_ms`,
-`throughput_ops`, `rss_bytes`, `cpu_seconds`, `iops`, `rpc_count`,
-`cache_clone_bytes`, `encode_bytes`, `save_bytes`, `oldest_age_seconds`,
-`walk_objects`, `cold_walk_objects`, `healed_objects`, `errors`, and `requests`.
-Requests, throughput, and p99 must be positive; errors must be zero. Repair
-counts must match the manifest when background work is on. Keep underlying
-request samples, counter reset checks, profiler captures, and per-node telemetry
-in the cell artifact directory; aggregate values alone do not establish their
-measurement provenance. Missing production instrumentation is a pending gate,
-not permission to report a fabricated zero.
-
-For P2, `measure.convergence` contains booleans `writes_stopped`,
-`last_mutation_observed`, `first_complete_publication`; numeric
-`last_mutation_time`, `last_mutation_observed_time`, `writes_stopped_time`, `window_start`, `window_end`,
-`budget_available_seconds`, `walk_objects`, and `full_walk_objects`. Times use
-one monotonic clock. The window starts after writes stop and the final mutation
-is observed, and ends at the first complete publication. The reference is an
-independent full walk of the same static namespace. Record available budget
-seconds to interpret elapsed time. During continuing writes, omit this proof
-and report useful-work ratio and justified invalidation/re-scan work separately;
-the runner reports P2 pending and does not impose a fixed cumulative walk bound.
-
-The nightly heal workflow clones **`rustfs/auto-testing`** separately and invokes
-`auto-testing/rustfs_heal_test.sh`; that script is not a local `scripts/test`
-entry point. If an adapter uses it, record and verify the external checkout's
-owner and full commit before use. The current workflow clones the default branch,
-so its contents must not be attributed to a RustFS source SHA.
-
-#### Evidence Gates
-
-`report.json` records each group's verdict and the raw responses remain in their
-cell directories. Candidate/build p99 regression must be at most 5% and
-throughput loss at most 3%; candidate background on/off limits are 10% and 5%.
-P1 requires cold-hot walk reduction of at least the baseline cold-walk share
-times 80%, rather than a fixed 80% reduction for every workload. P2 requires
-candidate post-stop work at most 1.2 times the independent full-walk reference.
-Missing candidate convergence proof yields `inconclusive`. A2/A1 or B2/B1 p99
-or throughput drift above 5% also yields `inconclusive`, with exit code 3.
-Correctness errors and non-noisy performance regressions exit 1. Every group
-must pass; a favorable median cannot hide a failing group.
-
-Synthetic success is explicitly `synthetic_validated`, with `performance:
-pending`. It validates orchestration and gate logic only. It proves no runtime,
-distributed, crash, mixed-version, or performance behavior and cannot close the
-performance acceptance gate. Run the fake-adapter self-tests with:
-
-```bash
-scripts/test_scanner_validation_harness.sh
-```
-
-They cover the complete 120-cell schedule, data isolation, missing builds and
-oracles, zero samples/requests, swallowed request errors, offered-load drift,
-incomplete repairs, missing metrics, noise, and P1/P2/p99 regressions. A real
-deployment adapter and actual ABBA artifacts remain required before any measured
-performance or release claim.
-
 Collect at least two runs on the same RustFS commit and the same workload. Keep hardware, commit, object count, object size, bucket count, scanner-enabled state, and foreground workload constant between runs.

 | Run | Purpose | Example scanner settings |
@@ -12,10 +12,10 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
 |---|---|---|---|
 | Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p <crate>`); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) |
 | ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) |
-| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) |
+| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`, `e2e-distributed`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); 4-node 4-disk map: [distributed-e2e.md](distributed-e2e.md) |
 | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` | PR, report-only (second half of the `End-to-End Tests` job) |
 | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only |
-| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree |
+| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` |
 | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus |
 | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p <crate>` | On demand; never a gate |

@@ -63,6 +63,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the
 | `e2e-full` | Merge-queue / main-push single-node e2e lane |
 | `e2e-repl-nightly` | Nightly slow / cross-process replication lane |
 | `e2e-nightly` | Nightly serial multi-process cluster fault lane |
+| `e2e-distributed` | Storage-sensitive PR and nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
 | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` |

 Membership of each e2e profile is pinned by a digest in `.config/e2e-<profile>-selection.txt` and checked by `scripts/check_test_wiring.py --check-profile <profile>` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only):
@@ -74,6 +74,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` |
 | `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
+| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` |
 | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
 | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
 | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |
@@ -125,108 +126,3 @@ The manifest records a minimum set of invariants: write quorum, metadata rollbac
 The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability.

 Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review.
-## Scanner/Heal Evidence Receipts
-
-The existing `scripts/check_test_wiring.py` also validates Scanner/Heal case
-evidence registered in `.config/scanner-heal-required-tests.json`. It records
-already-built binaries and checks existing nextest output; it does not build,
-run tests, deploy servers, inject faults, or start another CI lane.
-
-The initial case is `background-target-restart`, emitted by
-`heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart`.
-That test already runs in `e2e-nightly`. When `RUSTFS_SCANNER_HEAL_RUN_DIR` is set,
-it checks the actual server and test-executable hashes against `run.json`, pins
-the same server binary for all node starts, and writes its oracle only after
-the real assertions pass. The artifact contains the actual pre/post target
-PIDs, per-node S3 listings, expected and downloaded complete-body hashes/lengths,
-and target-disk `VersionShardCensus` fingerprints. Existing baseline objects
-must match their pre-fault physical manifests; the object created during the
-outage has no pre-fault target shard and is checked for complete physical parts
-and exact S3 content.
-
-This case is a **four-node, one-drive-per-node process-restart test**. It is not
-power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof
-of scanner enumeration, exact MRF disposition, legacy migration, or rollback.
-The registry keeps all G01-G14/P1-P4 and R-E/R-D/R-L release requirements pending
-until their actual feature-specific oracles and required topologies exist.
-Missing cases cannot be supplied by synthetic W20 results. W20's bounded JSON
-and file-hash helpers are reused; its ABBA performance contracts remain in
-`docs/operations/scanner-benchmark-runbook.md`.
-
-### Recording One Case
-
-Use a committed source tree, independently built current binaries, sufficient
-free disk space, and a task-owned artifact directory that does not yet exist.
-Set `SERVER_BINARY` and `TEST_BINARY` to those exact executable paths. The begin
-command requires the server's embedded `--version` commit to match the clean
-checkout and its embedded Git status to be clean. The E2E crate's build script
-embeds its build-time Git revision/dirty state, lockfile Git blob, enabled crate
-features, target, profile and encoded Rust flags. It tracks the crate/dependency
-trees, Cargo inputs and Git HEAD/ref/index, including `common.rs` restart logic.
-The producer checks this compiled identity against the receipt; it does not
-copy a current source revision into an older test binary's identity. The E2E
-uses its existing temporary cluster directories and cleanup. `CARGO_TARGET_DIR`
-controls compilation output; nextest's default report store remains the
-workspace's `target/nextest`. Execute the existing selected case as follows:
-
-```bash
-CASE=background-target-restart
-FILTER='test(test_cluster_root_heal_recovers_remote_shards_after_background_target_restart)'
-RUN_DIR="$PWD/artifacts/scanner-heal-run"
-export RUSTFS_E2E_EXPECTED_FEATURES=default
-scripts/python_bin.sh scripts/check_test_wiring.py \
-  --begin-scanner-heal "$RUN_DIR" "$SERVER_BINARY" "$TEST_BINARY"
-export RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR"
-export CARGO_BIN_EXE_rustfs="$SERVER_BINARY"
-cargo nextest list --profile e2e-nightly -p e2e_test -E "$FILTER" \
-  --message-format json > "$RUN_DIR/listing.json"
-rm -f target/nextest/e2e-nightly/junit.xml
-set +e
-cargo nextest run --profile e2e-nightly -p e2e_test -E "$FILTER"
-test_exit=$?
-set -e
-cp target/nextest/e2e-nightly/junit.xml "$RUN_DIR/junit.xml"
-scripts/python_bin.sh scripts/check_test_wiring.py --finish-scanner-heal "$RUN_DIR" "$test_exit"
-scripts/python_bin.sh scripts/check_test_wiring.py --check-scanner-heal "$RUN_DIR" "$CASE"
-```
-
-Set `RUSTFS_E2E_EXPECTED_FEATURES` to the actual intended e2e crate feature set,
-including `default` for a default-feature build, comma-separated for extra
-features, or empty for `--no-default-features`. It is mandatory when beginning
-a run. Crate features are distinct from the spawned server's build features.
-
-Do not replace a nonzero command exit with zero. Missing JUnit or an oracle
-emission failure also fails acceptance. Each retry needs a new run directory;
-the producer refuses to overwrite an existing oracle. Keep failed-run logs and
-artifacts. The receipt pins source revision, actual binary hashes, run identity,
-start/finish times, and the artifact hashes. `listing.json`, `junit.xml`, and
-each oracle are limited to 1 MiB; object evidence has the fixture's 9..65 object
-bound. Credentials are not included in the receipt.
-
-The checker binds nextest's flattened suite `binary-id`/`binary-path` to the
-actual test executable and requires the JUnit testcase's embedded execution
-timestamp to fall inside the receipt window (with millisecond precision).
-Copying an old JUnit file and refreshing its mtime does not make it new evidence.
-Schema versions, topology counts, PIDs, EC geometry and shard indices require
-actual integers: booleans and fractional values are rejected, and an index must
-fit the physical data-plus-parity geometry.
-
-The checker rejects unselected/ignored tests, zero/duplicate JUnit cases,
-failures, skipped tests, retry/flaky records, stale or changed artifacts,
-different builds or run IDs, unchanged process IDs, wrong topology, missing
-shard parts, and mismatched S3 content/listings. The raw oracle JSON is emitted
-by the real E2E producer, not accepted from an adapter copying expectations.
-
-`--check-scanner-heal "$RUN_DIR" release` checks available case evidence and
-returns nonzero for every pending release requirement. A focused case pass
-does not approve release. In particular, R-E requires fixed-budget real
-restarts without an unbudgeted final sweep, R-D requires the full
-manager/event/ledger disposition chain, and R-L requires source-conflict and
-crash/retirement evidence. Reader-only or unit fixtures cannot substitute for
-these. The external `rustfs/auto-testing` functional workflows propagate suite
-failures. Their workflow status does not establish this registry's required
-case coverage, build provenance, or object-level oracles.
-
-Run parser/receipt regressions with
-`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures
-validate the checker only and produce no runtime or performance evidence.
@@ -0,0 +1,80 @@
+# Distributed 4-node 4-disk e2e
+
+**Use this when:** adding or diagnosing GitHub Actions coverage for a 4-node cluster, or deciding whether a behaviour belongs in `e2e-distributed` versus the single-node `e2e-full` lane, the nightly cluster-fault lane, or the hardware functional chain.
+**Source of truth:** `crates/e2e_test/src/distributed/`, `[profile.e2e-distributed]` in `.config/nextest.toml`, `.github/workflows/e2e-distributed.yml`.
+
+## Topology
+
+The in-tree harness runs every node on `127.0.0.1` with a distinct port. That matches `RustFSTestClusterEnvironment` in `crates/e2e_test/src/common.rs`:
+
+| Layout | Constructor | Use |
+|---|---|---|
+| 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos |
+| 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release |
+| 1 single-node pool × 4 drives, then `append_single_node_pool` three times | expansion seed | Pool expand, then decommission / rebalance / integrity |
+
+A multi-pool layout in which any pool spans several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped expansion pools remain the hardware functional-chain / backlog #1313 / #1314 lane.
+
+Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran.
+
+The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated 1 GiB tmpfs filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use ext4 loop devices: the `sm-standard-4` ARC pods have no `/dev/loop-control`, so `mount -o loop` fails with `No such file or directory`. Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata.
+
+The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing.
+
+## What this lane covers
+
+`cargo nextest run --profile e2e-distributed -p e2e_test` selects `distributed::*`:
+
+- S3 put / get / head / list / copy / rename / delete / presign, range and conditional reads, special keys, metadata, tags, pagination, empty objects, multipart complete and abort
+- Object Lock COMPLIANCE, GOVERNANCE and bypass, legal hold, bucket default retention, and non-lock bucket rejection
+- Versioning, exact historical reads, delete-marker removal, and suspended null-version overwrite semantics
+- Bucket replication between two 4-node clusters, including metadata/tags and target-outage retry; hard quota admission and absence of rejected keys
+- Ready/live probes on every node, exact 4-server/16-disk inventory, realtime metrics on every node, and correlated audit-webhook delivery
+- Pool expand, decommission, rebalance, checksum integrity, versioned and multipart data, and S3 during active movement
+- Bidirectional site-replication convergence plus enabled/synchronized peer state on both sites
+- A 24-worker mixed PUT/HEAD/GET/COPY/DELETE workload; concurrent PUT during active decommission
+- Node kill/restart, full process restart, node-facing TCP blackhole/recovery, in-flight streaming GETs across a peer kill, and fresh-drive replacement verified by physical `xl.meta`/part-shard census
+- Multipart, cross-node listing, list-buckets agreement
+- Direct and rolling upgrade from the pinned previous release: historical objects, versioned history, and IAM user AK/SK still work afterwards
+
+## Existing Actions gaps this lane does not replace
+
+Those suites stay in place; this lane fills the in-tree 4×4 hole they leave.
+
+| Existing lane | Gap |
+|---|---|
+| `rustfs-*-test.yml` functional chain | Clones private `rustfs/auto-testing`, runs on three shared VMs (`vm000`–`vm002`), `continue-on-error: true`, not a merge signal, not 4 nodes. Hardware `rustfs-upgrade-test.yml` stays there |
+| `e2e-upgrade.yml` | Single-node SSE/multipart/delete-marker contracts plus mixed-version listing; does not pin IAM user AK/SK on a 4-node cluster |
+| `e2e-smoke` / `e2e-full` | Most selected cases are single-node; distributed modules are intentionally owned by this serialized lane |
+| `e2e-nightly` | 4-node cluster faults and heal, not S3/lock/versioning/quota/decommission matrix |
+| `e2e-repl-nightly` | Site and bucket replication on 1–3 *single-node* processes |
+| `e2e-s3tests.yml` `multi` | Weekly ceph/s3-tests against Docker 4-node; not lock/WORM, decommission, chaos, or checksum integrity |
+| `crates/e2e_test/src/chaos.rs` | Single-node disk faults only |
+
+Hardware power-loss, physical NIC pull, authenticated inter-node partition, firmware/media errors, and replacement-server provisioning still belong on the hardware validation VMs. This lane provides deterministic process kill, fresh local-volume replacement, and node-facing TCP blackhole analogues; it does not claim physical fault certification.
+
+## Run
+
+```bash
+cargo build -p rustfs --bins
+# Expansion/decommission/rebalance cases require four paths on distinct filesystems.
+# If you do not already have four disks, sized tmpfs is enough:
+#   for p in 0 1 2 3; do
+#     sudo mkdir -p /mnt/rustfs-pool-$p
+#     sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs /mnt/rustfs-pool-$p
+#   done
+export RUSTFS_E2E_POOL_ROOTS=/mnt/rustfs-pool-0:/mnt/rustfs-pool-1:/mnt/rustfs-pool-2:/mnt/rustfs-pool-3
+# Upgrade cases require the pinned previous binary (CI downloads it).
+export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2
+cargo nextest run --profile e2e-distributed -p e2e_test
+```
+
+Without `RUSTFS_UPGRADE_SOURCE_BINARY` the two `distributed::upgrade_test::*` cases fail closed. Without four distinct `RUSTFS_E2E_POOL_ROOTS`, the expansion and data-movement cases fail closed. Filter upgrades out for a local run that is not checking upgrade:
+
+```bash
+cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distributed::upgrade_test::/)'
+```
+
+The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check.
+
+Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linux and Darwin entries with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed <listing.json> <platform>` after adding or renaming a case.
@@ -712,8 +712,6 @@ mod tests {
                    session_token: None,
                }),
                tls: TlsConfig::default(),
-                azure: None,
-                gcs: None,
            },
            filter: FilterConfig {
                prefix: filter_prefix.map(str::to_string),
@@ -54,8 +54,6 @@ their issue closes.
 | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` |
 | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` |
 | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — |
-| `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` |
-| `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` |
 | `test_build_rustfs_options.sh` | dev-tool | Shell test for rustfs build-option wiring | `make test` (script-tests) |
 | `test_entrypoint_credentials.sh` | dev-tool | Container entrypoint credential-handling test | `make test` (script-tests) |
 | `test_helm_chart_version.sh` | dev-tool | Test for `helm_chart_version.sh` | — |
@@ -12,15 +12,11 @@ import sys
 import tempfile
 import tomllib
 import unittest
-import uuid
-import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
 from unittest import mock
 from pathlib import Path
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError

-from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json
-

 ROOT = Path(__file__).resolve().parents[1]
 SCHEDULED_ALERT_WORKFLOWS = tuple(
@@ -571,7 +567,7 @@ def check_quick_checks(root: Path) -> list[str]:
        errors.append(f"{relative}: missing composite action")
        return errors
    steps = yaml_block(runs, "steps", 2) or []
-    for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"):
+    for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"):
        step = workflow_step_block(steps, command, key="run", indent=4)
        if step is None:
            errors.append(f"{relative}: missing direct execution of {command}")
@@ -877,187 +873,6 @@ def check_core_listing(root: Path, listing: Path) -> list[str]:
        return [f"cannot read core nextest listing: {error}"]


-def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> int:
-    require(type(value) is int and minimum <= value <= maximum, f"invalid integer {name}")
-    return value
-
-
-def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None:
-    """Record an existing build; this command never builds or runs a test."""
-    require(not directory.exists(), "scanner/heal run directory must be new")
-    require(not subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=no"], cwd=root, text=True).strip(),
-            "commit tracked source changes before creating evidence")
-    builds = {}
-    for label, path in (("binary", binary), ("test_binary", test_binary)):
-        path = path.resolve(strict=True)
-        require(path.is_file() and os.access(path, os.X_OK), f"missing executable {label}")
-        builds[label] = {"path": str(path), "sha256": digest(path)}
-    revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip()
-    require(re.fullmatch(r"[0-9a-f]{40}", revision), "invalid source revision")
-    version = subprocess.check_output([builds["binary"]["path"], "--version"], text=True, timeout=30)
-    embedded_revision = re.search(r"^git commit\s*:\s*([0-9a-f]{40})\s*$", version, re.MULTILINE)
-    embedded_status = re.search(r"^git status\s*:\s*(.*)\Z", version, re.MULTILINE | re.DOTALL)
-    require(embedded_revision is not None and embedded_revision[1] == revision, "server binary source revision mismatch")
-    require(embedded_status is not None and not embedded_status[1].strip(), "server binary was built from dirty/unknown source")
-    lock_blob = subprocess.check_output(["git", "hash-object", "Cargo.lock"], cwd=root, text=True).strip()
-    require(re.fullmatch(r"[0-9a-f]{40}", lock_blob), "invalid Cargo.lock identity")
-    features = os.environ.get("RUSTFS_E2E_EXPECTED_FEATURES")
-    require(features is not None, "set RUSTFS_E2E_EXPECTED_FEATURES to the compiled e2e crate feature set")
-    features = ",".join(sorted(set(filter(None, (feature.strip() for feature in features.split(","))))))
-    require(all(re.fullmatch(r"[a-z0-9-]+", feature) for feature in features.split(",") if feature), "invalid expected features")
-    directory.mkdir(parents=True)
-    write_json(directory / "run.json", {"schema": 1, "run_id": uuid.uuid4().hex,
-                                       "source_revision": revision,
-                                       "binary_source_revision": embedded_revision[1],
-                                       "test_build": {"source_revision": revision, "dirty": False,
-                                                      "lock_blob": lock_blob, "features": features},
-                                       "started_at": datetime.now(timezone.utc).timestamp(), **builds})
-
-
-def finish_scanner_heal_receipt(directory: Path, exit_code: int) -> None:
-    require(type(exit_code) is int and 0 <= exit_code <= 255, "invalid test exit code")
-    require(not (directory / "execution.json").exists(), "execution receipt already exists")
-    run = read_json(directory / "run.json")
-    artifacts = {}
-    for name in ("listing.json", "junit.xml", "background-target-restart.json"):
-        path = directory / name
-        if exit_code != 0 and not path.exists():
-            continue
-        require(path.is_file() and 0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}")
-        require(path.stat().st_mtime >= run["started_at"], f"stale {name}")
-        artifacts[name] = digest(path)
-    write_json(directory / "execution.json", {"run_id": run["run_id"], "exit_code": exit_code,
-                                             "finished_at": datetime.now(timezone.utc).timestamp(),
-                                             "artifacts": artifacts})
-
-
-def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> list[str]:
-    """Validate one actual case, or fail the release while required lanes are pending."""
-    try:
-        registry = read_json(root / ".config/scanner-heal-required-tests.json")
-        evidence_integer(registry.get("schema"), "registry schema", 1, 1)
-        require(registry.get("cases"), "invalid scanner/heal registry")
-        selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]}
-        run = read_json(directory / "run.json")
-        execution = read_json(directory / "execution.json")
-        evidence_integer(run.get("schema"), "run schema", 1, 1)
-        require(re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), "invalid run identity")
-        require(re.fullmatch(r"[0-9a-f]{40}", run["source_revision"]), "invalid source revision")
-        require(run.get("binary_source_revision") == run["source_revision"], "server source provenance missing")
-        expected_build = run["test_build"]
-        require(expected_build["source_revision"] == run["source_revision"] and expected_build["dirty"] is False,
-                "test source provenance missing")
-        require(re.fullmatch(r"[0-9a-f]{40}", expected_build["lock_blob"]), "invalid test lockfile identity")
-        require(isinstance(expected_build["features"], str), "missing test features")
-        require(execution.get("run_id") == run["run_id"], "execution belongs to another run")
-        require(type(execution.get("exit_code")) is int and execution["exit_code"] == 0, "test command failed or did not run")
-        number(run["started_at"], "started_at", 1)
-        number(execution["finished_at"], "finished_at", run["started_at"])
-        for label in ("binary", "test_binary"):
-            require(sha(run[label]["sha256"]) and digest(Path(run[label]["path"])) == run[label]["sha256"],
-                    f"{label} changed or missing")
-        for name in ("listing.json", "junit.xml"):
-            path = directory / name
-            require(0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}")
-            require(run["started_at"] <= path.stat().st_mtime <= execution["finished_at"], f"{name} outside run window")
-            require(digest(path) == execution["artifacts"][name], f"{name} hash mismatch")
-        suites = read_json(directory / "listing.json")["rust-suites"]
-        xml = (directory / "junit.xml").read_bytes()
-        require(b"<!DOCTYPE" not in xml and b"<!ENTITY" not in xml, "JUnit entities are forbidden")
-        junit = ET.fromstring(xml)
-        cases = list(junit.iter("testcase"))
-        require(bool(cases), "JUnit has zero testcases")
-        for case in cases:
-            require(not any(child.tag in ("failure", "error", "skipped", "rerunFailure", "rerunError", "flakyFailure", "flakyError")
-                            for child in case), "JUnit contains failed, skipped or retried tests")
-        errors = []
-        for name, requirement in selected.items():
-            suite, test = requirement["suite"], requirement["name"]
-            for key in ("nodes", "drives_per_node"):
-                evidence_integer(requirement["topology"][key], f"required {key}", 1, 16)
-            listing_suite = suites[suite]
-            require(listing_suite["binary-id"] == suite, "nextest suite binary identity mismatch")
-            listed_binary = Path(listing_suite["binary-path"]).resolve(strict=True)
-            require(listed_binary == Path(run["test_binary"]["path"]).resolve(strict=True) and
-                    digest(listed_binary) == run["test_binary"]["sha256"], "nextest selected another test binary")
-            require(listing_suite["package-name"] == "e2e_test" and listing_suite["build-platform"] in ("host", "target"),
-                    "unexpected nextest suite metadata")
-            listed = listing_suite.get("testcases", {}).get(test, {})
-            require(listed.get("ignored") is False and listed.get("filter-match", {}).get("status") == "matches",
-                    f"required test not selected: {suite}::{test}")
-            matches = [case for case in cases if case.get("name") == test and case.get("classname") == suite]
-            require(len(matches) == 1, f"missing/duplicate JUnit case: {suite}::{test}")
-            started = datetime.fromisoformat(matches[0].attrib["timestamp"].replace("Z", "+00:00"))
-            require(started.tzinfo is not None, "JUnit timestamp must include timezone")
-            # quick-junit truncates timestamps to milliseconds.
-            require(run["started_at"] - 0.001 <= started.timestamp() <= execution["finished_at"],
-                    "JUnit testcase executed outside this run")
-            path = directory / requirement["oracle"]
-            require(path.resolve().is_relative_to(directory.resolve()), "oracle path escapes run directory")
-            require(run["started_at"] <= path.stat().st_mtime <= execution["finished_at"], "oracle outside run window")
-            require(digest(path) == execution["artifacts"][requirement["oracle"]], "oracle hash mismatch")
-            oracle = read_json(path)
-            evidence_integer(oracle.get("schema"), "oracle schema", 1, 1)
-            require(oracle.get("evidence") == "process-restart", "not real process-restart evidence")
-            require(oracle.get("case") == name and oracle.get("run_id") == run["run_id"], "oracle belongs to another case/run")
-            require(oracle.get("source_revision") == run["source_revision"], "oracle source mismatch")
-            built = oracle["test_build"]
-            for key in ("source_revision", "dirty", "lock_blob", "features"):
-                require(built[key] == expected_build[key], f"compiled test {key} mismatch")
-            require(built["dirty"] is False, "test binary was compiled from dirty source")
-            require(all(isinstance(built[key], str) and built[key] and built[key] != "unknown" for key in ("target", "profile")),
-                    "missing compiled target/profile")
-            require(isinstance(built["rustflags_hex"], str) and re.fullmatch(r"(?:[0-9a-f]{2})*", built["rustflags_hex"]) is not None,
-                    "invalid compiled rustflags")
-            for label in ("binary", "test_binary"):
-                require(oracle.get(f"{label}_sha256") == run[label]["sha256"], f"oracle {label} mismatch")
-            require(oracle.get("topology") == requirement["topology"], "oracle topology mismatch")
-            for key in ("nodes", "drives_per_node"):
-                evidence_integer(oracle["topology"][key], f"observed {key}", 1, 16)
-            evidence_integer(oracle.get("pid_before"), "pid_before", 1, 2**32 - 1)
-            evidence_integer(oracle.get("pid_after"), "pid_after", 1, 2**32 - 1)
-            require(oracle["pid_before"] != oracle["pid_after"], "no process restart witnessed")
-            objects = oracle["objects"]
-            require(isinstance(objects, list) and requirement["min_objects"] <= len(objects) <= requirement["max_objects"],
-                    "incomplete/oversized object oracle")
-            require(len({obj["key"] for obj in objects}) == len(objects), "duplicate object identity")
-            require(sum(obj["expected_physical"] is None for obj in objects) == 1,
-                    "only the outage object may lack a pre-fault target manifest")
-            for obj in objects:
-                require(isinstance(obj["key"], str) and 0 < len(obj["key"].encode()) <= 1024, "invalid object identity")
-                require(obj["version_id"] is None, "this case only covers unversioned objects")
-                require(type(obj["expected_bytes"]) is int and obj["expected_bytes"] > 0, "missing expected bytes")
-                require(type(obj["actual_bytes"]) is int and obj["actual_bytes"] == obj["expected_bytes"], "S3 body length mismatch")
-                require(sha(obj["expected_sha256"]) and obj["actual_sha256"] == obj["expected_sha256"], "S3 body digest mismatch")
-                physical = obj["physical"]
-                if obj["expected_physical"] is not None:
-                    require(physical == obj["expected_physical"], "target shard differs from pre-fault manifest")
-                for geometry in [physical] + ([obj["expected_physical"]] if obj["expected_physical"] is not None else []):
-                    data = evidence_integer(geometry["data_blocks"], "EC data blocks", 1, 16)
-                    parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16)
-                    require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"],
-                            "EC geometry differs from this case's single set")
-                    evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity)
-                require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata")
-                parts = physical["expected_part_numbers"]
-                require(isinstance(parts, list) and 0 < len(parts) <= 10000, "no physical part coverage")
-                require(all(type(part) is int and part > 0 for part in parts) and len(set(parts)) == len(parts),
-                        "invalid physical part identity")
-                require({str(part) for part in parts} == set(physical["present_part_fingerprints"]), "target shard parts missing")
-                for part in physical["present_part_fingerprints"].values():
-                    require(type(part["size"]) is int and part["size"] > 0 and sha(part["sha256"]), "invalid target shard fingerprint")
-            node_listings = oracle["node_listings"]
-            require(isinstance(node_listings, list) and len(node_listings) == requirement["topology"]["nodes"],
-                    "missing per-node S3 listing")
-            require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings),
-                    "S3 listing differs from object oracle")
-        if case_id == "release":
-            errors.extend(f"pending {gate}: {reason}" for gate, reason in registry["release_pending"].items())
-        return errors
-    except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error:
-        return [f"scanner/heal evidence rejected: {error}"]
-
-
 def validate(root: Path) -> list[str]:
    errors: list[str] = []
    errors.extend(check_core_fixtures(root))
@@ -1091,7 +906,6 @@ class SelfTests(unittest.TestCase):
                "    - name: Lint workflows\n      shell: bash\n      run: shellcheck --version && actionlint\n"
                "    - name: Error format ratchet\n      shell: bash\n"
                "      run: ./scripts/check_error_other_format_ratchet.sh\n"
-                "    - name: Script tests\n      shell: bash\n      run: make script-tests\n"
            )
            sources = {
                ".github/workflows/ci.yml": caller.replace(
@@ -1150,8 +964,6 @@ class SelfTests(unittest.TestCase):
                "only installed actionlint": action.replace("run: shellcheck --version && actionlint", "run: echo actionlint"),
                "missing shellcheck preflight": action.replace("shellcheck --version && ", ""),
                "missing ratchet": action.replace("run: ./scripts/check_error_other_format_ratchet.sh", "run: echo skipped"),
-                "missing script tests": action.replace("run: make script-tests", "run: echo skipped"),
-                "swallowed script failure": action.replace("run: make script-tests", "run: make script-tests || true"),
                "swallowed lint failure": action.replace("&& actionlint", "&& actionlint || true"),
                "swallowed ratchet failure": action.replace("ratchet.sh", "ratchet.sh || true"),
                "conditional lint": action.replace("run: shellcheck", "if: false\n      run: shellcheck"),
@@ -1161,7 +973,7 @@ class SelfTests(unittest.TestCase):
                    "name: Lint workflows", "name: |\n        run: shellcheck --version && actionlint"
                ).replace("\n      run: shellcheck --version && actionlint\n", "\n      run: shellcheck --version && actionlint\n        || true\n"),
            }
-            for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh", "make script-tests"):
+            for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"):
                for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'):
                    mutations[f"quoted {command} {key}"] = action.replace(f"run: {command}", f"{key}\n      run: {command}")
                for separator in ("", "\n", "        # continued command\n"):
@@ -1182,10 +994,9 @@ class SelfTests(unittest.TestCase):
            root = Path(tmp)
            (root / "scripts").mkdir()
            commands = ("shellcheck", "actionlint", "./scripts/check_error_other_format_ratchet.sh")
-            (root / "Makefile").write_text(".PHONY: script-tests\nscript-tests:\n\texit 17\n")
-            for failing in (*commands, "make script-tests"):
+            for failing in commands:
                with self.subTest(command=failing):
-                    run = "shellcheck --version && actionlint" if failing in ("shellcheck", "actionlint") else failing
+                    run = "shellcheck --version && actionlint" if failing != commands[-1] else failing
                    step = workflow_step_block(steps, run, key="run", indent=4)
                    self.assertIsNotNone(step)
                    run_index = next(index for index, line in enumerate(step[1]) if line.startswith("      run:"))
@@ -1200,220 +1011,13 @@ class SelfTests(unittest.TestCase):
                        cwd=root, env=dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}"),
                        capture_output=True, text=True,
                    )
-                    self.assertEqual(result.returncode, 2 if failing == "make script-tests" else 17, result.stderr)
+                    self.assertEqual(result.returncode, 17, result.stderr)

    def test_validate_includes_quick_checks(self) -> None:
        error = "Quick Checks wiring regression"
        with mock.patch(__name__ + ".check_quick_checks", return_value=[error]):
            self.assertIn(error, validate(ROOT))

-    def scanner_heal_fixture(self, directory: Path) -> tuple[Path, Path]:
-        """Parser fixtures only; these files are never runtime evidence."""
-        root, run_dir = directory / "repo", directory / "run"
-        (root / ".config").mkdir(parents=True)
-        run_dir.mkdir()
-        registry = read_json(ROOT / ".config/scanner-heal-required-tests.json")
-        write_json(root / ".config/scanner-heal-required-tests.json", registry)
-        requirement = registry["cases"]["background-target-restart"]
-        binary = directory / "fake-binary"
-        binary.write_bytes(b"parser fixture, not a real build")
-        binary.chmod(0o700)
-        build = {"path": str(binary), "sha256": digest(binary)}
-        write_json(run_dir / "run.json", {"schema": 1, "run_id": "a" * 32, "source_revision": "b" * 40,
-                                         "binary_source_revision": "b" * 40,
-                                         "test_build": {"source_revision": "b" * 40, "dirty": False,
-                                                        "lock_blob": "c" * 40, "features": "default"},
-                                         "started_at": datetime.now(timezone.utc).timestamp() - 1,
-                                         "binary": build, "test_binary": build})
-        write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: {
-            "binary-id": requirement["suite"], "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target",
-            "testcases": {
-            requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}}
-        }}}})
-        (run_dir / "junit.xml").write_text(
-            f'<testsuites><testsuite><testcase name="{requirement["name"]}" classname="{requirement["suite"]}" '
-            f'timestamp="{datetime.now(timezone.utc).isoformat(timespec="milliseconds")}"/></testsuite></testsuites>')
-        physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
-                    "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
-                    "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
-                    "inline_data_fingerprint": None}
-        obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16,
-               "expected_sha256": "d" * 64, "actual_sha256": "d" * 64,
-               "expected_physical": physical, "physical": physical}
-        objects = [dict(obj, key=f"object-{index}") for index in range(9)]
-        objects[-1] = dict(objects[-1], expected_physical=None)
-        write_json(run_dir / "background-target-restart.json", {
-            "schema": 1, "evidence": "process-restart", "case": "background-target-restart",
-            "run_id": "a" * 32, "source_revision": "b" * 40,
-            "test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40,
-                           "features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""},
-            "binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
-            "topology": {"nodes": 4, "drives_per_node": 1}, "pid_before": 10, "pid_after": 11,
-            "objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
-        })
-        finish_scanner_heal_receipt(run_dir, 0)
-        return root, run_dir
-
-    def test_scanner_heal_case_does_not_approve_pending_release(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            root, run_dir = self.scanner_heal_fixture(Path(tmp))
-            self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), [])
-            errors = check_scanner_heal_evidence(root, run_dir, "release")
-            self.assertEqual(len(errors), 21)
-            self.assertTrue(any(error.startswith("pending R-E:") for error in errors))
-            self.assertTrue(any(error.startswith("pending R-D:") for error in errors))
-            self.assertTrue(any(error.startswith("pending R-L:") for error in errors))
-
-    def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None:
-        for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale",
-                      "hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"):
-            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
-                root, run_dir = self.scanner_heal_fixture(Path(tmp))
-                path = run_dir / "background-target-restart.json"
-                oracle = read_json(path)
-                if fault == "exit":
-                    receipt = read_json(run_dir / "execution.json")
-                    receipt["exit_code"] = 42
-                    write_json(run_dir / "execution.json", receipt)
-                elif fault == "missing":
-                    path.unlink()
-                elif fault == "zero":
-                    (run_dir / "junit.xml").write_text("<testsuites/>")
-                elif fault in ("skipped", "failed", "retry"):
-                    junit = run_dir / "junit.xml"
-                    tag = {"skipped": "skipped", "failed": "failure", "retry": "rerunFailure"}[fault]
-                    junit.write_text(junit.read_text().replace("/></testsuite>", f"><{tag}/></testcase></testsuite>"))
-                elif fault in ("filtered", "ignored"):
-                    listing = read_json(run_dir / "listing.json")
-                    case = next(iter(listing["rust-suites"]["e2e_test"]["testcases"].values()))
-                    case["ignored"] = fault == "ignored"
-                    case["filter-match"]["status"] = "mismatch" if fault == "filtered" else "matches"
-                    write_json(run_dir / "listing.json", listing)
-                elif fault == "stale":
-                    os.utime(path, (1, 1))
-                elif fault == "hash":
-                    path.write_text(path.read_text() + " ")
-                elif fault == "binary":
-                    Path(read_json(run_dir / "run.json")["binary"]["path"]).write_bytes(b"another build")
-                else:
-                    if fault == "synthetic":
-                        oracle["evidence"] = "synthetic"
-                    elif fault == "wrong-run":
-                        oracle["run_id"] = "f" * 32
-                    elif fault == "same-pid":
-                        oracle["pid_after"] = oracle["pid_before"]
-                    elif fault == "body":
-                        oracle["objects"][0]["actual_sha256"] = "e" * 64
-                    elif fault == "parts":
-                        oracle["objects"][0]["physical"]["present_part_fingerprints"] = {}
-                    elif fault == "listing":
-                        oracle["node_listings"][0] = []
-                    elif fault == "topology":
-                        oracle["topology"] = {"nodes": 3, "drives_per_node": 4}
-                    write_json(path, oracle)
-                if fault not in ("exit", "missing", "stale", "hash", "binary"):
-                    (run_dir / "execution.json").unlink()
-                    finish_scanner_heal_receipt(run_dir, 0)
-                self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault)
-
-    def test_scanner_heal_receipts_reject_reuse_and_missing_builds(self) -> None:
-        with tempfile.TemporaryDirectory() as tmp:
-            root, run_dir = self.scanner_heal_fixture(Path(tmp))
-            with self.assertRaisesRegex(ValueError, "already exists"):
-                finish_scanner_heal_receipt(run_dir, 0)
-            with self.assertRaisesRegex(ValueError, "must be new"):
-                begin_scanner_heal_receipt(root, run_dir, Path("missing"), Path("missing"))
-            with mock.patch("subprocess.check_output", side_effect=["", "b" * 40]):
-                with self.assertRaises(FileNotFoundError):
-                    begin_scanner_heal_receipt(root, Path(tmp) / "new-run", Path(tmp) / "missing", Path(tmp) / "missing")
-
-    def test_scanner_heal_begin_requires_embedded_source_provenance(self) -> None:
-        for kind in ("current", "stale", "dirty", "unknown"):
-            with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
-                root, _ = self.scanner_heal_fixture(Path(tmp))
-                sources = root / "crates/e2e_test/src"
-                sources.mkdir(parents=True)
-                (sources / "heal_erasure_disk_rebuild_test.rs").write_bytes(b"oracle source")
-                (sources / "chaos.rs").write_bytes(b"census source")
-                revision = "c" * 40 if kind == "stale" else "b" * 40
-                version = f"rustfs\ngit commit   : {revision}\ngit status   :\n"
-                if kind == "dirty":
-                    version += "modified source\n"
-                if kind == "unknown":
-                    version = "rustfs without build provenance"
-                with mock.patch("subprocess.check_output", side_effect=["", "b" * 40, version, "c" * 40]), \
-                     mock.patch.dict(os.environ, {"RUSTFS_E2E_EXPECTED_FEATURES": "default"}):
-                    directory = Path(tmp) / "fresh"
-                    binary = Path(tmp) / "fake-binary"
-                    if kind == "current":
-                        begin_scanner_heal_receipt(root, directory, binary, binary)
-                        self.assertEqual(read_json(directory / "run.json")["binary_source_revision"], "b" * 40)
-                    else:
-                        with self.assertRaisesRegex(ValueError, "server binary"):
-                            begin_scanner_heal_receipt(root, directory, binary, binary)
-                        self.assertFalse(directory.exists())
-
-    def test_scanner_heal_rejects_copied_junit_and_wrong_suite_build(self) -> None:
-        for fault in ("old-junit", "missing-time", "wrong-binary", "common-source", "lockfile", "features", "dirty-build"):
-            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp:
-                root, run_dir = self.scanner_heal_fixture(Path(tmp))
-                if fault in ("old-junit", "missing-time"):
-                    path = run_dir / "junit.xml"
-                    xml = ET.fromstring(path.read_bytes())
-                    testcase = next(xml.iter("testcase"))
-                    if fault == "old-junit":
-                        testcase.set("timestamp", "2000-01-01T00:00:00.000Z")
-                    else:
-                        del testcase.attrib["timestamp"]
-                    # Rewriting/copying gives an old execution a fresh mtime.
-                    path.write_bytes(ET.tostring(xml))
-                elif fault == "wrong-binary":
-                    path = run_dir / "listing.json"
-                    listing = read_json(path)
-                    another = Path(tmp) / "another-binary"
-                    another.write_bytes(Path(tmp, "fake-binary").read_bytes())
-                    listing["rust-suites"]["e2e_test"]["binary-path"] = str(another)
-                    write_json(path, listing)
-                else:
-                    path = run_dir / "background-target-restart.json"
-                    oracle = read_json(path)
-                    key, value = {"common-source": ("source_revision", "f" * 40), "lockfile": ("lock_blob", "f" * 40),
-                                  "features": ("features", "default,sftp"), "dirty-build": ("dirty", True)}[fault]
-                    oracle["test_build"][key] = value
-                    write_json(path, oracle)
-                (run_dir / "execution.json").unlink()
-                finish_scanner_heal_receipt(run_dir, 0)
-                self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault)
-
-    def test_scanner_heal_rejects_boolean_fractional_and_out_of_geometry_integers(self) -> None:
-        valid = {"schema": 1, "nodes": 4, "drives_per_node": 1, "pid_before": 10, "pid_after": 11,
-                 "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2}
-        cases = [(field, value) for field, correct in valid.items() for value in (True, float(correct))]
-        cases += [("erasure_index", 5), ("erasure_index", 0), ("pid_after", -1)]
-        for field, value in cases:
-            with self.subTest(field=field, value=value), tempfile.TemporaryDirectory() as tmp:
-                root, run_dir = self.scanner_heal_fixture(Path(tmp))
-                path = run_dir / "background-target-restart.json"
-                oracle = read_json(path)
-                if field in ("nodes", "drives_per_node"):
-                    oracle["topology"][field] = value
-                elif field in ("erasure_index", "data_blocks", "parity_blocks"):
-                    oracle["objects"][-1]["physical"][field] = value
-                else:
-                    oracle[field] = value
-                write_json(path, oracle)
-                (run_dir / "execution.json").unlink()
-                finish_scanner_heal_receipt(run_dir, 0)
-                self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"))
-        for filename in ("run.json", ".config/scanner-heal-required-tests.json"):
-            with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp:
-                root, run_dir = self.scanner_heal_fixture(Path(tmp))
-                path = (root if filename.startswith(".config") else run_dir) / filename
-                content = read_json(path)
-                content["schema"] = True
-                write_json(path, content)
-                self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"))
-
    def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
@@ -2056,25 +1660,6 @@ def main() -> int:
    if sys.argv[1:] == ["--self-test"]:
        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
        return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
-    if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"]):
-        try:
-            if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal":
-                begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4]))
-                return 0
-            if len(sys.argv) == 4 and sys.argv[1] == "--finish-scanner-heal":
-                finish_scanner_heal_receipt(Path(sys.argv[2]), int(sys.argv[3]))
-                return 0
-            if len(sys.argv) == 4 and sys.argv[1] == "--check-scanner-heal":
-                errors = check_scanner_heal_evidence(ROOT, Path(sys.argv[2]), sys.argv[3])
-                for error in errors:
-                    print(f"ERROR: {error}", file=sys.stderr)
-                if not errors:
-                    print(f"Case evidence verified: {sys.argv[3]}; this does not approve release")
-                return 1 if errors else 0
-            raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, or --check-scanner-heal DIR CASE|release")
-        except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error:
-            print(f"ERROR: {error}", file=sys.stderr)
-            return 1
    if len(sys.argv) == 3 and sys.argv[1] == "--check-core":
        errors = check_core_listing(ROOT, Path(sys.argv[2]))
        for error in errors:
@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-"""Preserve every functional case execution and its suite context in reports."""
-
-from __future__ import annotations
-
-import argparse
-from pathlib import Path
-import re
-
-
-def generate_report(log_file: Path, case_file: Path, matrix_file: Path | None = None) -> bool:
-    ansi = re.compile(r"\x1b\[[0-9;]*m")
-    start_re = re.compile(r"^---\s+([A-Z][A-Z0-9]*-[0-9]+)\s+(.+?)\s+---$")
-    done_re = re.compile(r"^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z][A-Z0-9]*-[0-9]+)\b")
-    context_re = re.compile(r"^(?:\[INFO\]\s+)?==\s+((?:topology|suite):.+?)\s+==$")
-    topo_re = re.compile(r"^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$")
-    rows = []
-    pending = {}
-    topo_rows = []
-    context = "context not recorded"
-    complete = True
-    try:
-        with log_file.open(encoding="utf-8", errors="replace") as log:
-            for raw in log:
-                line = ansi.sub("", raw).strip()
-                if match := context_re.match(line):
-                    context = match[1]
-                    pending.clear()
-                elif match := topo_re.match(line):
-                    topo_rows.append(match.groups())
-                elif match := start_re.match(line):
-                    case_id, name = match.groups()
-                    pending[case_id] = len(rows)
-                    rows.append([case_id, f"{name} ({context})", "RUNNING"])
-                elif match := done_re.match(line):
-                    status, case_id = match.groups()
-                    index = pending.pop(case_id, None)
-                    if index is None:
-                        complete = False
-                        rows.append([case_id, f"{case_id} ({context}; start not recorded)", status])
-                    else:
-                        rows[index][2] = status
-    except FileNotFoundError:
-        pass
-
-    counts = {status: sum(row[2] == status for row in rows) for status in ("PASS", "FAIL", "UNSUPPORTED", "RUNNING")}
-    with case_file.open("w", encoding="utf-8") as out:
-        out.write(f"## Case Summary\n\n- Total: {len(rows)}\n")
-        for status, count in counts.items():
-            out.write(f"- {status}: {count}\n")
-        out.write("\n| Case | Name | Status |\n| --- | --- | --- |\n")
-        for row in rows:
-            out.write("| " + " | ".join(value.replace("|", "&#124;") for value in row) + " |\n")
-        if not rows:
-            out.write("\nNo case execution was recorded; the log is missing, empty, or stopped before the cases.\n")
-
-    valid = complete and bool(rows) and not counts["FAIL"] and not counts["RUNNING"]
-    if matrix_file is not None:
-        with matrix_file.open("w", encoding="utf-8") as out:
-            out.write("## Upgrade Matrix\n\n| Topology | KMS Backend | From Version | To Version | Result |\n")
-            out.write("| --- | --- | --- | --- | --- |\n")
-            for topo, backend, old_v, new_v, npass, nfail in topo_rows:
-                result = "PASS" if nfail == "0" else "FAIL"
-                out.write(f"| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n")
-            if not topo_rows:
-                out.write("| - | - | - | - | NOT RUN (suite failed before upgrade) |\n")
-        valid = valid and bool(topo_rows) and all(row[-1] == "0" for row in topo_rows)
-    return valid
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("log_file", type=Path)
-    parser.add_argument("case_file", type=Path)
-    parser.add_argument("matrix_file", type=Path, nargs="?")
-    args = parser.parse_args()
-    raise SystemExit(0 if generate_report(args.log_file, args.case_file, args.matrix_file) else 1)
@@ -1,12 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail

-if [[ "${1:-}" == "--abba" ]]; then
-  shift
-  SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-  exec "$SCRIPT_DIR/python_bin.sh" "$SCRIPT_DIR/scanner_abba.py" "$@"
-fi
-
 ALIAS=""
 ENDPOINT=""
 ACCESS_KEY="${RUSTFS_ACCESS_KEY:-}"
@@ -29,7 +23,6 @@ TELEMETRY_PIDS=()
 usage() {
  cat <<'USAGE'
 Usage:
-  scripts/run_scanner_validation_harness.sh --abba --help
  scripts/run_scanner_validation_harness.sh --alias <admin-alias> \
    --endpoint <url> [options]

@@ -1,446 +0,0 @@
-#!/usr/bin/env python3
-"""Run isolated scanner/heal ABBA cells through a deployment-specific adapter."""
-
-import argparse
-from decimal import Decimal
-import hashlib
-import json
-import math
-import os
-from pathlib import Path
-import select
-import shutil
-import signal
-import subprocess
-import sys
-import time
-
-SCENARIOS = ("cold-hot", "fresh-hot", "multi-hot-new", "running-heal", "mrf-replay")
-LEGS = ("A1", "B1", "B2", "A2")
-MAX_JSON_BYTES = 1024 * 1024
-METRICS = (
-    "p99_ms", "throughput_ops", "rss_bytes", "cpu_seconds", "iops", "rpc_count",
-    "cache_clone_bytes", "encode_bytes", "save_bytes", "oldest_age_seconds",
-    "walk_objects", "cold_walk_objects", "healed_objects", "errors", "requests",
-)
-REPEATABILITY_LIMIT = Decimal("0.05")
-P2_WORK_MULTIPLE_LIMIT = Decimal("1.2")
-
-
-def require(condition, message):
-    if not condition:
-        raise ValueError(message)
-
-
-def number(value, name, minimum=0):
-    require(type(value) in (float, int) and math.isfinite(value) and value >= minimum,
-            f"invalid {name}")
-    return value
-
-
-def decimal_number(value, name, minimum=0):
-    if isinstance(value, Decimal):
-        require(value.is_finite() and value >= Decimal(str(minimum)), f"invalid {name}")
-        return value
-    number(value, name, minimum)
-    return Decimal(str(value))
-
-
-def ratio(numerator, denominator, name):
-    denominator = decimal_number(denominator, f"{name} denominator")
-    require(denominator > 0, f"invalid {name} denominator")
-    return decimal_number(numerator, name) / denominator
-
-
-def relative_change(current, baseline, name):
-    return ratio(current, baseline, name) - Decimal("1")
-
-
-def repeatability_change(first, second, name):
-    first = decimal_number(first, name)
-    second = decimal_number(second, name)
-    if first == 0 and second == 0:
-        return Decimal("0")
-    if first == 0 or second == 0:
-        return Decimal("Infinity")
-    return abs(second / first - Decimal("1"))
-
-
-def report_number(value):
-    return None if value.is_infinite() else float(value)
-
-
-def digest(path):
-    with Path(path).open("rb") as stream:
-        return hashlib.file_digest(stream, "sha256").hexdigest()
-
-
-def read_json(path):
-    require(path.stat().st_size <= MAX_JSON_BYTES, f"oversized JSON: {path.name}")
-    with path.open() as stream:
-        value = json.load(stream)
-    require(isinstance(value, dict), f"expected JSON object: {path.name}")
-    return value
-
-
-def write_json(path, value):
-    data = json.dumps(value, indent=2, allow_nan=False) + "\n"
-    require(len(data.encode()) <= MAX_JSON_BYTES, "oversized result")
-    path.write_text(data)
-
-
-def sha(value):
-    return isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
-
-
-def validate_manifest(manifest):
-    require(manifest.get("schema") == 1, "unsupported manifest schema")
-    require(manifest.get("evidence") in ("synthetic", "measured"), "missing evidence type")
-    fixed = manifest["fixed"]
-    for key in ("config_sha256", "dataset_sha256"):
-        require(sha(fixed.get(key)), f"invalid fixed.{key}")
-    for key in ("release_flags", "durability", "disk_type", "cache_state", "load_command", "resource_isolation"):
-        require(isinstance(fixed.get(key), str) and fixed[key].strip(), f"missing fixed.{key}")
-    require(fixed.get("topology") == "EC8+4", "formal matrix requires EC8+4")
-    number(fixed.get("offered_load_ops"), "offered load", 1)
-    require(type(manifest.get("rounds")) is int and 3 <= manifest["rounds"] <= 10,
-            "rounds must be 3..10")
-    minimum = 900 if manifest["evidence"] == "measured" else 1
-    require(type(manifest.get("duration_seconds")) is int and
-            minimum <= manifest["duration_seconds"] <= 86400, "invalid duration_seconds")
-    number(manifest.get("min_free_bytes"), "min_free_bytes", 1)
-    for phase in ("baseline", "candidate"):
-        build = manifest[phase]
-        path = Path(build["binary"]).resolve(strict=True)
-        require(path.is_file() and os.access(path, os.X_OK), f"missing executable {phase} build")
-        require(sha(build.get("sha256")) and digest(path) == build["sha256"], f"{phase} binary hash mismatch")
-        require(isinstance(build.get("revision"), str) and len(build["revision"]) == 40 and
-                all(c in "0123456789abcdef" for c in build["revision"]), f"invalid {phase} revision")
-        build["binary"] = str(path)
-    for scenario in SCENARIOS:
-        expected = manifest["oracles"][scenario]
-        for key in ("objects", "versions", "bytes"):
-            require(type(expected.get(key)) is int and expected[key] > 0, f"missing {scenario} oracle {key}")
-        require(sha(expected.get("sha256")), f"missing {scenario} content/version digest")
-        number(manifest["expected_healed_objects"].get(scenario), f"{scenario} expected repairs")
-        if scenario in ("running-heal", "mrf-replay"):
-            require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs")
-
-
-class OwnedCommand:
-    """Keep the session leader unreaped until its group's last signal is sent."""
-
-    def __init__(self, args, log):
-        require(sys.platform == "darwin" or hasattr(os, "waitid"), "non-reaping child observation is unavailable")
-        self.args, self.status = args, None
-        self.queue = select.kqueue() if sys.platform == "darwin" else None
-        self.process = None
-        read_gate, write_gate = os.pipe()
-        try:
-            # The shell has already exec'd when Popen returns. Gate the target
-            # until kqueue is registered; preexec_fn would deadlock Popen here.
-            gate = f'read -r _scanner_gate <&{read_gate} || exit 125; exec {read_gate}<&-; exec "$@"'
-            self.process = subprocess.Popen(["bash", "-c", gate, "scanner-abba", *args], pass_fds=(read_gate,),
-                                            stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
-            if self.queue is not None:
-                # Darwin NOTE_EXITSTATUS is not exposed by Python's select constants.
-                event = select.kevent(self.process.pid, filter=select.KQ_FILTER_PROC,
-                                      flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT,
-                                      fflags=select.KQ_NOTE_EXIT | 0x04000000)
-                self.queue.control([event], 0, 0)
-            os.write(write_gate, b"\n")
-        except BaseException:
-            try:
-                if self.process is not None:
-                    try:
-                        self._signal_group(signal.SIGKILL)
-                    finally:
-                        self.process.wait(timeout=10)
-            finally:
-                if self.queue is not None:
-                    self.queue.close()
-            raise
-        finally:
-            os.close(read_gate)
-            os.close(write_gate)
-
-    def wait(self, timeout):
-        if self.status is not None:
-            return self.status
-        deadline = time.monotonic() + timeout
-        while True:
-            remaining = deadline - time.monotonic()
-            if remaining <= 0:
-                raise subprocess.TimeoutExpired(self.args, timeout)
-            if self.queue is not None:
-                events = self.queue.control(None, 1, remaining)
-                if events:
-                    self.status = os.waitstatus_to_exitcode(events[0].data)
-                    return self.status
-            else:
-                result = os.waitid(os.P_PID, self.process.pid, os.WEXITED | os.WNOWAIT | os.WNOHANG)
-                if result is not None:
-                    self.status = result.si_status if result.si_code == os.CLD_EXITED else -result.si_status
-                    return self.status
-                time.sleep(min(0.05, remaining))
-
-    def _signal_group(self, sig):
-        try:
-            os.killpg(self.process.pid, sig)
-            return True
-        except ProcessLookupError:
-            return False
-
-    def finish(self, terminate=False):
-        if self.process.returncode is not None:
-            return self.process.returncode
-        try:
-            if terminate:
-                try:
-                    self._signal_group(signal.SIGTERM)
-                    deadline = time.monotonic() + 10
-                    while time.monotonic() < deadline and self._signal_group(0):
-                        time.sleep(0.05)
-                finally:
-                    # Keep the PID reserved through the last group signal, even
-                    # when the cleanup grace period itself is interrupted.
-                    self._signal_group(signal.SIGKILL)
-        finally:
-            try:
-                returncode = self.process.wait(timeout=10)
-            finally:
-                if self.queue is not None:
-                    self.queue.close()
-        return returncode
-
-
-def invoke(adapter, action, request, timeout):
-    """The adapter writes bounded JSON separately; stderr/stdout remain raw evidence."""
-    output = request.parent / f"{action}.json"
-    with (request.parent / f"{action}.log").open("wb") as log:
-        process = OwnedCommand([str(adapter), action, str(request), str(output)], log)
-        try:
-            returncode = process.wait(timeout)
-            if returncode:
-                raise subprocess.CalledProcessError(returncode, [str(adapter), action])
-            result = read_json(output)
-        except BaseException:
-            process.finish(terminate=True)
-            raise
-        else:
-            # Successful prepare may intentionally leave adapter-owned services.
-            process.finish()
-            return result
-
-
-def validate_result(result, request, expected):
-    require(result.get("evidence") == request["evidence"], "adapter evidence type mismatch")
-    require(result.get("fixed") == request["fixed"], "offered load/config/cache/durability drift")
-    require(result.get("build") == request["build"], "deployed build provenance mismatch")
-    require(result.get("data_dir") == request["data_dir"], "adapter data isolation mismatch")
-    require(result.get("background") == request["background"], "background mode mismatch")
-    require(type(result.get("sample_count")) is int and 1 <= result["sample_count"] <= 3600,
-            "sample_count must be 1..3600")
-    number(result.get("elapsed_seconds"), "elapsed_seconds", request["duration_seconds"])
-    metrics = result["metrics"]
-    for key in METRICS:
-        number(metrics.get(key), key)
-    for key in ("requests", "p99_ms", "throughput_ops"):
-        require(metrics[key] > 0, f"zero {key}")
-    require(metrics["errors"] == 0, "workload request errors")
-    require(metrics["cold_walk_objects"] <= metrics["walk_objects"], "cold walk exceeds total walk")
-    require(result.get("oracle") == expected, "object/version/byte oracle mismatch")
-    if request["background"] == "on":
-        require(metrics["walk_objects"] > 0, "zero background walk")
-        require(metrics["healed_objects"] == request["expected_healed_objects"], "incomplete repair oracle")
-        if request["scenario"] in ("running-heal", "mrf-replay"):
-            require(metrics["healed_objects"] > 0, "zero completed repairs")
-    return result
-
-
-def convergence(result):
-    window = result.get("convergence")
-    if not window or window.get("writes_stopped") is not True or window.get("last_mutation_observed") is not True or window.get("first_complete_publication") is not True:
-        return None
-    for key in ("last_mutation_time", "last_mutation_observed_time", "writes_stopped_time", "window_start", "window_end", "walk_objects", "full_walk_objects", "budget_available_seconds"):
-        number(window.get(key), f"convergence.{key}")
-    require(window["last_mutation_time"] <= window["writes_stopped_time"] <= window["window_start"] < window["window_end"],
-            "invalid post-mutation convergence window")
-    require(window["last_mutation_time"] <= window["last_mutation_observed_time"] <= window["window_start"],
-            "convergence started before last mutation was observed")
-    require(window["full_walk_objects"] > 0, "zero full walk reference")
-    require(0 < window["budget_available_seconds"] <= window["window_end"] - window["window_start"],
-            "invalid convergence budget window")
-    return ratio(window["walk_objects"], window["full_walk_objects"], "convergence work")
-
-
-def evaluate(cells):
-    comparisons = []
-    inconclusive = False
-    failed = False
-    for offset in range(0, len(cells), 4):
-        group = cells[offset:offset + 4]
-        require([cell["leg"] for cell in group] == list(LEGS), "incomplete ABBA group")
-        a1, b1, b2, a2 = (cell["result"]["metrics"] for cell in group)
-        control = group[0]["comparison"] == "background"
-        drift = max(abs(relative_change(a2[k], a1[k], k)) for k in ("p99_ms", "throughput_ops"))
-        repeat_drift = max(abs(relative_change(b2[k], b1[k], k)) for k in ("p99_ms", "throughput_ops"))
-        noise = max(drift, repeat_drift) > REPEATABILITY_LIMIT
-        a = {key: (decimal_number(a1[key], key) + decimal_number(a2[key], key)) / Decimal("2") for key in METRICS}
-        b = {key: (decimal_number(b1[key], key) + decimal_number(b2[key], key)) / Decimal("2") for key in METRICS}
-        p99 = relative_change(b["p99_ms"], a["p99_ms"], "p99_ms")
-        throughput = relative_change(b["throughput_ops"], a["throughput_ops"], "throughput_ops")
-        thresholds = {"p99_regression": Decimal("0.10") if control else Decimal("0.05"),
-                      "throughput_loss": Decimal("0.05") if control else Decimal("0.03")}
-        passed = p99 <= thresholds["p99_regression"] and throughput >= -thresholds["throughput_loss"]
-        p1 = None
-        work_drift = None
-        if not control:
-            if group[0]["scenario"] == "cold-hot":
-                require(a["cold_walk_objects"] > 0, "cold-hot baseline has no cold walk samples")
-            work_drift = max(repeatability_change(a1[key], a2[key], key) for key in ("walk_objects", "cold_walk_objects"))
-            work_drift = max(work_drift, *(repeatability_change(b1[key], b2[key], key) for key in ("walk_objects", "cold_walk_objects")))
-            noise |= work_drift > REPEATABILITY_LIMIT
-            required = ratio(a["cold_walk_objects"], a["walk_objects"], "cold walk baseline") * Decimal("0.80")
-            reduction = Decimal("1") - ratio(b["walk_objects"], a["walk_objects"], "walk reduction")
-            p1 = {"required_reduction": float(required), "observed_reduction": float(reduction),
-                  "repeatability_drift": report_number(work_drift)}
-            if group[0]["scenario"] == "cold-hot":
-                # Compare counts before division can round repeating decimal ratios.
-                passed &= a["walk_objects"] - b["walk_objects"] >= a["cold_walk_objects"] * Decimal("0.80")
-        p2 = [convergence(cell["result"]) if cell["background"] == "on" else None for cell in group]
-        candidate_p2 = [value for cell, value in zip(group, p2) if cell["leg"].startswith("B")]
-        p2_pending = any(value is None for value in candidate_p2)
-        passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None)
-        p2_report = [None if value is None else float(value) for value in p2]
-        inconclusive |= noise or p2_pending
-        if not noise and not passed:
-            failed = True
-        comparisons.append({"scenario": group[0]["scenario"], "comparison": group[0]["comparison"],
-                            "round": group[0]["round"], "status": "inconclusive" if noise else ("fail" if not passed else "inconclusive" if p2_pending else "pass"),
-                            "a2_a1_drift": report_number(drift), "b2_b1_drift": report_number(repeat_drift),
-                            "p99_regression": float(p99), "throughput_change": float(throughput),
-                            "thresholds": {key: float(value) for key, value in thresholds.items()},
-                            "p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT),
-                            "p2_post_stop_work_multiples": p2_report})
-    return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons
-
-
-def collect_live(prepared, request, request_path, adapter):
-    collector = Path(__file__).with_name("run_scanner_validation_harness.sh")
-    # Only allow connection fields here; the runner owns cadence and output paths.
-    connection = prepared["collector"]
-    require(set(connection) == {"alias", "endpoint", "metrics_endpoints"}, "invalid collector connection")
-    require(all(isinstance(value, str) and value for value in connection.values()), "missing collector endpoint")
-    output = request_path.parent / "telemetry"
-    args = ["bash", str(collector), "--alias", connection["alias"], "--endpoint", connection["endpoint"],
-            "--metrics-endpoints", connection["metrics_endpoints"], "--deployment", "distributed",
-            "--samples", str(request["duration_seconds"] // 60 + 1), "--interval-secs", "60",
-            "--out-dir", str(output)]
-    with (request_path.parent / "collector.log").open("wb") as log:
-        process = OwnedCommand(args, log)
-        try:
-            started = time.monotonic()
-            result = invoke(adapter, "measure", request_path, request["duration_seconds"] + 300)
-            require(time.monotonic() - started >= request["duration_seconds"], "measurement ended before required window")
-            require(process.wait(120) == 0, "scanner collector failed")
-            require(output.joinpath("scanner-summary.csv").stat().st_size > 0, "missing collector samples")
-            samples = list((output / "status").glob("scanner-status.*.json"))
-            require(len(samples) == request["duration_seconds"] // 60 + 1, "missing scanner samples")
-            for sample in samples:
-                status = read_json(sample)
-                require(isinstance(status.get("metrics"), dict) and status["metrics"], "invalid scanner status response")
-            heals = list((output / "heal").glob("background-heal-status.*.json"))
-            require(bool(heals), "missing heal samples")
-            for sample in heals:
-                status = read_json(sample)
-                require(isinstance(status.get("healOperations"), dict) and status["healOperations"], "invalid heal status response")
-            return result
-        finally:
-            process.finish(terminate=True)
-
-
-def run(manifest, adapter, output, data_root):
-    validate_manifest(manifest)
-    require(adapter.is_file() and os.access(adapter, os.X_OK), "missing executable adapter")
-    require(not output.exists() and not data_root.exists(), "output/data root must be new; existing data is preserved")
-    require(output != data_root and output not in data_root.parents and data_root not in output.parents,
-            "output and data roots must not overlap")
-    output.mkdir(parents=True)
-    data_root.mkdir(parents=True)
-    require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space")
-    manifest["adapter_sha256"] = digest(adapter)
-    manifest["collector_sha256"] = digest(Path(__file__).with_name("run_scanner_validation_harness.sh"))
-    write_json(output / "manifest.json", manifest)
-    cells = []
-    write_json(output / "report.json", {"status": "incomplete", "performance": "pending"})
-    try:
-        for scenario in SCENARIOS:
-            for comparison in ("build", "background"):
-                for round_id in range(1, manifest["rounds"] + 1):
-                    for leg in LEGS:
-                        phase = "baseline" if comparison == "build" and leg.startswith("A") else "candidate"
-                        background = "off" if comparison == "background" and leg.startswith("A") else "on"
-                        name = f"{scenario}-{comparison}-{round_id}-{leg}"
-                        cell_dir = output / name
-                        cell_dir.mkdir()
-                        data_dir = data_root / name
-                        data_dir.mkdir()
-                        request = {"schema": 1, "scenario": scenario, "comparison": comparison, "round": round_id,
-                                   "leg": leg, "background": background, "build": manifest[phase],
-                                   "evidence": manifest["evidence"], "fixed": manifest["fixed"],
-                                   "duration_seconds": manifest["duration_seconds"], "data_dir": str(data_dir),
-                                   "expected_healed_objects": manifest["expected_healed_objects"][scenario],
-                                   "expected_oracle": manifest["oracles"][scenario]}
-                        require(digest(Path(request["build"]["binary"])) == request["build"]["sha256"], "binary changed during run")
-                        require(digest(adapter) == manifest["adapter_sha256"], "adapter changed during run")
-                        require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space")
-                        request_path = cell_dir / "request.json"
-                        write_json(request_path, request)
-                        print(name, flush=True)
-                        try:
-                            prepared = invoke(adapter, "prepare", request_path, 300)
-                            require(prepared.get("ready") is True, "deployment not ready")
-                            if manifest["evidence"] == "measured":
-                                result = collect_live(prepared, request, request_path, adapter)
-                            else:
-                                result = invoke(adapter, "measure", request_path, 300)
-                            # An independent operation must enumerate all object versions and bytes.
-                            oracle = invoke(adapter, "oracle", request_path, 300)
-                            require(oracle.get("complete") is True and oracle.get("errors") == 0, "correctness oracle failed")
-                            require(type(oracle.get("errors")) is int, "invalid oracle error count")
-                            result["oracle"] = oracle["actual"]
-                            validate_result(result, request, request["expected_oracle"])
-                            cells.append({**request, "result": result})
-                        finally:
-                            stopped = invoke(adapter, "stop", request_path, 300)
-                            require(stopped.get("stopped") is True, "adapter failed to stop deployment")
-        status, comparisons = evaluate(cells)
-        synthetic = manifest["evidence"] == "synthetic"
-        report = {"status": "synthetic_validated" if synthetic and status == "pass" else status,
-                  "evidence": manifest["evidence"], "performance": "pending" if synthetic else status,
-                  "cells": len(cells), "comparisons": comparisons}
-        write_json(output / "report.json", report)
-        return 0 if status == "pass" else 3 if status == "inconclusive" else 1
-    except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error:
-        write_json(output / "report.json", {"status": "failed", "performance": "pending",
-                                            "completed_cells": len(cells), "error": str(error)})
-        raise
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--manifest", type=Path, required=True)
-    parser.add_argument("--adapter", type=Path, required=True)
-    parser.add_argument("--out-dir", type=Path, required=True)
-    parser.add_argument("--data-root", type=Path, required=True)
-    args = parser.parse_args()
-    try:
-        return run(read_json(args.manifest), args.adapter.resolve(), args.out_dir.resolve(), args.data_root.resolve())
-    except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error:
-        print(f"ERROR: {error}", file=sys.stderr)
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
@@ -187,7 +187,7 @@ values = {}
 for element in root.iter():
    values[element.tag.rsplit("}", 1)[-1]] = element.text or ""
 for field in ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken"):
-    assert values.get(field), f"missing required STS field: {field}"
+    assert values.get(field), values
 print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "SessionToken")))
 PY
 )
@@ -218,6 +218,7 @@ TAMPERED_STATUS="$(curl --noproxy '*' -sS \
  --data-urlencode DurationSeconds=900 \
  --data-urlencode "WebIdentityToken=${TAMPERED_TOKEN}")"
 [[ "${TAMPERED_STATUS}" == 403 ]] || {
+  cat "${WORK_DIR}/sts-tampered.xml" >&2
  echo "expected tampered token to return HTTP 403, got ${TAMPERED_STATUS}" >&2
  exit 1
 }
@@ -234,6 +235,7 @@ BAD_STATUS="$(curl --noproxy '*' -sS \
  --data-urlencode DurationSeconds=900 \
  --data-urlencode "WebIdentityToken=${BAD_TOKEN}")"
 [[ "${BAD_STATUS}" == 403 ]] || {
+  cat "${WORK_DIR}/sts-bad.xml" >&2
  echo "expected wrong-audience token to return HTTP 403, got ${BAD_STATUS}" >&2
  exit 1
 }
@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-"""Exercise the nightly publication step without AWS, network or package builds."""
-
-import hashlib
-import json
-import os
-from pathlib import Path
-import subprocess
-import tempfile
-import unittest
-
-from check_test_wiring import yaml_block
-
-
-ROOT = Path(__file__).resolve().parents[1]
-WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml"
-
-
-class NightlyCandidateTests(unittest.TestCase):
-    def setUp(self):
-        self.temp = tempfile.TemporaryDirectory()
-        self.addCleanup(self.temp.cleanup)
-        self.root = Path(self.temp.name)
-        self.package = self.root / "rustfs-nightly-2026-09-06.deb"
-        self.package.write_bytes(b"built package bytes\x00\xff")
-        for command in (["git", "init", "-q"], ["git", "add", self.package.name],
-                        ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]):
-            subprocess.run(command, cwd=self.root, check=True, capture_output=True)
-        self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip()
-        self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest()
-        self.output = self.root / "github-output"
-        self.store = self.root / "store"
-        self.shims = self.root / "fake-tools.sh"
-        self.shims.write_text(r'''aws() {
-  printf '%s\n' "$*" >> "$FAKE_AWS_LOG"
-  if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi
-  if [[ "$*" == *--generate-cli-skeleton* ]]; then
-    if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then
-      printf '{}\n'
-    else
-      printf '{"IfNoneMatch":""}\n'
-    fi
-    return
-  fi
-  if [[ "$1 $2" == 's3api put-object' ]]; then
-    [[ "$FAKE_MODE" != upload-failure ]] || return 42
-    shift 2
-    local key="" body="" condition=""
-    while [[ $# -gt 0 ]]; do
-      case "$1" in
-        --key) key="$2";;
-        --body) body="$2";;
-        --if-none-match) condition="$2";;
-      esac
-      shift 2
-    done
-    [[ -z "$condition" || "$condition" == '*' ]] || return 43
-    if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi
-    mkdir -p "$(dirname "$FAKE_STORE/$key")"
-    cp "$body" "$FAKE_STORE/$key"
-  elif [[ "$1 $2" == 's3 cp' ]]; then
-    [[ "$FAKE_MODE" != alias-failure ]] || return 45
-    local destination="${4#s3://test-bucket/}"
-    [[ "$destination" != */ ]] || destination+="$(basename "$3")"
-    mkdir -p "$(dirname "$FAKE_STORE/$destination")"
-    cp "$3" "$FAKE_STORE/$destination"
-  else
-    return 46
-  fi
-}
-curl() {
-  local url="${!#}"
-  printf '%s\n' "$url" >> "$FAKE_CURL_LOG"
-  [[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22
-  [[ "$FAKE_MODE" != missing-public-url ]] || return 22
-  if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi
-  cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22
-  [[ "$FAKE_MODE" != incomplete-download ]] || return 47
-}
-sudo() {
-  [[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49
-  [[ "$FAKE_MODE" != bootstrap-failure ]] || return 48
-}
-python3() {
-  [[ "$1 $2" == '-m venv' ]] || return 50
-  mkdir -p "$3/bin"
-  cat > "$3/bin/python" <<'SH'
-#!/usr/bin/env bash
-[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51
-[[ "$FAKE_MODE" != install-failure ]] || exit 52
-: > "$FAKE_INSTALLED"
-SH
-  printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws"
-  chmod +x "$3/bin/python" "$3/bin/aws"
-}
-''')
-        self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name,
-                        R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket",
-                        RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output),
-                        FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success")
-        source = WORKFLOW.read_text()
-        job = yaml_block(source.splitlines(), "build", 2)
-        starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
-        self.steps = {
-            job[start].split(": ", 1)[1]: job[start:end]
-            for start, end in zip(starts, starts[1:] + [len(job)])
-        }
-        self.publish = self.steps["Upload DEB to Cloudflare R2"]
-        start = self.publish.index("        run: |") + 1
-        self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith("          "))
-
-    def run_publish(self, **overrides):
-        self.output.unlink(missing_ok=True)
-        return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell],
-                              cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True)
-
-    def manifest(self):
-        output = self.output.read_text().strip()
-        self.assertTrue(output.startswith("candidate_file="), output)
-        return json.loads(Path(output.split("=", 1)[1]).read_text())
-
-    def test_success_binds_actual_package_checkout_and_attempt(self):
-        result = self.run_publish()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        manifest = self.manifest()
-        self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1,
-                                   "package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"})
-        for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"):
-            self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes())
-        self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"])
-
-    def test_missing_credentials_remain_artifact_only(self):
-        for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"):
-            with self.subTest(missing=key):
-                result = self.run_publish(**{key: ""})
-                self.assertEqual(result.returncode, 0, result.stderr)
-                self.assertFalse(self.output.exists())
-                self.assertFalse((self.root / "aws.log").exists())
-                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
-
-    def test_publication_failures_never_emit_a_candidate(self):
-        for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")):
-            with self.subTest(mode=mode):
-                result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index))
-                self.assertNotEqual(result.returncode, 0, result.stdout)
-                self.assertFalse(self.output.exists())
-                self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
-
-    def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self):
-        result = self.run_publish(FAKE_MODE="old-cli")
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertTrue((self.root / "installed").exists())
-        self.assertEqual(self.manifest()["package_sha256"], self.digest)
-        self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
-
-    def test_failed_cli_bootstrap_cannot_publish(self):
-        for mode in ("bootstrap-failure", "install-failure", "broken-install"):
-            with self.subTest(mode=mode):
-                (self.root / "installed").unlink(missing_ok=True)
-                result = self.run_publish(FAKE_MODE=mode)
-                self.assertNotEqual(result.returncode, 0)
-                self.assertFalse(self.output.exists())
-                self.assertFalse(self.store.exists())
-                self.assertEqual(list(self.root.glob("nightly-awscli.*")), [])
-
-    def test_checkout_sha_mismatch_fails_before_upload(self):
-        result = self.run_publish(GITHUB_SHA="f" * 40)
-        self.assertNotEqual(result.returncode, 0)
-        self.assertIn("Checkout SHA", result.stderr)
-        self.assertFalse(self.output.exists())
-        self.assertFalse((self.root / "aws.log").exists())
-
-    def test_same_date_builds_and_reruns_keep_distinct_candidates(self):
-        urls = []
-        for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")):
-            result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt)
-            self.assertEqual(result.returncode, 0, result.stderr)
-            urls.append(self.manifest()["package_url"])
-        self.assertEqual(len(set(urls)), 3)
-        self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3)
-
-    def test_duplicate_key_is_not_overwritten_or_recertified(self):
-        result = self.run_publish()
-        self.assertEqual(result.returncode, 0, result.stderr)
-        key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/")
-        stored = self.store / key
-        stored.write_bytes(b"preexisting conflicting object")
-        (self.root / "nightly-candidate-12345-1.json").unlink()
-        result = self.run_publish()
-        self.assertNotEqual(result.returncode, 0)
-        self.assertEqual(stored.read_bytes(), b"preexisting conflicting object")
-        self.assertFalse(self.output.exists())
-        self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), [])
-
-    def test_manifest_upload_requires_publication_output(self):
-        upload = self.steps["Upload nightly candidate manifest"]
-        self.assertIn("        id: publish", self.publish)
-        self.assertIn("          DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish)
-        self.assertIn("        if: ${{ steps.publish.outputs.candidate_file != '' }}", upload)
-        self.assertIn("          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload)
-        self.assertIn("          path: ${{ steps.publish.outputs.candidate_file }}", upload)
-        self.assertIn("          if-no-files-found: error", upload)
-        self.assertNotIn("        continue-on-error: true", self.publish)
-        self.assertNotIn("          overwrite: true", upload)
-
-
-if __name__ == "__main__":
-    unittest.main()
@@ -62,14 +62,20 @@ exit 1
 STUB
 chmod +x "$TMP_ROOT/bin/python3"

-ln -s "$(command -v bash)" "$TMP_ROOT/bin/bash"
-if PATH="$TMP_ROOT/bin" RUSTFS_PYTHON="" "$RESOLVER" -c 'pass' \
-    >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then
-    fail "resolver succeeded with no usable interpreter on PATH"
+SANDBOX_PATH="$TMP_ROOT/bin:/usr/bin:/bin"
+if PATH="$SANDBOX_PATH" command -v uv >/dev/null 2>&1; then
+    # uv is reachable even from the sandbox PATH, so the resolver would
+    # legitimately fall back to it instead of failing. Skip this case.
+    echo "ℹ️  uv is on the sandbox PATH; skipping the no-interpreter case"
+else
+    if PATH="$SANDBOX_PATH" "$RESOLVER" -c 'pass' \
+        >"$TMP_ROOT/none.out" 2>"$TMP_ROOT/none.err"; then
+        fail "resolver succeeded with no usable interpreter on PATH"
+    fi
+    grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \
+        || fail "missing-interpreter failure did not name the requirement"
+    grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \
+        || fail "missing-interpreter failure did not point at the override"
 fi
-grep -q 'No Python 3.11+ interpreter found' "$TMP_ROOT/none.err" \
-    || fail "missing-interpreter failure did not name the requirement"
-grep -q 'RUSTFS_PYTHON=' "$TMP_ROOT/none.err" \
-    || fail "missing-interpreter failure did not point at the override"

 echo "✅ scripts/python_bin.sh resolver checks passed"
@@ -1,360 +0,0 @@
-#!/usr/bin/env python3
-"""Synthetic adapter and failure-propagation tests; never start a RustFS server."""
-
-import contextlib
-import copy
-import fcntl
-import io
-import json
-import os
-from pathlib import Path
-import shlex
-import signal
-import subprocess
-import sys
-import tempfile
-import time
-import unittest
-from unittest.mock import patch
-
-import scanner_abba as harness
-
-
-def fake_adapter():
-    action, request_path, output_path = sys.argv[1:]
-    request = harness.read_json(Path(request_path))
-    fault = os.environ.get("SCANNER_ABBA_TEST_FAULT", "")
-    if fault == "stubborn-child" and action in ("prepare", "measure"):
-        marker = Path(request_path).parent / "stubborn.pid"
-        if os.fork() == 0:
-            os.execv(sys.executable, [sys.executable, str(Path(__file__).resolve()), "--stubborn-worker", str(marker)])
-        wait_for_marker(marker)
-        if action == "measure":
-            time.sleep(60)
-    if action == "prepare":
-        result = {"ready": True}
-    elif action == "stop":
-        if fault == "stubborn-child":
-            reap_fixture(Path(request_path).parent / "stubborn.pid")
-        result = {"stopped": True}
-    elif action == "oracle":
-        if fault == "oracle-exit":
-            return 42
-        if fault == "missing-oracle":
-            return 0
-        result = {"complete": True, "errors": 0, "actual": request["expected_oracle"]}
-        if fault == "oracle-mismatch":
-            result["actual"]["bytes"] += 1
-    else:
-        if fault == "measure-exit":
-            return 42
-        result = {key: request[key] for key in ("evidence", "fixed", "build", "data_dir", "background")}
-        result.update({"sample_count": 10, "elapsed_seconds": request["duration_seconds"],
-                       "metrics": dict.fromkeys(harness.METRICS, 10)})
-        baseline = request["comparison"] == "build" and request["leg"].startswith("A")
-        result["metrics"].update(p99_ms=10, throughput_ops=100, errors=0, requests=100,
-                                 walk_objects=100 if baseline else 20, cold_walk_objects=100 if baseline else 0,
-                                 healed_objects=request["expected_healed_objects"])
-        result["convergence"] = {"writes_stopped": True, "last_mutation_observed": True,
-                                 "first_complete_publication": True, "last_mutation_time": 1,
-                                 "last_mutation_observed_time": 2,
-                                 "writes_stopped_time": 2, "window_start": 2, "window_end": 3,
-                                 "budget_available_seconds": 1, "walk_objects": 110, "full_walk_objects": 100}
-        if fault == "zero-samples":
-            result["sample_count"] = 0
-        elif fault == "request-errors":
-            result["metrics"]["errors"] = 1
-        elif fault == "load-drift":
-            result["fixed"]["offered_load_ops"] += 1
-        elif fault == "noise" and request["leg"] == "A2":
-            result["metrics"]["p99_ms"] = 20
-        elif fault == "zero-requests":
-            result["metrics"]["requests"] = 0
-        elif fault == "no-publication":
-            result["convergence"]["first_complete_publication"] = False
-        elif fault == "p2-regression":
-            result["convergence"]["walk_objects"] = 121
-        elif fault == "latency-regression" and request["leg"].startswith("B"):
-            result["metrics"]["p99_ms"] = 12
-        elif fault == "exact-thresholds" and request["leg"].startswith("B"):
-            result["metrics"].update(p99_ms=10.5, throughput_ops=97)
-        elif fault == "just-over-threshold" and request["comparison"] == "build" and request["leg"].startswith("B"):
-            result["metrics"]["p99_ms"] = 10.500001
-        elif fault == "p1-regression" and not baseline:
-            result["metrics"]["walk_objects"] = 30
-        elif fault in ("p1-exact-fraction", "p1-over-fraction"):
-            result["metrics"].update(walk_objects=9 if baseline else 5 + (fault == "p1-over-fraction"),
-                                     cold_walk_objects=5 if baseline else 0)
-        elif fault == "missing-metric":
-            del result["metrics"]["save_bytes"]
-        elif fault == "incomplete-repair":
-            result["metrics"]["healed_objects"] = 0
-    harness.write_json(Path(output_path), result)
-    return 0
-
-
-def wait_for_marker(marker):
-    deadline = time.monotonic() + 5
-    while time.monotonic() < deadline:
-        if marker.exists() and marker.stat().st_size:
-            return
-        time.sleep(0.01)
-    raise AssertionError("fixture child did not become ready")
-
-
-def child_released(marker, timeout=1):
-    deadline = time.monotonic() + timeout
-    with marker.open("r+") as stream:
-        while True:
-            try:
-                fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                return True
-            except BlockingIOError:
-                if time.monotonic() >= deadline:
-                    return False
-                time.sleep(0.01)
-
-
-def reap_fixture(marker):
-    if marker.exists() and marker.stat().st_size and not child_released(marker, timeout=0):
-        # The unique file lock proves the original fixture process still owns this PID.
-        os.kill(int(marker.read_text()), signal.SIGKILL)
-        if not child_released(marker, timeout=5):
-            raise AssertionError("fixture child did not release its process-owned lock")
-
-
-class ScannerAbbaTest(unittest.TestCase):
-    def setUp(self):
-        self.temp = tempfile.TemporaryDirectory()
-        self.addCleanup(self.temp.cleanup)
-        self.root = Path(self.temp.name)
-        self.binary = Path(sys.executable).resolve()
-        self.adapter = Path(__file__).resolve()
-        self.manifest = {
-            "schema": 1, "evidence": "synthetic", "rounds": 3, "duration_seconds": 1, "min_free_bytes": 1,
-            "fixed": {"config_sha256": "1" * 64, "dataset_sha256": "2" * 64,
-                      "release_flags": "--release", "durability": "drive-sync=on",
-                      "disk_type": "synthetic", "cache_state": "cold", "load_command": "fake",
-                      "topology": "EC8+4", "offered_load_ops": 100, "resource_isolation": "synthetic"},
-            "oracles": {s: {"objects": 10, "versions": 20, "bytes": 30, "sha256": "3" * 64} for s in harness.SCENARIOS},
-            "expected_healed_objects": {s: 10 for s in harness.SCENARIOS},
-        }
-        build = {"binary": str(self.binary), "sha256": harness.digest(self.binary), "revision": "a" * 40}
-        self.manifest.update(baseline=build.copy(), candidate=build.copy())
-
-    def run_harness(self, fault=""):
-        with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()):
-            return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data")
-
-    def test_adapter_timeout_reaps_group_after_parent_exits_on_term(self):
-        request = self.root / "request.json"
-        harness.write_json(request, {})
-        marker = self.root / "stubborn.pid"
-        try:
-            with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}):
-                with self.assertRaises(subprocess.TimeoutExpired):
-                    harness.invoke(self.adapter, "measure", request, 3)
-            wait_for_marker(marker)
-            self.assertTrue(child_released(marker), "TERM-exited parent left its TERM-ignoring child alive")
-        finally:
-            reap_fixture(marker)
-
-    def test_collector_failure_reaps_group_after_parent_exits_on_term(self):
-        request = self.root / "request.json"
-        harness.write_json(request, {})
-        marker = self.root / "stubborn.pid"
-        collector = self.root / "run_scanner_validation_harness.sh"
-        command = [sys.executable, str(self.adapter), "measure", str(request), str(self.root / "unused.json")]
-        collector.write_text("#!/usr/bin/env bash\nexec " + shlex.join(command) + "\n")
-
-        def failed_measure(*_):
-            wait_for_marker(marker)
-            raise ValueError("injected measurement failure")
-
-        try:
-            with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \
-                 patch.object(harness, "__file__", str(self.root / "scanner_abba.py")), \
-                 patch.object(harness, "invoke", side_effect=failed_measure):
-                with self.assertRaisesRegex(ValueError, "injected measurement failure"):
-                    harness.collect_live({"collector": {"alias": "fixture", "endpoint": "fixture", "metrics_endpoints": "fixture"}},
-                                         {"duration_seconds": 900}, request, self.adapter)
-            self.assertTrue(child_released(marker), "collector parent exit did not end its telemetry child")
-        finally:
-            reap_fixture(marker)
-
-    def test_successful_prepare_keeps_service_alive(self):
-        request = self.root / "request.json"
-        harness.write_json(request, {})
-        marker = self.root / "stubborn.pid"
-        try:
-            with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}):
-                self.assertEqual(harness.invoke(self.adapter, "prepare", request, 5), {"ready": True})
-                self.assertFalse(child_released(marker, timeout=0), "successful prepare must preserve its service")
-                self.assertEqual(harness.invoke(self.adapter, "stop", request, 5), {"stopped": True})
-            self.assertTrue(child_released(marker), "adapter stop must release its service")
-        finally:
-            reap_fixture(marker)
-
-    def test_reaped_owner_never_signals_a_reused_process_group(self):
-        with (self.root / "owner.log").open("wb") as log:
-            owner = harness.OwnedCommand([sys.executable, "-c", "pass"], log)
-            self.assertEqual(owner.wait(5), 0)
-            self.assertEqual(owner.finish(), 0)
-            with patch.object(harness.os, "killpg", side_effect=AssertionError("released PGID must not be signalled")):
-                self.assertEqual(owner.finish(terminate=True), 0)
-
-    def test_cleanup_interruption_still_kills_group_and_reaps_leader(self):
-        request = self.root / "request.json"
-        harness.write_json(request, {})
-        marker = self.root / "stubborn.pid"
-        original_sleep = time.sleep
-        interrupted = False
-
-        def interrupt_once(delay):
-            nonlocal interrupted
-            if not interrupted:
-                interrupted = True
-                raise KeyboardInterrupt
-            original_sleep(delay)
-
-        with (self.root / "interrupted.log").open("wb") as log:
-            with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}):
-                owner = harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log)
-            try:
-                wait_for_marker(marker)
-                with patch.object(harness.time, "sleep", side_effect=interrupt_once):
-                    with self.assertRaises(KeyboardInterrupt):
-                        owner.finish(terminate=True)
-                self.assertTrue(child_released(marker), "cleanup cancellation left its child alive")
-                self.assertIsNotNone(owner.process.returncode, "cleanup cancellation must reap its leader")
-            finally:
-                reap_fixture(marker)
-                owner.process.wait(timeout=5)
-
-    def test_constructor_failure_after_gate_release_kills_group(self):
-        request = self.root / "request.json"
-        harness.write_json(request, {})
-        marker = self.root / "stubborn.pid"
-        original_write = os.write
-
-        def release_then_fail(fd, data):
-            original_write(fd, data)
-            wait_for_marker(marker)
-            raise OSError("injected failure after gate release")
-
-        try:
-            with (self.root / "construction.log").open("wb") as log, \
-                 patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \
-                 patch.object(harness.os, "write", side_effect=release_then_fail):
-                with self.assertRaisesRegex(OSError, "injected failure after gate release"):
-                    harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log)
-            self.assertTrue(child_released(marker), "initialization failure left its child alive")
-        finally:
-            reap_fixture(marker)
-
-    def test_complete_synthetic_matrix_is_not_performance_evidence(self):
-        self.assertEqual(self.run_harness(), 0)
-        report = harness.read_json(self.root / "out/report.json")
-        self.assertEqual((report["status"], report["performance"], report["cells"]), ("synthetic_validated", "pending", 120))
-        requests = [harness.read_json(path) for path in (self.root / "out").glob("*/request.json")]
-        self.assertEqual(len({r["data_dir"] for r in requests}), 120)
-        for scenario in harness.SCENARIOS:
-            for comparison in ("build", "background"):
-                for round_id in (1, 2, 3):
-                    legs = [r for r in requests if (r["scenario"], r["comparison"], r["round"]) == (scenario, comparison, round_id)]
-                    self.assertEqual({r["leg"] for r in legs}, set(harness.LEGS))
-        self.assertTrue(all(c["p2_max_work_multiple"] == 1.2 for c in report["comparisons"]))
-
-    def test_fail_closed_adapter_and_data_errors(self):
-        for fault in ("measure-exit", "oracle-exit", "missing-oracle", "oracle-mismatch", "zero-samples",
-                      "zero-requests", "request-errors", "load-drift", "missing-metric", "incomplete-repair"):
-            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
-                self.root = Path(directory)
-                with self.assertRaises((ValueError, OSError, subprocess.SubprocessError)):
-                    self.run_harness(fault)
-                report = harness.read_json(self.root / "out/report.json")
-                self.assertEqual(report["status"], "failed")
-                self.assertTrue(list((self.root / "out").glob("*/stop.json")))
-
-    def test_noise_is_inconclusive_and_nonzero(self):
-        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
-            self.assertEqual(self.run_harness("noise"), 3)
-        self.assertEqual(harness.read_json(self.root / "out/report.json")["status"], "inconclusive")
-
-    def test_missing_first_publication_is_inconclusive(self):
-        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
-            self.assertEqual(self.run_harness("no-publication"), 3)
-
-    def test_performance_regressions_fail(self):
-        for fault in ("p1-regression", "p2-regression", "latency-regression"):
-            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
-                self.root = Path(directory)
-                with patch.object(harness, "SCENARIOS", ("cold-hot",)):
-                    self.assertEqual(self.run_harness(fault), 1)
-
-    def test_exact_threshold_boundaries_pass(self):
-        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
-            self.assertEqual(self.run_harness("exact-thresholds"), 0)
-
-    def test_just_over_threshold_fails(self):
-        with patch.object(harness, "SCENARIOS", ("cold-hot",)):
-            self.assertEqual(self.run_harness("just-over-threshold"), 1)
-
-    def test_p1_fractional_boundary(self):
-        for fault, expected in (("p1-exact-fraction", 0), ("p1-over-fraction", 1)):
-            with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
-                self.root = Path(directory)
-                with patch.object(harness, "SCENARIOS", ("cold-hot",)):
-                    self.assertEqual(self.run_harness(fault), expected)
-
-    def test_manifest_rejects_missing_build_or_oracle(self):
-        for section, key in (("baseline", "binary"), ("oracles", "cold-hot")):
-            manifest = copy.deepcopy(self.manifest)
-            del manifest[section][key]
-            with self.subTest(section=section), self.assertRaises((ValueError, KeyError)):
-                harness.validate_manifest(manifest)
-
-    def test_short_measured_window_and_fewer_rounds_rejected(self):
-        self.manifest["evidence"] = "measured"
-        with self.assertRaisesRegex(ValueError, "duration_seconds"):
-            harness.validate_manifest(self.manifest)
-        self.manifest["duration_seconds"] = 900
-        self.manifest["rounds"] = 2
-        with self.assertRaisesRegex(ValueError, "rounds"):
-            harness.validate_manifest(self.manifest)
-
-    def test_existing_data_preserved(self):
-        (self.root / "data").mkdir()
-        marker = self.root / "data/keep"
-        marker.write_text("existing")
-        with self.assertRaisesRegex(ValueError, "preserved"):
-            self.run_harness()
-        self.assertEqual(marker.read_text(), "existing")
-
-    def test_invalid_or_live_write_window_does_not_claim_p2(self):
-        self.assertIsNone(harness.convergence({"convergence": {"writes_stopped": False}}))
-        with self.assertRaises(ValueError):
-            harness.convergence({"convergence": {"writes_stopped": True, "last_mutation_observed": True,
-                                                 "first_complete_publication": True}})
-
-    def test_nan_and_oversized_samples_rejected(self):
-        with self.assertRaises(ValueError):
-            harness.number(float("nan"), "latency")
-        path = self.root / "oversized.json"
-        path.write_bytes(b" " * (harness.MAX_JSON_BYTES + 1))
-        with self.assertRaisesRegex(ValueError, "oversized"):
-            harness.read_json(path)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) == 3 and sys.argv[1] == "--stubborn-worker":
-        signal.signal(signal.SIGTERM, signal.SIG_IGN)
-        with Path(sys.argv[2]).open("w+") as marker:
-            fcntl.flock(marker, fcntl.LOCK_EX)
-            marker.write(str(os.getpid()))
-            marker.flush()
-            while True:
-                time.sleep(1)
-    if len(sys.argv) == 4 and sys.argv[1] in ("prepare", "measure", "oracle", "stop"):
-        sys.exit(fake_adapter())
-    unittest.main()
@@ -312,5 +312,3 @@ if PATH="$BIN_DIR:$PATH" "$SCRIPT" --secret-key rustfsadmin >"$secret_arg_log" 2
 fi

 grep -q -- 'unknown arg: --secret-key' "$secret_arg_log"
-
-"$ROOT_DIR/scripts/python_bin.sh" "$ROOT_DIR/scripts/test_scanner_abba.py"
@@ -1,20 +1,16 @@
 #!/usr/bin/env python3
-"""Exercise functional failures, chain dispatch, and security evidence without remote VMs."""
+"""Exercise functional workflow failures and security evidence without remote VMs."""

 from __future__ import annotations

-import glob
-import json
 import os
 import re
 import subprocess
-import sys
 import tempfile
 import unittest
 from pathlib import Path

 from check_test_wiring import yaml_block
-from functional_case_report import generate_report


 ROOT = Path(__file__).resolve().parents[1]
@@ -42,46 +38,7 @@ def shell_body(lines: list[str]) -> str:
    return "\n".join(shell_lines)


-class WorkflowSteps:
-    def uploaded_files(self) -> set[Path]:
-        upload = next(lines for lines in self.steps.values() if any("uses: actions/upload-artifact@" in line for line in lines))
-        start = upload.index("          path: |") + 1
-        paths = []
-        for line in upload[start:]:
-            if not line.startswith("            "):
-                break
-            paths.extend(Path(path) for path in glob.glob(self.render(line.strip())))
-        return {file for path in paths for file in (path.rglob("*") if path.is_dir() else [path]) if file.is_file()}
-
-    def render(self, value: str) -> str:
-        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
-
-    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
-        result = {}
-        for line in yaml_block(lines, "env", indent) or []:
-            if line.strip() and not line.lstrip().startswith("#"):
-                key, value = line.strip().split(": ", 1)
-                result[key] = self.render(value.strip("'\""))
-        return result
-
-    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
-        lines = self.steps[name]
-        result = subprocess.run(
-            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
-            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
-        )
-        for line in lines:
-            if line.startswith("        id: "):
-                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
-        if Path(self.env["GITHUB_ENV"]).exists():
-            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
-                key, value = line.split("=", 1)
-                self.env[key] = value
-                self.context[f"env.{key}"] = value
-        return result
-
-
-class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
+class SecurityWorkflowTests(unittest.TestCase):
    def setUp(self) -> None:
        self.source = WORKFLOW.read_text()
        self.job = yaml_block(self.source.splitlines(), "security-test", 2)
@@ -121,7 +78,6 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
            '#!/usr/bin/env bash\nset -euo pipefail\n'
            'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
            'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n'
-            'echo "CURRENT SUITE STDOUT"; echo "CURRENT SUITE STDERR" >&2\n'
            'case "$FAKE_REPORT" in\n'
            f'  present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
            '  empty) : > "$REPORT_FILE" ;;\n'
@@ -130,6 +86,33 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
            'exit "$FAKE_EXIT"\n'
        )

+    def render(self, value: str) -> str:
+        return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
+
+    def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
+        result = {}
+        for line in yaml_block(lines, "env", indent) or []:
+            if line.strip() and not line.lstrip().startswith("#"):
+                key, value = line.strip().split(": ", 1)
+                result[key] = self.render(value.strip("'\""))
+        return result
+
+    def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
+        lines = self.steps[name]
+        result = subprocess.run(
+            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render(shell_body(lines))],
+            cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
+        )
+        for line in lines:
+            if line.startswith("        id: "):
+                self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
+        if Path(self.env["GITHUB_ENV"]).exists():
+            for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
+                key, value = line.split("=", 1)
+                self.env[key] = value
+                self.context[f"env.{key}"] = value
+        return result
+
    def test_workflow_wiring(self) -> None:
        names = list(self.steps)
        self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
@@ -145,7 +128,7 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
        for name in ("Upload functional report to dashboard", "Upload report and logs"):
            self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
        artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
-        self.assertIn("          path: |", artifact_settings)
+        self.assertIn("          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings)
        self.assertIn("          if-no-files-found: error", artifact_settings)

    def test_suite_report_and_result_matrix(self) -> None:
@@ -164,7 +147,7 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
                if outcome != "skipped" or mode == "present":
                    suite = self.run_step("Run security suite")
                    self.assertEqual(suite.returncode, exit_code, suite.stderr)
-                    logs = list(Path(str(self.artifacts) + "-scratch").glob("rustfs-security.*/suite.log"))
+                    logs = list(self.artifacts.glob("rustfs-security.*/suite.log"))
                    self.assertEqual(len(logs), 1)
                    self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
                self.context["steps.test.outcome"] = outcome
@@ -186,195 +169,37 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
                summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
                self.assertEqual(summary, contents)
                self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
-                expected = {self.artifacts / "report.md"}
-                if outcome != "skipped" or mode == "present":
-                    expected.add(self.artifacts / "suite.log")
-                    self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE STDOUT\nCURRENT SUITE STDERR\n")
-                if mode in ("present", "empty"):
-                    expected.add(self.artifacts / "suite-report.md")
-                (self.artifacts / "unexpected-token.json").write_text("FAKE-SECRET-CANARY")
-                scratch = Path(str(self.artifacts) + "-scratch")
-                (scratch / "case.out").write_text("FAKE-SECRET-CANARY")
-                self.assertEqual(self.uploaded_files(), expected)
-
-
-    def test_oidc_negative_responses_never_print_issued_credentials(self):
-        source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
-        for variable, filename in (("TAMPERED_STATUS", "sts-tampered.xml"), ("BAD_STATUS", "sts-bad.xml")):
-            start = source.index('[[ "${' + variable + '}" == 403 ]]')
-            end = source.index("\n", source.index("grep -q '<Code>AccessDenied</Code>'", start))
-            guard = source[start:end]
-            credential_xml = "<Credentials><AccessKeyId>FAKE-ACCESS-CANARY</AccessKeyId><SecretAccessKey>FAKE-SECRET-CANARY</SecretAccessKey><SessionToken>FAKE-SESSION-CANARY</SessionToken></Credentials>"
-            for status, body, expected in (("200", credential_xml, 1), ("403", credential_xml, 1),
-                                           ("403", "<Code>AccessDenied</Code>", 0)):
-                with self.subTest(variable=variable, status=status, expected=expected):
-                    (self.directory / filename).write_text(body)
-                    result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", guard],
-                                            env={**self.env, variable: status, "WORK_DIR": str(self.directory)},
-                                            capture_output=True, text=True)
-                    self.assertEqual(result.returncode, expected, result.stderr)
-                    for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
-                        self.assertNotIn(canary, result.stdout + result.stderr)
-                    if status == "200":
-                        self.assertIn("HTTP 403, got 200", result.stderr)
-
-    def test_oidc_incomplete_credentials_report_only_the_missing_field(self):
-        source = (ROOT / "scripts/test/oidc_keycloak_live.sh").read_text()
-        start = source.index("IFS=$'\\t' read -r STS_ACCESS_KEY")
-        end = source.index("\n)\n", start) + 3
-        extract = source[start:end]
-        values = {"AccessKeyId": "FAKE-ACCESS-CANARY", "SecretAccessKey": "FAKE-SECRET-CANARY",
-                  "SessionToken": "FAKE-SESSION-CANARY", "Expiration": "2099-01-01T00:00:00Z",
-                  "SubjectFromWebIdentityToken": "alice"}
-        for missing in (None, "Expiration", "SubjectFromWebIdentityToken"):
-            with self.subTest(missing=missing):
-                xml = "<Credentials>" + "".join(f"<{key}>{value}</{key}>" for key, value in values.items() if key != missing) + "</Credentials>"
-                (self.directory / "sts-good.xml").write_text(xml)
-                result = subprocess.run(["bash", "-e", "-o", "pipefail", "-c", extract],
-                                        env={**self.env, "WORK_DIR": str(self.directory)}, capture_output=True, text=True)
-                self.assertEqual(result.returncode, 1 if missing else 0, result.stderr)
-                for canary in ("FAKE-ACCESS-CANARY", "FAKE-SECRET-CANARY", "FAKE-SESSION-CANARY"):
-                    self.assertNotIn(canary, result.stdout + result.stderr)
-                if missing:
-                    self.assertIn(f"missing required STS field: {missing}", result.stderr)

    def test_existing_evidence_directory_is_rejected(self) -> None:
-        for suffix in ("", "-scratch"):
-            self.setUp()
-            existing = Path(str(self.artifacts) + suffix)
-            existing.mkdir()
-            stale = existing / "suite-report.md"
-            stale.write_text("OLD RUN REPORT")
-            self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
-            self.assertEqual(stale.read_text(), "OLD RUN REPORT")
-            self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
-            (self.artifacts / "report.md").write_text("OLD RUN REPORT")
-            self.context.update({
-                "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
-            })
-            fake_bin = self.directory / "bin"
-            fake_bin.mkdir()
-            gh = fake_bin / "gh"
-            gh.write_text(
-                '#!/usr/bin/env bash\nset -euo pipefail\n'
-                'if [ "$1 $2" = "issue create" ]; then\n'
-                '  while [ "$#" -gt 0 ]; do\n'
-                '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
-                '    shift\n'
-                '  done\n'
-                'fi\n'
-            )
-            gh.chmod(0o755)
-            body = self.directory / "issue-body.md"
-            self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
-            result = self.run_step("File failure issue in rustfs/backlog")
-            self.assertEqual(result.returncode, 0, result.stderr)
-            self.assertNotIn("OLD RUN REPORT", body.read_text())
-            self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
-
-    def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
-        for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
-            with self.subTest(suite=suite):
-                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
-                # Workflow-level concurrency covers every job, including cleanup,
-                # regardless of trigger or the runner hosting the job.
-                self.assertEqual([
-                    line.strip() for line in yaml_block(source, "concurrency", 0)
-                    if line.strip() and not line.lstrip().startswith("#")
-                ], [
-                    "group: rustfs-shared-functional-tests", "cancel-in-progress: false",
-                ])
-                self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
-                self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
-                cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
-                cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
-                self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
-
-    def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
-        for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
-            with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
-                self.setUp()
-                fake_bin = self.directory / "bin"
-                fake_bin.mkdir()
-                commands = {
-                    "gh": '''#!/usr/bin/env bash
-set -euo pipefail
-if [ "$1" = api ]; then
-  printf '%s\\n' "$*" >> "$DISPATCHES"
-  attempt=$(wc -l < "$DISPATCHES")
-  [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
-elif [ "$1 $2" = 'issue create' ]; then
-  printf 'issue\\n' >> "$EXECUTED"
-  while [ "$#" -gt 0 ]; do
-    if [ "$1" = --body-file ]; then
-      cat "$2" > "$CAPTURE_BODY"
-      printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
-    fi
-    shift
-  done
-  exit "$ISSUE_EXIT"
-else
-  exit 99
-fi
-''',
-                    "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
-                    "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
-                }
-                for name, contents in commands.items():
-                    command = fake_bin / name
-                    command.write_text(contents)
-                    command.chmod(0o755)
-                dispatches = self.directory / "dispatches"
-                executed = self.directory / "executed"
-                body = self.directory / "issue-body.md"
-                body_path = self.directory / "issue-body-path"
-                self.env.update(
-                    PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
-                    EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
-                    FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
-                    RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
-                    RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
-                )
-                self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
-                driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
-                self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
-                self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
-                started = self.run_step("Dispatch first suite (upgrade)")
-                self.assertEqual(started.returncode, 0, started.stderr)
-                self.assertEqual(dispatches.read_text().splitlines(), [
-                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
-                ])
-                dispatches.unlink()
-
-                replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
-                job = yaml_block(replication.splitlines(), "replication-test", 2)
-                self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
-                self.steps = named_steps(job)
-                handoff = "Continue functional chain (next: Performance)"
-                self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
-                self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
-                self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
-                self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
-                suite = self.directory / "auto-testing/rustfs-replication-test.sh"
-                suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
-                failed = self.run_step("Run replication suite")
-                self.assertEqual(failed.returncode, 17, failed.stderr)
-                cleaned = self.run_step("Cleanup environment (after)")
-                self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
-                self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
-                self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
-                self.context["secrets.PF_TESTING_GH_TOKEN"] = token
-                forwarded = self.run_step(handoff)
-                self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
-                calls = dispatches.read_text().splitlines() if dispatches.exists() else []
-                self.assertEqual(calls, [
-                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
-                ] * (min(failed_attempts + 1, 3) if token else 0))
-                if failed_attempts == 3:
-                    self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
-                    self.assertIn("rustfs-chain-performance", body.read_text())
-                    self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
-                    self.assertFalse(Path(body_path.read_text().strip()).exists())
+        self.artifacts.mkdir()
+        stale = self.artifacts / "suite-report.md"
+        stale.write_text("OLD RUN REPORT")
+        self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
+        self.assertEqual(stale.read_text(), "OLD RUN REPORT")
+        self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
+        (self.artifacts / "report.md").write_text("OLD RUN REPORT")
+        self.context.update({
+            "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
+        })
+        fake_bin = self.directory / "bin"
+        fake_bin.mkdir()
+        gh = fake_bin / "gh"
+        gh.write_text(
+            '#!/usr/bin/env bash\nset -euo pipefail\n'
+            'if [ "$1 $2" = "issue create" ]; then\n'
+            '  while [ "$#" -gt 0 ]; do\n'
+            '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
+            '    shift\n'
+            '  done\n'
+            'fi\n'
+        )
+        gh.chmod(0o755)
+        body = self.directory / "issue-body.md"
+        self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
+        result = self.run_step("File failure issue in rustfs/backlog")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("OLD RUN REPORT", body.read_text())
+        self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())


 class FunctionalWorkflowTests(unittest.TestCase):
@@ -400,7 +225,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
                if suite in self.DIRECT_TESTS:
                    test = steps[self.DIRECT_TESTS[suite]]
                    self.assertNotRegex("\n".join(test), r'''(?m)^        ["']?continue-on-error["']?\s*:''')
-                    self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"])
+                    self.assertIn("        if: always()", steps["Generate report"])
                cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"]
                condition = next(line.strip() for line in cleanup if line.startswith("        if:"))
                self.assertIn(condition, (
@@ -409,7 +234,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
                    "if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}",
                ))
                if suite != "performance":
-                    handoff = next(
+                    handoff = steps["Chain complete"] if suite == "replication" else next(
                        value for name, value in steps.items() if name.startswith("Continue functional chain")
                    )
                    self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", handoff)
@@ -452,387 +277,13 @@ class FunctionalWorkflowTests(unittest.TestCase):
                self.assertIn("partial suite diagnostics", failed.stdout)
                cleanup = execute("Cleanup environment (after)")
                self.assertEqual(cleanup.returncode, 0, cleanup.stderr)
-                handoff_name = next(
+                handoff_name = "Chain complete" if suite == "replication" else next(
                    name for name in steps if name.startswith("Continue functional chain")
                )
                handoff = execute(handoff_name)
                self.assertEqual(handoff.returncode, 0, handoff.stderr)
                markers = (root / "executed").read_text().splitlines()
-                self.assertEqual(markers, ["cleanup", "dispatch"])
-
-
-class FunctionalCaseReportTests(unittest.TestCase):
-    def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
-        with tempfile.TemporaryDirectory() as directory:
-            root = Path(directory)
-            log = root / "suite.log"
-            if text is not None:
-                log.write_text(text)
-            valid = generate_report(log, root / "cases.md", root / "matrix.md" if matrix else None)
-            return valid, (root / "cases.md").read_text(), (root / "matrix.md").read_text() if matrix else ""
-
-    def test_repeated_case_executions_preserve_failure_and_context(self):
-        # log() from rustfs/auto-testing@6120aa0a76de, rustfs-kms-test.sh:131.
-        log = subprocess.check_output(["bash", "-c", r'''
-log() { printf '\033[1;36m[INFO]\033[0m %s\n' "$*"; }
-log '== topology: single-single  kms-backend: local =='
-printf '\033[32m--- KMS-101 roundtrip ---\033[0m\n[FAIL] KMS-101\n'
-log '== topology: single-multi  kms-backend: vault-kv2 =='
-printf '%s\n' '--- KMS-101 roundtrip ---' '[PASS] KMS-101'
-printf '%s\n' '--- KMS-101 roundtrip ---' '[UNSUPPORTED] KMS-101'
-'''], text=True)
-        valid, cases, _ = self.report(log)
-        self.assertFalse(valid)
-        self.assertEqual(cases.count("| KMS-101 |"), 3)
-        self.assertIn("- Total: 3\n- PASS: 1\n- FAIL: 1\n- UNSUPPORTED: 1\n- RUNNING: 0\n", cases)
-        self.assertIn("roundtrip (topology: single-single  kms-backend: local) | FAIL |", cases)
-        self.assertIn("roundtrip (topology: single-multi  kms-backend: vault-kv2) | PASS |", cases)
-        self.assertNotIn("\\n", cases)
-
-    def test_missing_empty_unfinished_and_orphan_results_are_not_success(self):
-        for text in (None, "", "setup failed\n", "--- KMS-101 roundtrip ---\n", "[PASS] KMS-101\n",
-                     "--- KMS-101 first ---\n--- KMS-101 second ---\n[PASS] KMS-101\n",
-                     "--- KMS-101 first ---\n[FAIL] KMS-101\n[PASS] KMS-101\n"):
-            with self.subTest(log=text):
-                valid, cases, _ = self.report(text)
-                self.assertFalse(valid)
-                self.assertIn("## Case Summary", cases)
-        valid, cases, _ = self.report("[INFO] == suite: bucket replication (REP-*) ==\n--- REP-101 unsupported ---\n[UNSUPPORTED] REP-101\n")
-        self.assertTrue(valid)
-        self.assertIn("suite: bucket replication", cases)
-        self.assertIn("- UNSUPPORTED: 1\n", cases)
-
-    def test_upgrade_matrix_is_preserved_and_required_for_complete_report(self):
-        case = "--- UPG-101 upgrade ---\n[PASS] UPG-101\n"
-        for suffix, expected in (("", False), ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n", True),
-                                 ("[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=1\n", False)):
-            with self.subTest(matrix=suffix):
-                valid, _, matrix = self.report(case + suffix, matrix=True)
-                self.assertEqual(valid, expected)
-                self.assertIn("| Topology | KMS Backend | From Version | To Version | Result |", matrix)
-                self.assertIn("| single-single | local | v1 | v2 |" if suffix else "NOT RUN", matrix)
-
-    def test_s3_case_identifiers_include_digits(self):
-        valid, cases, _ = self.report("--- S3C-101 CreateBucket ---\n[PASS] S3C-101\n")
-        self.assertTrue(valid)
-        self.assertIn("| S3C-101 | CreateBucket (context not recorded) | PASS |", cases)
-
-
-class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
-    SUITES = (*FunctionalWorkflowTests.DIRECT_TESTS, "heal", "performance")
-
-    def prepare(self, suite: str) -> None:
-        self.temp = tempfile.TemporaryDirectory()
-        self.addCleanup(self.temp.cleanup)
-        self.directory = Path(self.temp.name)
-        self.source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
-        self.steps = named_steps(yaml_block(self.source.splitlines(), FunctionalWorkflowTests.JOBS[suite], 2))
-        self.context = {expression: "" for expression in re.findall(r"\$\{\{\s*(.*?)\s*\}\}", self.source)}
-        self.context.update({
-            "github.server_url": "https://github.com", "github.repository": "rustfs/rustfs",
-            "github.run_id": "314159", "github.run_attempt": "2", "github.sha": "0123456789abcdef0123456789abcdef01234567",
-            "github.event_name": "repository_dispatch", "steps.test.outcome": "success",
-            "secrets.PF_TESTING_GH_TOKEN": "local-fixture", "env.PF_TESTING_GH_TOKEN": "local-fixture",
-        })
-        self.artifacts = self.directory / f"rustfs-{suite}-314159-2"
-        self.env = {
-            **os.environ, "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name,
-            "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "RUSTFS_NODES": "fixture-node",
-            "RUSTFS_NIGHTLY_PACKAGE_URL": "https://example.invalid/package.deb", "CAPTURE_BODY": str(self.directory / "issue.md"),
-        }
-        for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
-            self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
-        (self.directory / "scripts").mkdir()
-        (self.directory / "scripts/functional_case_report.py").symlink_to(ROOT / "scripts/functional_case_report.py")
-        fake_bin = self.directory / "bin"
-        fake_bin.mkdir()
-        (fake_bin / "python3").symlink_to(sys.executable)
-        for command, body in (
-            ("ssh", 'printf "fixture-version\\n"\n'),
-            ("gh", 'if [ "$1 $2" = "issue create" ]; then\n'
-                   '  while [ "$#" -gt 0 ]; do\n'
-                   '    if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
-                   '    shift\n'
-                   '  done\n'
-                   'elif [ "$1 $2" = "api --method" ]; then cat >/dev/null; fi\n'),
-        ):
-            script = fake_bin / command
-            script.write_text("#!/bin/sh\n" + body)
-            script.chmod(0o755)
-        self.env["PATH"] = f"{fake_bin}{os.pathsep}{os.environ['PATH']}"
-
-    def test_evidence_wiring_and_failed_initialization_cannot_publish_stale_files(self):
-        for suite, suffix in ((suite, suffix) for suite in self.SUITES for suffix in ("", "-scratch")):
-            with self.subTest(suite=suite, collision=suffix or "artifact"):
-                self.prepare(suite)
-                self.assertNotIn("/tmp/rustfs-", self.source)
-                names = list(self.steps)
-                self.assertLess(names.index("Initialize functional evidence"), names.index("Checkout auto-testing scripts (with retry)"))
-                if suite in FunctionalWorkflowTests.DIRECT_TESTS:
-                    self.assertLess(names.index("Checkout repository (for report parser)"), names.index("Checkout auto-testing scripts (with retry)"))
-                for name, lines in self.steps.items():
-                    if name in ("Generate report", "Upload functional report to dashboard") or any("uses: actions/upload-artifact@" in line for line in lines):
-                        self.assertIn("        if: ${{ always() && steps.evidence.outcome == 'success' }}", lines)
-                    if any("uses: actions/upload-artifact@" in line for line in lines):
-                        self.assertIn("          path: |", lines)
-                        self.assertIn("          if-no-files-found: error", lines)
-                existing = Path(str(self.artifacts) + suffix)
-                existing.mkdir()
-                for filename in ("report.md", "suite.log"):
-                    (existing / filename).write_text("OLD RUN EVIDENCE")
-                self.env.update(REPORT_FILE=str(existing / "report.md"), LOG_FILE=str(existing / "suite.log"))
-                initialized = self.run_step("Initialize functional evidence")
-                self.assertNotEqual(initialized.returncode, 0)
-                self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
-                issue = self.run_step("File failure issue in rustfs/backlog")
-                self.assertEqual(issue.returncode, 0, issue.stderr)
-                body = Path(self.env["CAPTURE_BODY"]).read_text()
-                self.assertNotIn("OLD RUN EVIDENCE", body)
-                self.assertIn("no report or log file was produced", body)
-                self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
-
-    def test_reports_use_only_current_complete_suite_evidence(self):
-        for suite in self.SUITES[:-1]:
-            good = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n"
-            partial = "--- KMS-101 roundtrip ---\n[PASS] KMS-101\n--- KMS-102 unfinished ---\n"
-            if suite == "s3-compat":
-                good, partial = good.replace("KMS-", "S3C-"), partial.replace("KMS-", "S3C-")
-            if suite == "upgrade":
-                good += "[UPG-TOPO] single-single local v1 v2 PASS=1 FAIL=0\n"
-            if suite == "heal":
-                good = "".join(f"[HEAL-STEP] {step} fixture PASS\n" for step in range(1, 8))
-                partial = "[HEAL-STEP] 1 fixture PASS\n"
-            for outcome, log in (("success", good), ("failure", good), ("success", partial), ("success", ""),
-                                 ("success", None), ("skipped", None), ("cancelled", good)):
-                with self.subTest(suite=suite, outcome=outcome, log=log):
-                    self.prepare(suite)
-                    stale = self.directory / "old-suite.log"
-                    stale.write_text("OLD RUN EVIDENCE\n" + good)
-                    self.env.update(LOG_FILE=str(stale), REPORT_FILE=str(stale))
-                    self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
-                    self.assertEqual(self.env["LOG_FILE"], str(self.artifacts / "suite.log"))
-                    self.assertEqual(self.env["TMPDIR"], str(self.artifacts) + "-scratch")
-                    if log is not None:
-                        Path(self.env["LOG_FILE"]).write_text(log)
-                    self.context["steps.test.outcome"] = outcome
-                    report = self.run_step("Generate report")
-                    success = outcome == "success" and log == good
-                    self.assertEqual(report.returncode == 0, success, report.stderr)
-                    contents = Path(self.env["REPORT_FILE"]).read_text()
-                    self.assertNotIn("OLD RUN EVIDENCE", contents)
-                    self.assertEqual("| PASS |" in contents, success)
-                    for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"],
-                                  f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"):
-                        self.assertIn(value, contents)
-                    self.assertEqual(Path(self.env["GITHUB_STEP_SUMMARY"]).read_text(), contents)
-                    evidence = (self.artifacts / ("steps.md" if suite == "heal" else "cases.md")).read_text()
-                    if log in (good, partial):
-                        self.assertIn("| PASS |", evidence)
-                    self.assertNotIn("OLD RUN EVIDENCE", evidence)
-
-    def test_actual_suite_commands_pass_the_current_log_and_scratch_paths(self):
-        for suite in self.SUITES:
-            with self.subTest(suite=suite):
-                self.prepare(suite)
-                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
-                if suite == "heal":
-                    self.assertEqual(self.env["RUSTFS_WARP_LOG_FILE"], str(self.artifacts / "warp.log"))
-                scripts = self.directory / "auto-testing"
-                scripts.mkdir()
-                filename = f"rustfs_{suite}_test.sh" if suite in ("heal", "performance") else f"rustfs-{suite}-test.sh"
-                script = scripts / filename
-                script.write_text(
-                    '#!/bin/bash\nset -euo pipefail\nlog=""\n'
-                    'while [ "$#" -gt 0 ]; do\n'
-                    '  if [ "$1" = "--log-file" ]; then log="$2"; shift; fi\n'
-                    '  shift\n'
-                    'done\n'
-                    '[ "$log" = "$LOG_FILE" ] || exit 31\n'
-                    'printf "CURRENT SUITE LOG\\n" > "$log"\n'
-                    'scratch=$(mktemp -d "$TMPDIR/fixture.XXXXXX")\n'
-                    'printf "CURRENT SCRATCH\\n" > "$scratch/trace.log"\n'
-                    'if [ -n "${RUSTFS_RESULT_DIR:-}" ]; then\n'
-                    '  mkdir -p "$RUSTFS_RESULT_DIR"\n'
-                    '  printf "CURRENT RESULTS\\n" > "$RUSTFS_RESULT_DIR/summary.md"\n'
-                    'fi\n'
-                )
-                script.chmod(0o755)
-                name = FunctionalWorkflowTests.DIRECT_TESTS.get(suite) or (
-                    "Run benchmark (GET/PUT/MIXED)" if suite == "performance" else "Run heal test (write -> outage -> heal -> verify)"
-                )
-                result = self.run_step(name)
-                self.assertEqual(result.returncode, 0, result.stderr)
-                self.assertEqual((self.artifacts / "suite.log").read_text(), "CURRENT SUITE LOG\n")
-                self.assertEqual(len(list(Path(self.env["TMPDIR"]).glob("fixture.*/trace.log"))), 1)
-                self.assertEqual(list(self.artifacts.glob("fixture.*")), [])
-                if suite == "performance":
-                    self.assertEqual((self.artifacts / "results/summary.md").read_text(), "CURRENT RESULTS\n")
-
-    def test_upload_allowlist_preserves_diagnostics_without_scratch(self):
-        extra = {
-            "kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"],
-            "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"],
-            "performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv",
-                            "results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"],
-        }
-        for suite in self.SUITES:
-            with self.subTest(suite=suite):
-                self.prepare(suite)
-                self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
-                expected = {self.artifacts / name for name in ["report.md", "suite.log", *extra[suite]]}
-                for path in expected:
-                    path.parent.mkdir(parents=True, exist_ok=True)
-                    path.write_text("PARTIAL FAILURE DIAGNOSTIC")
-                for directory in (self.artifacts, Path(self.env["TMPDIR"]), self.artifacts / "results"):
-                    directory.mkdir(exist_ok=True)
-                    (directory / "init.json").write_text('{"root_token":"FAKE-SECRET-CANARY"}')
-                self.assertEqual(self.uploaded_files(), expected)
-                self.assertTrue(all("FAKE-SECRET-CANARY" not in path.read_text() for path in self.uploaded_files()))
-
-    def test_kms_failure_after_vault_init_keeps_only_failure_evidence(self):
-        self.prepare("kms")
-        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
-        scripts = self.directory / "auto-testing"
-        scripts.mkdir()
-        # Vault file writes and EXIT cleanup from auto-testing@06cd3c097350:23-24,57,479-487.
-        # The Docker boundary returns synthetic credentials; setup fails before vault_stop.
-        (scripts / "rustfs-kms-test.sh").write_text(r'''#!/bin/bash
-set -Eeuo pipefail
-TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-test.XXXXXX")"
-trap 'rm -rf "${TEST_TMP}"' EXIT
-VAULT_DATA_DIR="${RUSTFS_VAULT_DATA_DIR:-${TMPDIR:-/tmp}/rustfs-vault-data}"
-VAULT_CONTAINER="rustfs-vault"
-heal_run() { "$@"; }
-while [ "$#" -gt 0 ]; do
-  if [ "$1" = "--log-file" ]; then LOG_FILE="$2"; shift; fi
-  shift
-done
-mkdir -p "${VAULT_DATA_DIR}"
-tmp_init="${TEST_TMP}/vault-init.json"
-tmp_err="${TEST_TMP}/vault-init.stderr"
-heal_run docker exec -e "VAULT_ADDR=http://127.0.0.1:8200" "${VAULT_CONTAINER}" \
-  vault operator init -key-shares=1 -key-threshold=1 -format=json \
-  > "${tmp_init}" 2>"${tmp_err}"
-cat "${tmp_init}" | heal_run tee "${VAULT_DATA_DIR}/init.json" >/dev/null
-printf '%s\n' 'vault initialized; root token acquired' 'fixture setup failed after init' > "${LOG_FILE}"
-exit 42
-''')
-        docker = self.directory / "bin/docker"
-        docker.write_text("""#!/bin/sh
-[ "$1" = exec ] || exit 99
-printf '%s\\n' '{"root_token":"FAKE-ROOT-CANARY","unseal_keys_b64":["FAKE-UNSEAL-CANARY"]}'
-""")
-        docker.chmod(0o755)
-        result = self.run_step("Run KMS suite")
-        self.assertEqual(result.returncode, 42, result.stderr)
-        vault_init = Path(self.env["TMPDIR"]) / "rustfs-vault-data/init.json"
-        self.assertEqual(json.loads(vault_init.read_text()), {"root_token": "FAKE-ROOT-CANARY", "unseal_keys_b64": ["FAKE-UNSEAL-CANARY"]})
-        self.assertNotIn(self.artifacts, vault_init.parents)
-        self.assertEqual(list(Path(self.env["TMPDIR"]).glob("rustfs-test.*")), [])
-        self.assertNotEqual(self.run_step("Generate report").returncode, 0)
-        self.assertEqual(self.uploaded_files(), {self.artifacts / name for name in ("suite.log", "cases.md", "report.md")})
-        self.assertIn("fixture setup failed after init", (self.artifacts / "suite.log").read_text())
-        for path in self.uploaded_files():
-            self.assertNotIn("FAKE-ROOT-CANARY", path.read_text())
-            self.assertNotIn("FAKE-UNSEAL-CANARY", path.read_text())
-
-    def test_heal_accumulates_actual_staged_steps_without_overwriting_failures(self):
-        self.prepare("heal")
-        self.assertEqual(self.run_step("Initialize functional evidence").returncode, 0)
-        script = self.directory / "auto-testing/rustfs_heal_test.sh"
-        script.parent.mkdir()
-        # Result printf and full-run condition from auto-testing@6120aa0a76de:143,1163-1168.
-        script.write_text(r'''#!/bin/bash
-set -euo pipefail
-SELECTED_STEPS=()
-PREFLIGHT=0
-while [ "$#" -gt 0 ]; do
-  case "$1" in
-    --steps) IFS=',' read -ra SELECTED_STEPS <<< "$2"; shift ;;
-    --log-file) LOG_FILE="$2"; shift ;;
-    --preflight) PREFLIGHT=1 ;;
-  esac
-  shift
-done
-if [ "$PREFLIGHT" -eq 1 ]; then
-  printf '\n' >> "$INVOKED_STEPS"
-  exit 0
-fi
-printf '%s\n' "${SELECTED_STEPS[*]}" >> "$INVOKED_STEPS"
-emit_step_result() {
-  local n="$1" desc="$2" status="$3"
-  printf '[HEAL-STEP] %s %s %s\n' "${n}" "${desc}" "${status}"
-}
-{
-  for step in "${SELECTED_STEPS[@]}"; do
-    emit_step_result "$step" "fixture step $step" PASS
-  done
-  want_all=1
-  for s in 1 2 3 4 5 6 7; do
-    [[ " ${SELECTED_STEPS[*]} " == *" ${s} "* ]] || want_all=0
-  done
-  if [ "${want_all}" -eq 1 ]; then
-    printf '[HEAL-RESULT] PASS all steps passed\n'
-  fi
-} >> "$LOG_FILE"
-''')
-        script.chmod(0o755)
-        self.env["INVOKED_STEPS"] = str(self.directory / "invoked-steps")
-        for name in ("Install RustFS package & start cluster", "Preflight checks", "Run heal test (write -> outage -> heal -> verify)"):
-            result = self.run_step(name)
-            self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertEqual(Path(self.env["INVOKED_STEPS"]).read_text().splitlines(), ["1 2", "", "3 4 5 6 7"])
-        log = Path(self.env["LOG_FILE"]).read_text()
-        self.assertNotIn("[HEAL-RESULT]", log)
-        self.assertEqual(log.count("[HEAL-STEP]"), 7)
-        report = self.run_step("Generate report")
-        self.assertEqual(report.returncode, 0, report.stderr)
-        failed_logs = ["\n".join(line for line in log.splitlines() if not line.startswith(f"[HEAL-STEP] {step} ")) + "\n"
-                       for step in range(1, 8)]
-        failed_logs += [
-            log.replace("[HEAL-STEP] 3", "[HEAL-STEP] 3 original failure FAIL\n[HEAL-STEP] 3"),
-            log + "[HEAL-STEP] 3 later step failure FAIL\n",
-            log + "[HEAL-RESULT] FAIL earlier failure\n[HEAL-RESULT] PASS later success\n",
-            log.replace("[HEAL-STEP] 4 fixture step 4 PASS", "[HEAL-STEP] 4 fixture step 4 SKIP"),
-        ]
-        for failed_log in failed_logs:
-            with self.subTest(log=failed_log):
-                Path(self.env["LOG_FILE"]).write_text(failed_log)
-                report = self.run_step("Generate report")
-                self.assertNotEqual(report.returncode, 0, report.stderr)
-                contents = Path(self.env["REPORT_FILE"]).read_text()
-                self.assertIn("Test Step Outcome: failure", contents)
-                self.assertNotIn("| PASS |", contents)
-                if "original failure" in failed_log:
-                    self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text())
-                if "later step failure" in failed_log:
-                    self.assertIn("| 3 | later step failure | FAIL |", (self.artifacts / "steps.md").read_text())
-
-    def test_performance_results_version_and_report_are_bound_to_the_run(self):
-        self.prepare("performance")
-        initialized = self.run_step("Initialize functional evidence")
-        self.assertEqual(initialized.returncode, 0, initialized.stderr)
-        self.assertEqual(self.env["RUSTFS_RESULT_DIR"], str(self.artifacts / "results"))
-        self.assertEqual(self.env["VERSION_FILE"], str(self.artifacts / "version.txt"))
-        version = self.run_step("Collect RustFS version info")
-        self.assertEqual(version.returncode, 0, version.stderr)
-        self.assertIn("fixture-version", Path(self.env["VERSION_FILE"]).read_text())
-        old_summary = self.directory / "old-results/summary.md"
-        old_summary.parent.mkdir()
-        old_summary.write_text("OLD RUN EVIDENCE")
-        upload = "Upload report to dashboard (reports/YYYY-MM-DD.md)"
-        self.assertNotEqual(self.run_step(upload).returncode, 0)
-        self.assertFalse(Path(self.env["REPORT_FILE"]).exists())
-        results = Path(self.env["RUSTFS_RESULT_DIR"])
-        results.mkdir()
-        (results / "summary.md").write_text("CURRENT PERFORMANCE RESULTS\n")
-        report = self.run_step(upload)
-        self.assertEqual(report.returncode, 0, report.stderr)
-        contents = Path(self.env["REPORT_FILE"]).read_text()
-        for value in ("actions/runs/314159", "**Attempt**: 2", "**Workflow Commit**: " + self.context["github.sha"],
-                      "CURRENT PERFORMANCE RESULTS", "fixture-version"):
-            self.assertIn(value, contents)
-        self.assertNotIn("OLD RUN EVIDENCE", contents)
+                self.assertEqual(markers, ["cleanup"] if suite == "replication" else ["cleanup", "dispatch"])


 if __name__ == "__main__":
Author	SHA1	Message	Date
Zhengchao An	be3c25c36e	Merge branch 'test/distributed-e2e-hardening' into cursor/e2e-pool-tmpfs-fallback-f29d	2026-09-05 23:56:21 +08:00
overtrue	d8580ec970	test(e2e): prove operations overlap data movement	2026-09-05 22:50:00 +08:00
Zhengchao An	034efa8dac	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 22:21:31 +08:00
Cursor Agent	ff88b1d160	ci(e2e): use tmpfs for distributed pool isolation sm-standard-4 is an ARC pod without usable loop devices, so mount -o loop fails with ENOENT before any pool filesystem is attached. Sized tmpfs still gives each pool a distinct st_dev and independent 1G statfs capacity. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-05 12:58:22 +00:00
Zhengchao An	1277c11f04	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 11:57:33 +08:00
RustFS	bdb069c0fd	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 09:29:28 +08:00
RustFS	b65ab747d3	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 09:17:48 +08:00
loverustfs	8c48f184b5	test(e2e): add distributed 4x4 validation	2026-09-05 02:51:03 +08:00