Merge branch 'test/distributed-e2e-hardening' into cursor/e2e-pool-tmpfs-fallback-f29d

test(e2e): prove operations overlap data movement
Merge branch 'main' into test/distributed-e2e-hardening
2026-09-05 19:55:37 +00:00 · 2026-09-05 23:56:21 +08:00 · 2026-09-05 22:50:00 +08:00 · 2026-09-05 22:21:31 +08:00 · 2026-09-05 12:58:22 +00:00 · 2026-09-05 11:57:33 +08:00
118 changed files with 5833 additions and 6961 deletions
@@ -0,0 +1,2 @@
+sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
+sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
@@ -41,7 +41,6 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
@@ -183,6 +183,13 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'

+# 4-node 4-drive distributed Actions suite: each case starts four rustfs
+# processes and up to sixteen data directories. Serialize across nextest's
+# process boundary so several 4x4 clusters never overlap.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^distributed::/)'
+test-group = 'e2e-cluster-nightly'
+
 # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
 # does not cross nextest process boundaries, so keep every Vault-backed test in
 # one group.
@@ -526,6 +533,27 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'

+# ---------------------------------------------------------------------------
+# e2e-distributed profile — 4-node 4-disk Actions suite
+# ---------------------------------------------------------------------------
+# Storage-sensitive PR / nightly / dispatch lane owned by
+# .github/workflows/e2e-distributed.yml.
+# Each case starts four rustfs processes (and for site replication, two
+# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
+# Serialized via e2e-cluster-nightly with no retries.
+[profile.e2e-distributed]
+default-filter = 'package(e2e_test) & test(/^distributed::/)'
+fail-fast = false
+# Decommission / rebalance cases poll for up to 180s with little stdout.
+slow-timeout = { period = "120s", terminate-after = 6 }
+
+[profile.e2e-distributed.junit]
+path = "junit.xml"
+
+[[profile.e2e-distributed.overrides]]
+filter = 'package(e2e_test)'
+test-group = 'e2e-cluster-nightly'
+
 # ---------------------------------------------------------------------------
 # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
 # ---------------------------------------------------------------------------
@@ -586,6 +614,10 @@ path = "junit.xml"
 #     cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
 #     excluded here because backlog#2213 promotes core heal rebuild coverage to
 #     this merge/main lane while retaining nightly coverage.
+#   * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
+#     replication, quota, observability, expand/decommission/rebalance, site
+#     replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
+#     .github/workflows/e2e-distributed.yml.
 #   * on_demand_migration::interop_test — the ODM-20 provider interoperability
 #     cases, which are meaningless without a source: they run in the dedicated
 #     [profile.e2e-odm-interop] lane below, where the workflow points them at a
@@ -607,6 +639,7 @@ default-filter = """
    package(e2e_test)
    & !test(/^protocols::/)
    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^distributed::/)
    & !test(/^replication_extension_test::/)
    & !test(/^replication_target_matrix_test::/)
    & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
@@ -94,17 +94,12 @@ runs:
      shell: bash
      run: ./scripts/check_embedded_secrets.sh

-    - name: Run script contract tests
-      shell: bash
-      run: make script-tests
-
    - name: Check test wiring
      shell: bash
      run: |
        python3 ./scripts/check_test_wiring.py --self-test
        python3 ./scripts/check_scheduled_validation_freshness.py --self-test
        python3 ./scripts/test_security_workflow.py
-        python3 ./scripts/test_nightly_candidate.py
        python3 ./scripts/check_test_wiring.py

    - name: Check no planning docs committed
@@ -4,6 +4,11 @@
  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  {
+    "workflow": ".github/workflows/e2e-distributed.yml",
+    "max_age_hours": 36,
+    "never_ran_grace_until": "2026-09-18T00:00:00Z"
+  },
  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
@@ -0,0 +1,206 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 4-node 4-disk distributed e2e lane.
+#
+# Each selected test starts a real localhost cluster via
+# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
+# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
+#
+# Isolated pool filesystems: expand/decommission/rebalance cases require
+# independent `statfs` capacity. `sm-standard-4` is an ARC pod
+# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no
+# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed:
+# No such file or directory"). The prepare step therefore mounts four 1 GiB
+# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`.
+
+name: e2e-distributed
+
+on:
+  pull_request:
+    paths:
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - ".config/nextest.toml"
+      - ".github/workflows/e2e-distributed.yml"
+      - "crates/audit/**"
+      - "crates/common/**"
+      - "crates/config/**"
+      - "crates/e2e_test/**"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/iam/**"
+      - "crates/lock/**"
+      - "crates/madmin/**"
+      - "crates/notify/**"
+      - "crates/replication/**"
+      - "crates/s3-client/**"
+      - "crates/s3-ops/**"
+      - "crates/s3-types/**"
+      - "crates/scanner/**"
+      - "crates/storage-api/**"
+      - "crates/utils/**"
+      - "rustfs/**"
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
+        required: false
+        default: ""
+  schedule:
+    # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
+    - cron: "53 5 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
+jobs:
+  distributed:
+    name: Distributed 4-node 4-disk e2e
+    runs-on: sm-standard-4
+    timeout-minutes: 180
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      NO_PROXY: 127.0.0.1,localhost
+      HTTP_PROXY: ""
+      HTTPS_PROXY: ""
+      # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
+      UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
+      UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
+      UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-e2e-distributed
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
+
+      - name: Prepare isolated filesystems for pool movement
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          mkdir -p "${mount_base}"
+          roots=()
+          for pool in 0 1 2 3; do
+            mountpoint="${mount_base}/pool-${pool}"
+            mkdir -p "${mountpoint}"
+            # sm-standard-4 is an ARC pod without usable loop devices, so
+            # `mount -o loop` fails with ENOENT. Sized tmpfs still reports a
+            # distinct st_dev and independent 1G statfs capacity.
+            sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"
+            sudo chmod 1777 "${mountpoint}"
+            roots+=("${mountpoint}")
+          done
+          printf -v joined_roots '%s:' "${roots[@]}"
+          echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}"
+
+      - name: Download pinned previous release
+        env:
+          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
+        run: |
+          set -euo pipefail
+          mkdir -p "$SOURCE_DIR"
+          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
+          curl --fail --location --retry 3 --output "$archive" \
+            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
+          unzip -q "$archive" -d "$SOURCE_DIR"
+          chmod +x "$SOURCE_DIR/rustfs"
+          test -x "$SOURCE_DIR/rustfs"
+          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
+
+      - name: Build rustfs binary
+        run: |
+          cargo build -p rustfs --bins
+          : > target/debug/rustfs.features
+
+      - name: Verify distributed e2e membership
+        env:
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+        run: |
+          cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
+
+      - name: Run distributed 4-node e2e suite
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
+          FILTER: ${{ inputs.filter }}
+        run: |
+          set -euo pipefail
+          if [ -n "${FILTER}" ]; then
+            cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
+          else
+            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+          fi
+
+      - name: Upload distributed e2e diagnostics
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-distributed-${{ github.run_number }}
+          path: |
+            target/nextest/e2e-distributed/junit.xml
+            ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+            ${{ runner.temp }}/rustfs-e2e-distributed-logs/
+          retention-days: 7
+          if-no-files-found: warn
+
+      - name: Unmount isolated pool filesystems
+        if: always()
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          for pool in 0 1 2 3; do
+            mountpoint="${mount_base}/pool-${pool}"
+            if mountpoint --quiet "${mountpoint}"; then
+              sudo umount "${mountpoint}"
+            fi
+          done
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [distributed]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -19,9 +19,7 @@ on:
    paths:
      - ".github/workflows/e2e-upgrade.yml"
      - "crates/e2e_test/src/common.rs"
-      - "crates/e2e_test/src/fake_s3_target/**"
      - "crates/e2e_test/src/lib.rs"
-      - "crates/e2e_test/src/replication_extension_test.rs"
      - "crates/e2e_test/src/upgrade_compatibility_test.rs"
      - "crates/ecstore/**"
      - "crates/filemeta/**"
@@ -46,9 +44,9 @@ concurrency:
 env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
-  UPGRADE_SOURCE_VERSION: 1.0.0-rc.5
-  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
-  UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
+  UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
+  UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
+  UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7

 jobs:
  upgrade:
@@ -57,31 +55,14 @@ jobs:
      fail-fast: false
      matrix:
        include:
-          # The two `_from_rc2_` tests keep their names: they assert
-          # release-independent object contracts and pass unchanged against the
-          # newer pinned source, so renaming them would only churn history and
-          # the CI required-check names. UPGRADE_SOURCE_VERSION above is the
-          # single source of truth for which release they actually run against.
-          - name: Direct upgrade from the previous release
+          - name: Direct upgrade from rc.2
            cache_key: e2e-direct-upgrade
            test: direct_upgrade_from_rc2_preserves_object_contracts
            artifact: direct-upgrade
-          - name: Mixed-version rolling upgrade from the previous release
+          - name: Mixed-version rolling upgrade from rc.2
            cache_key: e2e-mixed-version-upgrade
            test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
            artifact: mixed-version-upgrade
-          - name: Bucket configuration survives the upgrade
-            cache_key: e2e-bucket-config-upgrade
-            test: direct_upgrade_from_previous_release_preserves_bucket_configuration
-            artifact: bucket-config-upgrade
-          - name: Rollback reads current bucket metadata
-            cache_key: e2e-bucket-config-rollback
-            test: rollback_to_previous_release_reads_current_bucket_metadata
-            artifact: bucket-config-rollback
-          - name: ODM configuration recovery after rc.5 rollback
-            cache_key: e2e-odm-config-rollback
-            test: rc5_rollback_requires_restoring_odm_configuration
-            artifact: odm-config-rollback
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
@@ -166,9 +166,8 @@ jobs:
      # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
      # Skipped when the R2 secrets are not configured (artifact-only mode).
      - name: Upload DEB to Cloudflare R2
-        id: publish
+        if: env.R2_ACCESS_KEY_ID != ''
        env:
-          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
@@ -183,70 +182,28 @@ jobs:
            exit 0
          fi

+          if ! command -v aws >/dev/null 2>&1; then
+            sudo apt-get update && sudo apt-get install -y -qq awscli
+          fi
+
          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
          export AWS_DEFAULT_REGION="auto"

-          SOURCE_SHA="$(git rev-parse HEAD)"
-          if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
-            echo "Checkout SHA does not match the nightly build run" >&2
-            exit 1
-          fi
-          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
-          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
-          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
-
-          # Old AWS CLI models lack conditional PutObject support. Never fall
-          # back to an overwriting upload for a candidate.
-          AWS_CLI=aws
-          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
-            sudo apt-get update
-            sudo apt-get install -y -qq python3-venv
-            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
-            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
-            python3 -m venv "${AWS_CLI_DIR}"
-            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
-            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
-          fi
-          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
-          "${AWS_CLI}" --version
-          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
-            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
-          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
-          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
-            echo "Published candidate checksum does not match the built package" >&2
-            exit 1
-          fi
-
+          DEB_FILE="${{ steps.deb.outputs.deb_file }}"
          R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"

          echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
-          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
+          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors

          # Stable "latest" alias so tests can fetch the newest nightly
          # without knowing today's date.
          echo "📤 Uploading latest alias"
-          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
+          aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
            --endpoint-url "$R2_ENDPOINT" --only-show-errors

          echo "✅ R2 upload complete"

-          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
-          jq -n --arg source_sha "${SOURCE_SHA}" \
-            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
-            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
-            '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
-            > "${CANDIDATE_FILE}"
-          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
-
-      - name: Upload nightly candidate manifest
-        if: ${{ steps.publish.outputs.candidate_file != '' }}
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
-        with:
-          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ steps.publish.outputs.candidate_file }}
-          if-no-files-found: error
-
  # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
  #
  # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
@@ -14,8 +14,8 @@

 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication -> performance). Each suite attempts the next handoff even
-# when its tests fail.
+# replication, with performance on its own runner in parallel) and guarantees
+# the chain keeps moving even when individual suites fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,3 +59,16 @@ jobs:
          gh api --method POST repos/rustfs/rustfs/dispatches \
            -f event_type='rustfs-chain-upgrade' \
            -F 'client_payload[from_suite]=nightly-build'
+
+      - name: Dispatch performance suite (parallel, own runner)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=nightly-build'
@@ -59,21 +59,6 @@ jobs:
    # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -129,7 +114,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"

      - name: Preflight checks
        run: |
@@ -139,7 +124,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"

      - name: Run heal test (write -> outage -> heal -> verify)
        id: test
@@ -149,10 +134,13 @@ jobs:
            --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
            --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
            --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
-            --log-file "${LOG_FILE}"
+            --log-file /tmp/rustfs-heal-test.log

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-heal-test.log
+          REPORT_FILE: /tmp/rustfs-heal-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -161,9 +149,8 @@ jobs:
          else
            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
          fi
-          STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
-          CASE_RESULT=success
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
+          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
+          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
          import re
          import sys

@@ -175,7 +162,6 @@ jobs:

          steps = {}
          order = []
-          status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
          version = None
          version_node = None
          verdict = None
@@ -189,15 +175,14 @@ jobs:
                          n, desc, status = m.group(1), m.group(2), m.group(3)
                          if n not in steps:
                              order.append(n)
-                          if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
-                              steps[n] = (desc, status)
+                          steps[n] = (desc, status)  # later lines win (fail after pass)
                          continue
                      m = ver_re.match(line)
                      if m:
                          version, version_node = m.group(1), m.group(2)
                          continue
                      m = result_re.match(line)
-                      if m and verdict != 'FAIL':
+                      if m:
                          verdict, verdict_detail = m.group(1), m.group(2)
          except FileNotFoundError:
              pass
@@ -217,43 +202,30 @@ jobs:
                  out.write(f'| {n} | {desc} | {status} |\n')
              if not order:
                  out.write('| - | - | NOT RUN (no step result lines found) |\n')
-          complete = set(steps) == {str(n) for n in range(1, 8)}
-          sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
          PY
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
          {
            echo "# RustFS heal test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${STEPS_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
-            fi
+            cat "${STEPS_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-heal-report.md
          SUITE: heal
        run: |
          set -euo pipefail
@@ -263,32 +235,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'heal'
          SUITE_LABEL: 'Heal'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-heal-report.md'
+          LOG_FILE: '/tmp/rustfs-heal-test.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -316,16 +284,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -341,16 +307,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload test logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-heal-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
-          if-no-files-found: error
+            /tmp/rustfs-heal-test*.log
+            /tmp/rustfs-warp.*.log
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: ${{ always() && inputs.cleanup_after != 'false' }}
@@ -52,25 +52,6 @@ jobs:
    timeout-minutes: 420
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -127,6 +108,8 @@ jobs:

      - name: Run KMS suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-kms-test.sh
@@ -156,7 +139,10 @@ jobs:
          ./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-kms.log
+          REPORT_FILE: /tmp/rustfs-kms-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -168,43 +154,79 @@ jobs:
          else
            PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-kms-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS KMS test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-kms-report.md
          SUITE: kms
        run: |
          set -euo pipefail
@@ -214,32 +236,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'kms'
          SUITE_LABEL: 'KMS'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-kms-report.md'
+          LOG_FILE: '/tmp/rustfs-kms.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -267,16 +285,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -292,15 +308,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-kms-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-kms.log
+            /tmp/rustfs-kms-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -49,16 +49,17 @@ on:
        type: boolean
        default: true
  repository_dispatch:
-    # Chain handoff: dispatched when the replication suite finishes.
+    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
+    # pf-testing runner, in parallel with the shared-VM chain).
    types: [rustfs-chain-performance]

 permissions:
  contents: read

-# The default performance nodes overlap the other suites' remote VMs, even
-# though the runner differs. Hold the shared lock through cleanup as well.
+# Dedicated pf-testing runner/environment: own concurrency group so perf runs
+# never block (or are blocked by) the pool-expansion / heal tests.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-performance-test
  cancel-in-progress: false

 defaults:
@@ -75,6 +76,8 @@ env:
  # Package used by the nightly run (workflow_dispatch inputs are empty for
  # workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
  RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
+  # Fixed benchmark result directory so later steps can read summary.md
+  RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
  # Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
  PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}

@@ -86,22 +89,6 @@ jobs:
    # Skipped when nightly failed.
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -133,7 +120,7 @@ jobs:
        if: ${{ inputs.cleanup_before != 'false' }}
        run: |
          chmod +x auto-testing/rustfs_performance_test.sh
-          ./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
+          ./auto-testing/rustfs_performance_test.sh --step 1 -y

      - name: Install RustFS package & start cluster (4x4)
        run: |
@@ -143,7 +130,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"

      - name: Preflight checks
        run: |
@@ -153,7 +140,7 @@ jobs:
          else
            ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
          fi
-          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
+          ./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"

      - name: Run benchmark (GET/PUT/MIXED)
        id: benchmark
@@ -166,15 +153,17 @@ jobs:
            --step 5 -y \
            --warp-duration "${{ inputs.warp_duration || '5m' }}" \
            --warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
-            --log-file "${LOG_FILE}"
+            --log-file /tmp/rustfs-perf-test.log

      - name: Analyze results
        if: ${{ steps.benchmark.conclusion == 'success' }}
        run: |
-          ./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
+          ./auto-testing/rustfs_performance_test.sh --step 6 -y

      - name: Collect RustFS version info
        if: ${{ steps.benchmark.conclusion == 'success' }}
+        env:
+          VERSION_FILE: /tmp/rustfs-version.txt
        run: |
          set -euo pipefail
          read -r -a NODES <<< "${RUSTFS_NODES}"
@@ -194,6 +183,7 @@ jobs:
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
          RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
+          VERSION_FILE: /tmp/rustfs-version.txt
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -201,7 +191,7 @@ jobs:
            exit 0
          fi
          SUMMARY="${RESULT_DIR}/summary.md"
-          [ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
+          [ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="reports/${DATE}.md"
          {
@@ -209,8 +199,6 @@ jobs:
            echo ""
            echo "- **Date**: ${DATE}"
            echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
-            echo "- **Workflow Commit**: ${GITHUB_SHA}"
            echo "- **Trigger**: ${{ github.event_name }}"
            echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
            echo ""
@@ -220,8 +208,8 @@ jobs:
            echo '```text'
            cat "${VERSION_FILE}"
            echo '```'
-          } > "${REPORT_FILE}"
-          CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
+          } > /tmp/rustfs-perf-report.md
+          CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
            jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
@@ -240,10 +228,11 @@ jobs:
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'performance'
          SUITE_LABEL: 'Performance'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-perf-report.md'
+          LOG_FILE: '/tmp/rustfs-perf-test.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -271,16 +260,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -296,26 +283,20 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload test logs & results
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-perf-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
-          if-no-files-found: error
+            /tmp/rustfs-perf-test*.log
+            /tmp/rustfs-perf-results/**
+            /tmp/rustfs-version.txt
+          if-no-files-found: warn

      - name: Reset test environment (after)
        if: ${{ always() && inputs.cleanup_after != 'false' }}
        run: |
-          ./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
+          ./auto-testing/rustfs_performance_test.sh --step 7 -y

      - name: Notify on failure
        if: failure()
@@ -539,22 +539,17 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
@@ -34,7 +34,8 @@ on:
          - site
        default: all
  repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes.
+    # Chain handoff: dispatched when the security suite finishes. This is the
+    # last link of the functional chain.
    types: [rustfs-chain-replication]

 permissions:
@@ -64,25 +65,6 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -131,6 +113,8 @@ jobs:

      - name: Run replication suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-replication.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-replication-test.sh
@@ -153,7 +137,10 @@ jobs:
          ./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-replication.log
+          REPORT_FILE: /tmp/rustfs-replication-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -175,44 +162,80 @@ jobs:
              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
            fi
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-replication-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          try:
+              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+                  for raw in fh:
+                      line = ansi.sub('', raw).strip()
+                      m = start_re.match(line)
+                      if m:
+                          case_id, name = m.group(1), m.group(2)
+                          if case_id not in index:
+                              index[case_id] = len(rows)
+                              rows.append([case_id, name, 'RUNNING'])
+                          continue
+                      m = done_re.match(line)
+                      if m:
+                          status, case_id = m.group(1), m.group(2)
+                          if case_id in index:
+                              rows[index[case_id]][2] = status
+                          else:
+                              rows.append([case_id, case_id, status])
+                              index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+              rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+              counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+              out.write('## Case Summary\n\n')
+              out.write(f"- Total: {len(rows)}\\n")
+              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+              out.write('\\n')
+              out.write('| Case | Name | Status |\\n')
+              out.write('| --- | --- | --- |\\n')
+              for case_id, name, status in rows:
+                  out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS replication test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-replication-report.md
          SUITE: replication
        run: |
          set -euo pipefail
@@ -222,32 +245,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'replication'
          SUITE_LABEL: 'Replication (bucket + site)'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-replication-report.md'
+          LOG_FILE: '/tmp/rustfs-replication.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -275,16 +294,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -300,15 +317,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-replication-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-replication.log
+            /tmp/rustfs-replication-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -329,50 +345,13 @@ jobs:
            '
          done

-      - name: "Continue functional chain (next: Performance)"
+      - name: Chain complete
+        # Replication is the last link of the functional chain: nothing to
+        # dispatch after it. This step just records that the chain finished.
        if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
        run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
-              -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            trap 'rm -f "${BODY_FILE}"' EXIT
-            {
-              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          echo "Functional chain complete: replication (final suite) finished."
+          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"

      - name: Notify on failure
        if: failure()
@@ -40,25 +40,6 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -106,6 +87,8 @@ jobs:

      - name: Run S3 compatibility suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-s3-compat-test.sh
@@ -122,7 +105,10 @@ jobs:
          ./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-s3-compat.log
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -144,44 +130,83 @@ jobs:
              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
            fi
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          current = None
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  current = case_id
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+                  current = None
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS S3 compatibility test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-s3-compat-report.md
          SUITE: s3
        run: |
          set -euo pipefail
@@ -191,32 +216,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 's3'
          SUITE_LABEL: 'S3 compatibility'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
+          LOG_FILE: '/tmp/rustfs-s3-compat.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -244,16 +265,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -269,15 +288,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-s3-compat-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-s3-compat.log
+            /tmp/rustfs-s3-compat-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -77,14 +77,10 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      # Checkout the repository into its own subdirectory. Checking out at
-      # the workspace root would wipe the auto-testing clone above (that is
-      # exactly how run 33934141181 lost rustfs-security-test.sh).
      - name: Checkout repository (for the OIDC live gate script)
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
        with:
          persist-credentials: false
-          path: rustfs-repo

      - name: Initialize security evidence
        id: evidence
@@ -92,7 +88,7 @@ jobs:
          set -euo pipefail
          umask 077
          SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
+          mkdir -- "${SECURITY_ARTIFACTS_DIR}"
          printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"

      # auto-testing is private: clone it with the dedicated PF token (not
@@ -148,8 +144,8 @@ jobs:
        continue-on-error: true
        env:
          REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
-          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
-          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
+          TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
+          RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-security-test.sh
@@ -172,7 +168,7 @@ jobs:
          else
            ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
          fi
-          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
+          GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"

      - name: Generate report
        id: report
@@ -223,22 +219,17 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
@@ -305,10 +296,7 @@ jobs:
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
          name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
-          path: |
-            ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
-            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
-            ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
+          path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
          if-no-files-found: error
          retention-days: 3

@@ -49,25 +49,6 @@ jobs:
    timeout-minutes: 360
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -115,6 +96,8 @@ jobs:

      - name: Run storage engine suite
        id: test
+        env:
+          LOG_FILE: /tmp/rustfs-storage.log
        run: |
          set -euo pipefail
          chmod +x auto-testing/rustfs-storage-test.sh
@@ -137,7 +120,10 @@ jobs:
          ./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-storage.log
+          REPORT_FILE: /tmp/rustfs-storage-report.md
        run: |
          set -euo pipefail
          PACKAGE_URL='${{ inputs.package_url }}'
@@ -159,44 +145,83 @@ jobs:
              RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
            fi
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-storage-cases.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file = sys.argv[1], sys.argv[2]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
+
+          rows = []
+          index = {}
+          current = None
+          try:
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  current = case_id
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
+                  current = None
+          except FileNotFoundError:
+            rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+            counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
+          PY
          {
            echo "# RustFS storage engine test report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- Package: ${PACKAGE_SOURCE}"
            echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-storage-report.md
          SUITE: storage
        run: |
          set -euo pipefail
@@ -206,32 +231,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'storage'
          SUITE_LABEL: 'Storage engine'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-storage-report.md'
+          LOG_FILE: '/tmp/rustfs-storage.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -259,16 +280,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -284,15 +303,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-storage-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-          if-no-files-found: error
+            /tmp/rustfs-storage.log
+            /tmp/rustfs-storage-report.md
+          if-no-files-found: warn

      - name: Cleanup environment (after)
        if: always()
@@ -377,22 +377,17 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: Verify required tier evidence
        id: evidence_verify
@@ -82,25 +82,6 @@ jobs:
    timeout-minutes: 420
    if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
    steps:
-      - name: Checkout repository (for report parser)
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
-        with:
-          persist-credentials: false
-
-      - name: Initialize functional evidence
-        id: evidence
-        run: |
-          set -euo pipefail
-          umask 077
-          FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-          mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
-          {
-            printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-            printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
-          } >> "${GITHUB_ENV}"
-
      # auto-testing is private: clone it with the dedicated PF token (not
      # GITHUB_TOKEN) and retry transient GitHub/network failures.
      - name: Checkout auto-testing scripts (with retry)
@@ -161,6 +142,7 @@ jobs:
      - name: Run upgrade compatibility suite
        id: test
        env:
+          LOG_FILE: /tmp/rustfs-upgrade.log
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
        run: |
          set -euo pipefail
@@ -218,7 +200,10 @@ jobs:
          ./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"

      - name: Generate report
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
+        env:
+          LOG_FILE: /tmp/rustfs-upgrade.log
+          REPORT_FILE: /tmp/rustfs-upgrade-report.md
        run: |
          set -euo pipefail
          FROM_URL='${{ inputs.from_url }}'
@@ -239,47 +224,103 @@ jobs:
          else
            TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
          fi
-          CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
-          MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
-          CASE_RESULT=success
-          python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
-          RESULT=failure
-          if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
-            RESULT=success
-          fi
+          CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
+          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
+          import re
+          import sys
+
+          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
+          ansi = re.compile(r'\x1b\[[0-9;]*m')
+          start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
+          done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
+          topo_re = re.compile(
+              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
+
+          rows = []
+          index = {}
+          topo_rows = []
+          try:
+              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+                  for raw in fh:
+                      line = ansi.sub('', raw).strip()
+                      m = topo_re.match(line)
+                      if m:
+                          topo_rows.append(m.groups())
+                          continue
+                      m = start_re.match(line)
+                      if m:
+                          case_id, name = m.group(1), m.group(2)
+                          if case_id not in index:
+                              index[case_id] = len(rows)
+                              rows.append([case_id, name, 'RUNNING'])
+                          continue
+                      m = done_re.match(line)
+                      if m:
+                          status, case_id = m.group(1), m.group(2)
+                          if case_id in index:
+                              rows[index[case_id]][2] = status
+                          else:
+                              rows.append([case_id, case_id, status])
+                              index[case_id] = len(rows) - 1
+          except FileNotFoundError:
+              rows = []
+
+          counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
+          for _, _, status in rows:
+              counts[status] = counts.get(status, 0) + 1
+
+          with open(out_file, 'w', encoding='utf-8') as out:
+              out.write('## Case Summary\n\n')
+              out.write(f"- Total: {len(rows)}\\n")
+              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+              out.write('\\n')
+              out.write('| Case | Name | Status |\\n')
+              out.write('| --- | --- | --- |\\n')
+              for case_id, name, status in rows:
+                  out.write(f'| {case_id} | {name} | {status} |\\n')
+
+          # Upgrade matrix: one row per topology/backend with the versions
+          # captured on the nodes (rustfs --version) and the aggregated
+          # result. The dashboard renders this table directly.
+          with open(matrix_file, 'w', encoding='utf-8') as out:
+              out.write('## Upgrade Matrix\n\n')
+              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
+              out.write('| --- | --- | --- | --- | --- |\n')
+              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
+                  result = 'PASS' if nfail == '0' else 'FAIL'
+                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
+              if not topo_rows:
+                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
+          PY
          {
            echo "# RustFS upgrade compatibility report"
            echo ""
            echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${{ github.event_name }}"
            echo "- From: ${FROM_SOURCE}"
            echo "- To: ${TO_SOURCE}"
-            echo "- Test Step Outcome: ${RESULT}"
-            echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
+            echo "- Test Step Outcome: ${{ steps.test.outcome }}"
            echo ""
-            if [ "${RESULT}" = "success" ]; then
-              cat "${MATRIX_TABLE}"
-              echo ""
-              cat "${CASE_TABLE}"
-              echo ""
-              echo "## Log tail"
-              echo '```text'
-              tail -n 200 "${LOG_FILE}"
-              echo '```'
-            else
-              echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
-            fi
+            cat "${MATRIX_TABLE}" || true
+            echo ""
+            cat "${CASE_TABLE}" || true
+            echo ""
+            echo "## Log tail"
+            echo '```text'
+            tail -n 200 "${LOG_FILE}" || true
+            echo '```'
          } | tee "${REPORT_FILE}"
          cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          [ "${RESULT}" = "success" ]

      - name: Upload functional report to dashboard
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        continue-on-error: true
        env:
          GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
+          REPORT_FILE: /tmp/rustfs-upgrade-report.md
          SUITE: upgrade
        run: |
          set -euo pipefail
@@ -289,32 +330,28 @@ jobs:
          fi
          DATE="$(date -u +%Y-%m-%d)"
          REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
-          # Base64-encode the report into a temp file and feed it to jq via
-          # --rawfile: large reports (e.g. pool) exceed the OS argv limit and
-          # make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
-          B64_FILE="$(mktemp)"
-          python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
+          CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
          SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
          if [ -n "${SHA}" ]; then
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
-              '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
+              '{message:$msg, content:$content, sha:$sha}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          else
-            jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
-              '{message:$msg, content:($content|rtrimstr("\n"))}' \
+            jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
+              '{message:$msg, content:$content}' \
              | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
          fi
-          rm -f "${B64_FILE}"

      - name: File failure issue in rustfs/backlog
        if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
-          EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
          SUITE: 'upgrade'
          SUITE_LABEL: 'Upgrade compatibility'
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
+          LOG_FILE: '/tmp/rustfs-upgrade.log'
        run: |
          set -euo pipefail
          if [ -z "${GH_TOKEN:-}" ]; then
@@ -342,16 +379,14 @@ jobs:
            echo ""
            echo "- Suite: \`${SUITE}\`"
            echo "- Run: ${RUN_URL}"
-            echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
-            echo "- Workflow Commit: ${GITHUB_SHA}"
            echo "- Trigger: ${GITHUB_EVENT_NAME}"
            echo "- Date: $(date -u +%Y-%m-%d)"
            echo ""
            echo "## Report (errors and symptoms)"
            echo ""
-            if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
+            if [ -s "${REPORT_FILE}" ]; then
              redact < "${REPORT_FILE}"
-            elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
+            elif [ -s "${LOG_FILE:-}" ]; then
              echo "(report file missing; log tail below)"
              echo ""
              tail -n 200 "${LOG_FILE}" | redact
@@ -367,16 +402,14 @@ jobs:
          echo "filed backlog issue for suite ${SUITE}"

      - name: Upload report and logs
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: always()
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
        with:
-          name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
+          name: rustfs-upgrade-test-${{ github.run_id }}
          path: |
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
-            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
-          if-no-files-found: error
+            /tmp/rustfs-upgrade-report.md
+            /tmp/rustfs-upgrade.*/*
+          if-no-files-found: ignore
          retention-days: 3

      - name: Cleanup environment (after)
@@ -22,6 +22,7 @@ on:
      - "Continuous Integration"
      - "coverage"
      - "e2e-nightly"
+      - "e2e-distributed"
      - "e2e-s3tests"
      - "Fuzz"
      - "mint"
@@ -18,10 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  - Read paths: an object at or below `policy.inline_max_bytes` (16 MiB by default) is teed to the client and to the local store in a single source read; a larger object or a Range read streams through and a background pull stores the whole object. A HEAD miss is proxied to the source and stores nothing (`policy.head = local_only` disables it). Every source-backed response carries `x-rustfs-on-demand-migration: source`
  - Protections: a per-source circuit breaker, a per-key negative cache, singleflight per key, a concurrency limit and a bounded pull queue shared by the inline and background paths, an optional bandwidth limit, an anti-loop request marker, and the shared outbound-endpoint (SSRF) policy
  - Metrics under `rustfs_on_demand_migration_*` (`requests_total`, `pulled_bytes_total`, `pulled_objects_total`, `pull_failures_total`, `inflight_pulls`, `queue_depth`, `source_latency_seconds_*`, `breaker_state`), mirrored per node by the admin status route
-  - Listings: `ListObjects` v1 remains local with ordinary key markers. `ListObjectsV2` can merge source objects when `policy.list_through = true`; this is off by default
-  - Upgrade and rollback: finish upgrading every node before enabling ODM. An rc.5 node that writes bucket configuration drops the ODM fields from metadata; neither a later restart nor moving the service out of ECStore recovers them. Before rollback, disable ODM and securely retain the original full configuration and credentials. After every node returns to a compatible version, restore and validate that configuration. Redacted exports cannot replace the credential backup; source-only objects are unavailable through RustFS while ODM is disabled. See the upgrade and rollback section of `docs/operations/on-demand-migration.md`
-  - Optional Google dependencies: default and `full` server builds retain native GCS support. `cargo build -p rustfs --no-default-features --features ftps,webdav` excludes Google SDKs while preserving configuration decoding and redaction; native GCS ODM and tier operations require the `gcs` feature. Do not use that build with existing GCS-tiered data
-  - Limitations: PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
+  - Limitations: listings show only local objects (the source is not merged into `ListObjectsV2`); PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
 - **NATS JetStream Publish Path**: Opt-in at-least-once delivery for the NATS notify and audit targets. A NATS Core publish flushes to the connection without awaiting a broker acknowledgement, so an event can be lost across a broker restart or a reconnect after the send queue has already cleared it. A queued event now clears only after the JetStream `PublishAck`, so bucket notifications survive those interruptions. Off by default and byte-identical to the NATS Core path when disabled.
  - Three configuration keys per target: `JETSTREAM_ENABLE`, `JETSTREAM_STREAM_NAME`, and `JETSTREAM_ACK_TIMEOUT_SECS`, under the `RUSTFS_NOTIFY_NATS_` and `RUSTFS_AUDIT_NATS_` prefixes
  - Durable store-and-forward with a stable dedup id sent as the `Nats-Msg-Id` header, so a replay after a crash is collapsed by the server duplicate window
@@ -1679,7 +1679,7 @@ version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]

 [[package]]
@@ -1698,7 +1698,7 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]

 [[package]]
@@ -2110,7 +2110,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
 dependencies = [
- "crypto-common 0.1.6",
+ "crypto-common 0.1.7",
 "inout 0.1.4",
 ]

@@ -2580,7 +2580,7 @@ version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "rand_core 0.6.4",
 "subtle",
 "zeroize",
@@ -2605,11 +2605,11 @@ dependencies = [

 [[package]]
 name = "crypto-common"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "typenum",
 ]

@@ -3901,7 +3901,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
 "block-buffer 0.10.4",
 "const-oid 0.9.6",
- "crypto-common 0.1.6",
+ "crypto-common 0.1.7",
 "subtle",
 ]

@@ -4166,7 +4166,7 @@ dependencies = [
 "crypto-bigint 0.5.5",
 "digest 0.10.7",
 "ff 0.13.1",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "group 0.13.0",
 "hkdf 0.12.4",
 "pem-rfc7468 0.7.0",
@@ -4499,7 +4499,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
 dependencies = [
 "io-lifetimes 2.0.4",
 "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.52.0",
 ]

 [[package]]
@@ -4622,9 +4622,9 @@ dependencies = [

 [[package]]
 name = "generic-array"
-version = "0.14.9"
+version = "0.14.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
 "typenum",
 "version_check",
@@ -4637,7 +4637,7 @@ version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
 dependencies = [
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "rustversion",
 "typenum",
 ]
@@ -5319,9 +5319,9 @@ dependencies = [

 [[package]]
 name = "hotpath-macros"
-version = "0.25.1"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "846bde0d9600d98434e1aac376977d7718bfe3d2f5312a041b7c59a6a466c51a"
+checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5660,7 +5660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
 dependencies = [
 "block-padding 0.3.3",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 ]

 [[package]]
@@ -5693,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
 dependencies = [
 "io-lifetimes 3.0.1",
- "windows-sys 0.60.2",
+ "windows-sys 0.52.0",
 ]

 [[package]]
@@ -7115,7 +7115,7 @@ version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.21.7",
 "chrono",
 "getrandom 0.2.17",
 "http 1.5.0",
@@ -9493,8 +9493,6 @@ dependencies = [
 "atomic_enum",
 "aws-config",
 "aws-sdk-s3",
- "aws-smithy-runtime-api",
- "aws-smithy-types",
 "axum",
 "base64-simd",
 "bytes",
@@ -9503,13 +9501,11 @@ dependencies = [
 "clap",
 "const-str",
 "datafusion",
- "faster-hex",
 "flatbuffers",
 "flate2",
 "futures",
 "futures-lite",
 "futures-util",
- "google-cloud-auth",
 "hashbrown 0.17.1",
 "hex-simd",
 "hmac 0.13.0",
@@ -9528,7 +9524,6 @@ dependencies = [
 "metrics",
 "metrics-util",
 "mime_guess",
- "moka",
 "opentelemetry",
 "opentelemetry_sdk",
 "p256 0.14.0",
@@ -9621,7 +9616,6 @@ dependencies = [
 "urlencoding",
 "uuid",
 "x509-parser",
- "xxhash-rust",
 "zeroize",
 "zip",
 "zstd 0.14.0",
@@ -9798,6 +9792,7 @@ dependencies = [
 "path-absolutize",
 "pin-project-lite",
 "proptest",
+ "quick-xml",
 "rand 0.10.2",
 "ratelimit",
 "rcgen",
@@ -11405,7 +11400,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
 dependencies = [
 "base16ct 0.2.0",
 "der 0.7.10",
- "generic-array 0.14.9",
+ "generic-array 0.14.7",
 "pkcs8 0.10.2",
 "subtle",
 "zeroize",
@@ -12405,7 +12400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
 "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
 "once_cell",
 "rustix",
 "windows-sys 0.61.2",
@@ -13657,15 +13652,6 @@ dependencies = [
 "windows-targets 0.52.6",
 ]

-[[package]]
-name = "windows-sys"
-version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
-dependencies = [
- "windows-targets 0.52.6",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.60.2"
@@ -13838,7 +13824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
 dependencies = [
 "bitflags 2.13.1",
- "windows-sys 0.59.0",
+ "windows-sys 0.52.0",
 ]

 [[package]]
@@ -371,8 +371,7 @@ dav-server = "0.11.0"

 # Performance Analysis and Memory Profiling
 rustfs-mimalloc = { version = "0.5.3" }
-# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
-hotpath = { version = "=0.25.0", default-features = false }
+hotpath = { version = "0.25.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }

@@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |

@@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker.
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
 | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
+# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
+# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
+cargo nextest run --profile e2e-distributed -p e2e_test
 # Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
@@ -1700,6 +1700,69 @@ impl RustFSTestClusterEnvironment {
        Ok(())
    }

+    /// Append a new single-node erasure pool to a stopped multi-pool cluster.
+    ///
+    /// Used to simulate pool expansion on localhost: every pool already owns
+    /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
+    /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
+    /// allocated a fresh port and empty drive directories; callers must
+    /// [`Self::start`] afterwards so every process picks up the extended
+    /// volumes argument. Existing data directories are left untouched.
+    pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+        if self.nodes.iter().any(|node| node.process.is_some()) {
+            return Err("stop the cluster before appending a pool".into());
+        }
+        if self.topology.drives_per_node < 2 {
+            return Err(
+                "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
+                    .into(),
+            );
+        }
+
+        let mut pools = self.topology.normalized_pools();
+        for (pool_idx, nodes) in pools.iter().enumerate() {
+            if nodes.len() != 1 {
+                return Err(format!(
+                    "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
+                    nodes.len()
+                )
+                .into());
+            }
+        }
+
+        let new_idx = self.nodes.len();
+        let port = RustFSTestEnvironment::find_available_port().await?;
+        let address = format!("127.0.0.1:{port}");
+        let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
+            .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
+            .collect();
+        for dir in &data_dirs {
+            fs::create_dir_all(dir).await?;
+        }
+
+        self.nodes.push(ClusterNode {
+            url: format!("http://{address}"),
+            address,
+            data_dir: data_dirs[0].clone(),
+            data_dirs,
+            pool_idx: pools.len(),
+            process: None,
+        });
+        pools.push(vec![new_idx]);
+        self.topology.node_count = self.nodes.len();
+        self.topology.pools = pools;
+        self.node_extra_env.push(Vec::new());
+        self.node_capture_log_paths.push(None);
+        self.volume_proxy_addresses.push(None);
+
+        if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
+            self.extra_env
+                .push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
+        }
+
+        Ok(new_idx)
+    }
+
    /// Gracefully stop one cluster node and wait for its process to exit.
    ///
    /// This is intentionally separate from [`Self::stop_node`]: the latter is
@@ -0,0 +1,222 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
+    wait_for_ready, wait_until,
+};
+use crate::chaos::{census_object_version_on_disk, signed_admin_post};
+use crate::common::{build_test_s3_config, init_logging};
+use crate::fault_proxy::FaultMode;
+use aws_sdk_s3::Client;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{Barrier, mpsc};
+use tokio::time::timeout;
+
+#[tokio::test]
+async fn kill_and_restart_node_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("killnode");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x11u8; 128 * 1024];
+    put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
+
+    dist.cluster.stop_node(3)?;
+    retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn full_cluster_restart_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("pwr");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x44u8; 64 * 1024];
+    put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
+    let bucket = unique_bucket("baddrive");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
+
+    let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
+    let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(
+        baseline.is_complete(),
+        "replacement target did not hold a complete baseline shard: {baseline:?}"
+    );
+    assert!(
+        !baseline.expected_part_numbers.is_empty(),
+        "replacement witness must use physical part shards: {baseline:?}"
+    );
+
+    dist.cluster.stop_node(0)?;
+    let format_path = replaced_drive.join(".rustfs.sys/format.json");
+    let format = std::fs::read(&format_path)?;
+    let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
+    std::fs::rename(&replaced_drive, &retired_drive)?;
+    std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
+    std::fs::write(&format_path, format)?;
+    let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
+
+    dist.cluster.start_node(0).await?;
+    wait_for_ready(&dist.cluster).await?;
+    let heal_body =
+        r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+    let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
+    signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+    wait_until(
+        Duration::from_secs(90),
+        || async {
+            let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+            Ok(healed.matches_manifest(&baseline))
+        },
+        "fresh replacement contains the original complete shard manifest",
+    )
+    .await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("getkill");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
+
+    let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
+    let worker_count = 12;
+    let release = Arc::new(Barrier::new(worker_count + 1));
+    let (started_tx, mut started_rx) = mpsc::unbounded_channel();
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let client = live[idx % live.len()].clone();
+        let bucket = bucket.clone();
+        let body = body.clone();
+        let release = release.clone();
+        let started_tx = started_tx.clone();
+        handles.push(tokio::spawn(async move {
+            let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
+            if response.content_length() != Some(body.len() as i64) {
+                return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
+                    format!("worker {idx} received a wrong content length").into(),
+                );
+            }
+            started_tx.send(idx)?;
+            release.wait().await;
+            let actual = response.body.collect().await?.into_bytes();
+            if actual.as_ref() != body.as_slice() {
+                return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
+            }
+            Ok(())
+        }));
+    }
+    drop(started_tx);
+    for _ in 0..worker_count {
+        timeout(Duration::from_secs(30), started_rx.recv())
+            .await?
+            .ok_or("a streaming GET exited before reaching the kill barrier")?;
+    }
+
+    dist.cluster.stop_node(3)?;
+    release.wait().await;
+    for handle in handles {
+        handle.await??;
+    }
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
+    let proxied_url = format!("http://{}", proxy.local_addr());
+    let proxied_client = Client::from_conf(build_test_s3_config(
+        &proxied_url,
+        &dist.cluster.access_key,
+        &dist.cluster.secret_key,
+        None,
+        "distributed-network-chaos",
+    ));
+
+    let result: TestResult = async {
+        let bucket = unique_bucket("netfault");
+        dist.create_bucket(&bucket).await?;
+        let baseline = payload_for("network/baseline.bin", 1024 * 1024);
+        put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
+        assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
+
+        proxy.set_mode(FaultMode::Blackhole);
+        assert_eq!(proxy.mode(), FaultMode::Blackhole);
+        if let Ok(Ok(_)) = timeout(
+            Duration::from_secs(5),
+            proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
+        )
+        .await
+        {
+            return Err("blackholed node endpoint unexpectedly completed a GET".into());
+        }
+
+        let during = payload_for("network/during.bin", 1024 * 1024);
+        timeout(Duration::from_secs(30), async {
+            put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
+            assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
+            assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
+        })
+        .await??;
+
+        proxy.set_mode(FaultMode::Pass);
+        retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
+        Ok(())
+    }
+    .await;
+
+    proxy.set_mode(FaultMode::Pass);
+    proxy.shutdown().await;
+    result
+}
@@ -0,0 +1,98 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use crate::common::init_logging;
+use std::collections::BTreeSet;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("conc");
+    dist.create_bucket(&bucket).await?;
+    let clients = Arc::new(dist.clients()?);
+    let worker_count = 24;
+    let rounds = 4;
+    let barrier = Arc::new(Barrier::new(worker_count));
+
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let writer = &clients[idx % clients.len()];
+            let reader = &clients[(idx + 1) % clients.len()];
+            let copier = &clients[(idx + 2) % clients.len()];
+            let mut retained = Vec::with_capacity(rounds);
+            for round in 0..rounds {
+                let key = format!("source/worker-{idx:02}-round-{round}.bin");
+                let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
+                let body = payload_for(&key, 64 * 1024);
+                put_object(writer, &bucket, &key, body.clone()).await?;
+
+                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                if head.content_length() != Some(body.len() as i64) {
+                    return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
+                }
+                assert_object_bytes(reader, &bucket, &key, &body).await?;
+
+                copier
+                    .copy_object()
+                    .bucket(&bucket)
+                    .key(&copy_key)
+                    .copy_source(format!("{bucket}/{key}"))
+                    .send()
+                    .await?;
+                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+
+                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                let missing = reader
+                    .head_object()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .send()
+                    .await
+                    .expect_err("deleted source key must not remain visible");
+                if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
+                    return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
+                }
+                retained.push((copy_key, body));
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
+        }));
+    }
+
+    let mut inventory = Vec::new();
+    for handle in handles {
+        inventory.extend(handle.await??);
+    }
+
+    let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
+    for (node_idx, client) in clients.iter().enumerate() {
+        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+        assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
+        for (key, body) in &inventory {
+            assert_object_bytes(client, &bucket, key, body)
+                .await
+                .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
+        }
+    }
+    Ok(())
+}
@@ -0,0 +1,74 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
+    unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("concdecom");
+    dist.create_bucket(&bucket).await?;
+    let baseline_client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(17));
+    let mut handles = Vec::new();
+    for idx in 0..16 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("live/{idx:02}.bin");
+            let body = payload_for(&key, 8 * 1024);
+            retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    barrier.wait().await;
+
+    let mut live_objects = Vec::new();
+    for handle in handles {
+        live_objects.push(handle.await??);
+    }
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let checker = dist.client(2)?;
+    assert_inventory(&checker, &bucket, &inventory).await?;
+    for (key, body) in live_objects {
+        retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,156 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
+    sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("integrity");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    enable_versioning(&client, &bucket).await?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
+
+    let versioned_key = "history/versioned.bin";
+    let version_one = b"historical bytes before data movement".to_vec();
+    let version_two = b"current bytes before data movement".to_vec();
+    let version_one_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_one.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("historical PUT omitted version ID")?
+        .to_string();
+    let version_two_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_two.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("current PUT omitted version ID")?
+        .to_string();
+
+    let multipart_key = "multipart/moved.bin";
+    let first_part = vec![0x31; 5 * 1024 * 1024];
+    let second_part = vec![0x72; 1024 * 1024];
+    let upload = client
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
+    let uploaded_one = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(first_part.clone()))
+        .send()
+        .await?;
+    let uploaded_two = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(2)
+        .body(ByteStream::from(second_part.clone()))
+        .send()
+        .await?;
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after_client = dist.client(2)?;
+    assert_inventory(&after_client, &bucket, &inventory).await?;
+    for (key, expected_hash) in before {
+        let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
+        let body = got.body.collect().await?.into_bytes();
+        assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
+    }
+    for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
+        let got = after_client
+            .get_object()
+            .bucket(&bucket)
+            .key(versioned_key)
+            .version_id(version_id)
+            .send()
+            .await?
+            .body
+            .collect()
+            .await?
+            .into_bytes();
+        assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
+    }
+    let mut expected_multipart = first_part;
+    expected_multipart.extend_from_slice(&second_part);
+    let got_multipart = after_client
+        .get_object()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?
+        .body
+        .collect()
+        .await?
+        .into_bytes();
+    assert_eq!(
+        sha256_hex(got_multipart.as_ref()),
+        sha256_hex(&expected_multipart),
+        "multipart checksum changed after decommission"
+    );
+    Ok(())
+}
@@ -0,0 +1,81 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
+    put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
+    wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("expand");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
+    assert_inventory(&client, &bucket, &inventory).await?;
+
+    for expected_nodes in 2..=4 {
+        let new_node = dist.append_pool_and_restart().await?;
+        assert_eq!(new_node + 1, expected_nodes);
+        assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
+    }
+    assert_eq!(dist.cluster.nodes.len(), 4);
+
+    // Prove that the expanded pool map is durable, and clear any recovery
+    // latch raised while the newly-added pool replicas converged.
+    dist.restart_current_binary_gracefully().await?;
+
+    let after_expand = dist.client(0)?;
+    assert_inventory(&after_expand, &bucket, &inventory).await?;
+    let peer = dist.client(3)?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("decom");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    let pools_before = list_pools_json(&dist.cluster).await?;
+    let pool_count = pools_before
+        .as_array()
+        .map(Vec::len)
+        .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
+        .ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
+    assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after = dist.client(2)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,149 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("extra");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+
+    let key = "multipart.bin";
+    let part1 = vec![0x41u8; 5 * 1024 * 1024];
+    let part2 = vec![0x42u8; 5 * 1024 * 1024];
+    let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
+
+    let uploaded1 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(part1.clone()))
+        .send()
+        .await?;
+    let uploaded2 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(2)
+        .body(ByteStream::from(part2.clone()))
+        .send()
+        .await?;
+
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded1.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded2.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let mut expected = part1;
+    expected.extend_from_slice(&part2);
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
+    }
+
+    put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
+    put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
+    let mut seen = Vec::new();
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let listed = dist
+            .client(node_idx)?
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("list/")
+            .send()
+            .await?;
+        let keys: Vec<String> = listed
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        seen.push(keys);
+    }
+    for keys in &seen[1..] {
+        assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
+    }
+
+    let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
+    assert_eq!(got, b"a");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("listed");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node_idx)?;
+        let name = bucket.clone();
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = client.clone();
+                let name = name.clone();
+                async move {
+                    let listed = client.list_buckets().send().await?;
+                    Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
+                }
+            },
+            &format!("node {node_idx} lists {bucket}"),
+        )
+        .await?;
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = dist.client(node_idx).expect("client");
+                let name = bucket.clone();
+                async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
+            },
+            &format!("node {node_idx} reads seed.bin"),
+        )
+        .await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,35 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node 4-drive distributed e2e coverage.
+//!
+//! Selected by `[profile.e2e-distributed]` and run from
+//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
+//! each case starts four real `rustfs` processes.
+
+mod chaos_test;
+mod concurrency_stability_test;
+mod concurrent_data_movement_test;
+mod data_integrity_movement_test;
+mod expand_decommission_rebalance_test;
+mod extra_test;
+mod harness;
+mod object_lock_test;
+mod observability_test;
+mod replication_quota_test;
+mod s3_basic_test;
+mod s3_during_data_movement_test;
+mod site_replication_test;
+mod upgrade_test;
+mod versioning_test;
@@ -0,0 +1,219 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
+use crate::common::init_logging;
+use crate::object_lock::common::{
+    delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::operation::delete_object::DeleteObjectError;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{
+    DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
+    ObjectLockRule,
+};
+use chrono::{Duration as ChronoDuration, Utc};
+
+fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
+    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+    if code == Some("AccessDenied") {
+        Ok(())
+    } else {
+        Err(format!("{context}: expected AccessDenied, got {error:?}").into())
+    }
+}
+
+async fn expect_versioned_delete_denied(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    bypass: bool,
+    context: &str,
+) -> TestResult {
+    match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
+        Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
+        Err(error) => delete_denied(error.as_ref(), context),
+    }
+}
+
+#[tokio::test]
+async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let client = dist.client(0)?;
+    let peer = dist.client(2)?;
+    let bucket = unique_bucket("objlock");
+
+    client
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    let retain_until = Utc::now() + ChronoDuration::days(1);
+
+    let compliance_key = "compliance.bin";
+    let compliance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        compliance_key,
+        b"locked-compliance",
+        ObjectLockRetentionMode::Compliance,
+        retain_until,
+    )
+    .await?;
+
+    // Unversioned DELETE is allowed: it only creates a delete marker. WORM
+    // applies to a specific version id.
+    let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
+    assert_eq!(
+        marker.delete_marker(),
+        Some(true),
+        "unversioned DELETE on a locked object must create a delete marker"
+    );
+
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
+        .await?;
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
+
+    let governance_key = "governance.bin";
+    let governance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        governance_key,
+        b"locked-governance",
+        ObjectLockRetentionMode::Governance,
+        retain_until,
+    )
+    .await?;
+
+    expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
+        .await?;
+    delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
+    let deleted_governance = peer
+        .head_object()
+        .bucket(&bucket)
+        .key(governance_key)
+        .version_id(&governance_version)
+        .send()
+        .await
+        .expect_err("GOVERNANCE bypass must remove the retained version");
+    assert_eq!(
+        deleted_governance.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
+    );
+
+    let hold_key = "legal-hold.bin";
+    let hold_version =
+        put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let bucket = unique_bucket("default-lock");
+
+    writer
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+    put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
+
+    let key = "default-governance.bin";
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"default retention payload"))
+        .send()
+        .await?;
+    let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
+
+    let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
+    let default_retention = config
+        .object_lock_configuration()
+        .and_then(|configuration| configuration.rule())
+        .and_then(|rule| rule.default_retention())
+        .ok_or("GetObjectLockConfiguration omitted default retention")?;
+    assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    assert_eq!(default_retention.days(), Some(1));
+
+    let retention = reader
+        .get_object_retention()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
+    assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    let retain_until = retention
+        .retain_until_date()
+        .ok_or("default retention omitted retain-until date")?;
+    assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
+
+    let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
+    assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
+    expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
+        .await?;
+
+    let plain_bucket = unique_bucket("no-lock");
+    dist.create_bucket(&plain_bucket).await?;
+    let configuration = ObjectLockConfiguration::builder()
+        .object_lock_enabled(ObjectLockEnabled::Enabled)
+        .rule(
+            ObjectLockRule::builder()
+                .default_retention(
+                    DefaultRetention::builder()
+                        .mode(ObjectLockRetentionMode::Governance)
+                        .days(1)
+                        .build(),
+                )
+                .build(),
+        )
+        .build();
+    let error = writer
+        .put_object_lock_configuration()
+        .bucket(&plain_bucket)
+        .object_lock_configuration(configuration)
+        .send()
+        .await
+        .expect_err("an unversioned bucket must reject Object Lock enablement");
+    let service_error = error
+        .as_service_error()
+        .ok_or("non-lock bucket rejection was not an S3 service error")?;
+    assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
+    assert_eq!(
+        service_error.message(),
+        Some("Object Lock configuration cannot be enabled on existing buckets"),
+        "unexpected error: {error:?}"
+    );
+
+    Ok(())
+}
@@ -0,0 +1,236 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
+use crate::common::{admin_request, init_logging, local_http_client};
+use aws_sdk_s3::operation::RequestId;
+use aws_sdk_s3::primitives::ByteStream;
+use bytes::Bytes;
+use http::Method;
+use http_body_util::{BodyExt, Empty};
+use hyper::body::Incoming;
+use hyper::service::service_fn;
+use hyper::{Request, Response};
+use hyper_util::rt::TokioIo;
+use local_ip_address::local_ip;
+use rustfs_madmin::metrics::RealtimeMetrics;
+use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
+use serde_json::Value;
+use std::convert::Infallible;
+use std::time::Duration;
+use tokio::net::TcpListener;
+use tokio::sync::mpsc;
+use tokio::task::JoinHandle;
+use tokio::time::{Instant, timeout};
+
+async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
+    let listener = TcpListener::bind("0.0.0.0:0").await?;
+    let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
+    let (tx, rx) = mpsc::unbounded_channel();
+    let handle = tokio::spawn(async move {
+        loop {
+            let Ok((stream, _)) = listener.accept().await else {
+                return;
+            };
+            let tx = tx.clone();
+            tokio::spawn(async move {
+                let service = service_fn(move |request: Request<Incoming>| {
+                    let tx = tx.clone();
+                    async move {
+                        let method = request.method().clone();
+                        if let Ok(body) = request.into_body().collect().await
+                            && method == Method::POST
+                            && let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
+                        {
+                            if let Some(records) = payload["Records"].as_array() {
+                                for entry in records {
+                                    let _ = tx.send(entry.clone());
+                                }
+                            } else {
+                                let _ = tx.send(payload);
+                            }
+                        }
+                        Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
+                    }
+                });
+                let _ = hyper::server::conn::http1::Builder::new()
+                    .serve_connection(TokioIo::new(stream), service)
+                    .await;
+            });
+        }
+    });
+    Ok((endpoint, rx, handle))
+}
+
+async fn wait_for_audit_entry(
+    rx: &mut mpsc::UnboundedReceiver<Value>,
+    bucket: &str,
+    key: &str,
+    request_id: &str,
+) -> TestResult<Value> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    let mut seen = Vec::new();
+    loop {
+        let remaining = deadline.saturating_duration_since(Instant::now());
+        if remaining.is_zero() {
+            return Err(format!(
+                "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                seen.len()
+            )
+            .into());
+        }
+        let entry = match timeout(remaining, rx.recv()).await {
+            Ok(Some(entry)) => entry,
+            Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
+            Err(_) => {
+                return Err(format!(
+                    "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                    seen.len()
+                )
+                .into());
+            }
+        };
+        if entry["api"]["name"].as_str() == Some("s3:PutObject")
+            && entry["api"]["bucket"].as_str() == Some(bucket)
+            && entry["api"]["object"].as_str() == Some(key)
+            && entry["requestID"].as_str() == Some(request_id)
+        {
+            return Ok(entry);
+        }
+        if seen.len() < 8 {
+            seen.push(format!(
+                "api={:?} bucket={:?} object={:?} requestID={:?}",
+                entry["api"]["name"].as_str(),
+                entry["api"]["bucket"].as_str(),
+                entry["api"]["object"].as_str(),
+                entry["requestID"].as_str()
+            ));
+        }
+    }
+}
+
+#[tokio::test]
+async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
+    init_logging();
+    let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
+    let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
+    let audit_env = [
+        ("RUSTFS_AUDIT_ENABLE", "true"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
+        (ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
+    ];
+    let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
+        tokio::fs::create_dir_all(&queue_dir).await?;
+        dist.cluster
+            .set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
+    }
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+
+    let http = local_http_client();
+    for node in &dist.cluster.nodes {
+        for probe in ["ready", "live"] {
+            let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
+            assert!(
+                response.status().is_success(),
+                "node {} {probe} probe failed: {}",
+                node.address,
+                response.status()
+            );
+        }
+    }
+
+    let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    let info: Value = serde_json::from_str(&info_body)?;
+    let servers = info["info"]["servers"]
+        .as_array()
+        .ok_or_else(|| format!("admin info omitted servers: {info}"))?;
+    assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
+
+    let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    let storage: Value = serde_json::from_str(&storage_body)?;
+    let disks = storage["info"]["disks"]
+        .as_array()
+        .ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
+    assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
+    assert!(
+        disks.iter().all(|disk| {
+            disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
+                && disk["runtimeState"]
+                    .as_str()
+                    .is_some_and(|state| state.eq_ignore_ascii_case("online"))
+        }),
+        "storageinfo reported a drive that was not healthy and online: {storage}"
+    );
+
+    for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
+        let (status, metrics_body) = admin_request(
+            &node.url,
+            Method::GET,
+            "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
+            None,
+            &dist.cluster.access_key,
+            &dist.cluster.secret_key,
+        )
+        .await?;
+        assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
+        let sample: RealtimeMetrics = serde_json::from_str(
+            metrics_body
+                .lines()
+                .next()
+                .ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
+        )?;
+        assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
+        assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
+        assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
+    }
+
+    let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
+    let targets: Value = serde_json::from_str(&targets_body)?;
+    let configured = targets["audit_endpoints"]
+        .as_array()
+        .ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
+        .iter()
+        .any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
+    assert!(configured, "configured audit webhook was missing: {targets}");
+
+    let bucket = unique_bucket("audit");
+    dist.create_bucket(&bucket).await?;
+    let key = "correlated/audit-object.bin";
+    let put = dist
+        .client(2)?
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"distributed audit payload"))
+        .send()
+        .await?;
+    let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
+    let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
+    assert_eq!(
+        audit["api"]["status_code"].as_i64(),
+        Some(200),
+        "audit entry did not report success: {audit}"
+    );
+    assert!(
+        !audit.to_string().contains(&dist.cluster.secret_key),
+        "audit entry leaked the root secret key"
+    );
+
+    collector.abort();
+    Ok(())
+}
@@ -0,0 +1,191 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
+    set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
+};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use http::Method;
+use std::time::Duration;
+
+async fn wait_for_replication_status(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    expected: &[&str],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            let head = client.head_object().bucket(bucket).key(key).send().await?;
+            Ok(head
+                .replication_status()
+                .is_some_and(|status| expected.contains(&status.as_str())))
+        },
+        &format!("replication status for {bucket}/{key} in {expected:?}"),
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
+    init_logging();
+    let (source, mut target) = DistCluster::start_replication_pair().await?;
+    let source_bucket = unique_bucket("replsrc");
+    let target_bucket = unique_bucket("repldst");
+    source.create_bucket(&source_bucket).await?;
+    target.create_bucket(&target_bucket).await?;
+
+    let source_client = source.client(0)?;
+    let target_client = target.client(0)?;
+    enable_versioning(&source_client, &source_bucket).await?;
+    enable_versioning(&target_client, &target_bucket).await?;
+
+    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    let key = "replicated/metadata-and-tags.bin";
+    let body = b"distributed-bucket-replication".to_vec();
+    source_client
+        .put_object()
+        .bucket(&source_bucket)
+        .key(key)
+        .metadata("origin", "four-node-source")
+        .tagging("suite=distributed&shape=metadata")
+        .body(ByteStream::from(body.clone()))
+        .send()
+        .await?;
+    wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
+
+    let peer_read = target.client(3)?;
+    wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
+    let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
+    assert_eq!(
+        replica_head
+            .metadata()
+            .and_then(|metadata| metadata.get("origin"))
+            .map(String::as_str),
+        Some("four-node-source")
+    );
+    assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
+    let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
+    let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
+    assert_eq!(tags.get("suite"), Some(&"distributed"));
+    assert_eq!(tags.get("shape"), Some(&"metadata"));
+
+    target.cluster.stop();
+    let outage_key = "replicated/queued-during-target-outage.bin";
+    let outage_body = b"retry-after-target-restart".to_vec();
+    put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
+    wait_for_replication_status(
+        &source_client,
+        &source_bucket,
+        outage_key,
+        &["PENDING", "FAILED"],
+        Duration::from_secs(30),
+    )
+    .await?;
+
+    target.cluster.start().await?;
+    wait_for_ready(&target.cluster).await?;
+    wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
+    let bucket = unique_bucket("quota");
+    dist.create_bucket(&bucket).await?;
+    set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
+
+    let client = dist.client(1)?;
+    retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, body) = super::harness::cluster_admin(
+                &dist.cluster,
+                Method::GET,
+                &format!("/rustfs/admin/v3/quota-stats/{bucket}"),
+                None,
+            )
+            .await?;
+            if !status.is_success() {
+                return Ok(false);
+            }
+            let stats: serde_json::Value =
+                serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
+            let usage = stats
+                .get("current_usage")
+                .and_then(serde_json::Value::as_u64)
+                .ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
+            Ok(usage >= 1024)
+        },
+        "quota stats observe small object",
+    )
+    .await?;
+
+    let oversized_key = "too-big.bin";
+    let error = client
+        .put_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .body(vec![0u8; 16 * 1024].into())
+        .send()
+        .await
+        .expect_err("hard quota must reject the oversized PUT");
+    let service_error = error
+        .as_service_error()
+        .ok_or("quota rejection was not an S3 service error")?;
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(400),
+        "quota rejection must be HTTP 400: {error:?}"
+    );
+    assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
+    assert!(
+        service_error
+            .message()
+            .is_some_and(|message| message.starts_with("Bucket quota exceeded")),
+        "PUT must fail specifically at quota admission: {error:?}"
+    );
+
+    let missing = client
+        .head_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .send()
+        .await
+        .expect_err("an object rejected by quota must not become visible");
+    assert_eq!(
+        missing.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "quota-rejected object returned an unexpected HEAD result: {missing:?}"
+    );
+
+    let listed = client.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
+        "quota-rejected key leaked into ListObjectsV2"
+    );
+    Ok(())
+}
@@ -0,0 +1,258 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
+use crate::common::{init_logging, local_http_client};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::presigning::PresigningConfig;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3basic");
+    dist.create_bucket(&bucket).await?;
+
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let key = "dir/object.bin";
+    let body = vec![0xA5u8; 256 * 1024];
+    put_object(&writer, &bucket, key, body.clone()).await?;
+
+    let head = reader.head_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(head.content_length(), Some(body.len() as i64));
+    assert_object_bytes(&reader, &bucket, key, &body).await?;
+
+    let ranged = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .range("bytes=0-15")
+        .send()
+        .await?;
+    let ranged_body = ranged.body.collect().await?.into_bytes();
+    assert_eq!(ranged_body.as_ref(), &body[..16]);
+
+    let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
+    let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+    assert_eq!(keys, vec![key]);
+
+    let copy_key = "dir/object-copy.bin";
+    reader
+        .copy_object()
+        .bucket(&bucket)
+        .key(copy_key)
+        .copy_source(format!("{bucket}/{key}"))
+        .metadata_directive(MetadataDirective::Copy)
+        .send()
+        .await?;
+    assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
+
+    let moved_key = "dir/object-moved.bin";
+    writer
+        .copy_object()
+        .bucket(&bucket)
+        .key(moved_key)
+        .copy_source(format!("{bucket}/{copy_key}"))
+        .send()
+        .await?;
+    writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
+    match writer.head_object().bucket(&bucket).key(copy_key).send().await {
+        Ok(_) => return Err("copied source still present after rename delete".into()),
+        Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
+        Err(error) => return Err(error.into()),
+    }
+    assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
+
+    let presigned = writer
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
+        .await?;
+    let response = local_http_client().get(presigned.uri().to_string()).send().await?;
+    assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
+    let presigned_body = response.bytes().await?;
+    assert_eq!(presigned_body.as_ref(), body.as_slice());
+
+    let empty_key = "empty";
+    put_object(&writer, &bucket, empty_key, Vec::new()).await?;
+    let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
+    assert!(empty.is_empty());
+
+    let deleted = writer
+        .delete_objects()
+        .bucket(&bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key(key).build()?)
+                .objects(ObjectIdentifier::builder().key(moved_key).build()?)
+                .objects(ObjectIdentifier::builder().key(empty_key).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
+    assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
+
+    let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3matrix");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+
+    let special_key = "unicode/测试 space+percent%25.txt";
+    let special_body = b"metadata and tagging survive distributed routing".to_vec();
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .metadata("test-meta", "distributed")
+        .tagging("purpose=compatibility&scope=four-by-four")
+        .body(ByteStream::from(special_body.clone()))
+        .send()
+        .await?;
+    let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
+
+    let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
+    assert_eq!(
+        head.metadata()
+            .and_then(|metadata| metadata.get("test-meta"))
+            .map(String::as_str),
+        Some("distributed")
+    );
+    assert_eq!(head.e_tag(), Some(etag.as_str()));
+    let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
+    let actual_tags: std::collections::BTreeMap<_, _> = tags
+        .tag_set()
+        .iter()
+        .map(|tag| (tag.key().to_string(), tag.value().to_string()))
+        .collect();
+    assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
+    assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
+
+    let conditional = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .if_match(&etag)
+        .send()
+        .await?;
+    assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
+    let invalid_range = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .range("bytes=999999-1000000")
+        .send()
+        .await
+        .expect_err("an unsatisfiable range must fail");
+    assert_eq!(
+        invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("InvalidRange"),
+        "unexpected invalid-range error: {invalid_range:?}"
+    );
+
+    let upload_key = "multipart/aborted.bin";
+    let upload = writer
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
+    writer
+        .upload_part()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
+        .send()
+        .await?;
+    let pending = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
+    writer
+        .abort_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    let after_abort = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
+    let aborted_head = reader
+        .head_object()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await
+        .expect_err("aborted multipart upload must not create an object");
+    assert_eq!(
+        aborted_head.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
+    );
+
+    for index in 0..113 {
+        let key = format!("page/{index:04}.txt");
+        put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
+    }
+    let mut token = None;
+    let mut paged_keys = Vec::new();
+    loop {
+        let page = reader
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("page/")
+            .max_keys(37)
+            .set_continuation_token(token.take())
+            .send()
+            .await?;
+        paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
+        if page.is_truncated() != Some(true) {
+            break;
+        }
+        token = Some(
+            page.next_continuation_token()
+                .ok_or("truncated ListObjectsV2 page omitted next continuation token")?
+                .to_string(),
+        );
+    }
+    assert_eq!(paged_keys.len(), 113);
+    let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
+    assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
+    Ok(())
+}
@@ -0,0 +1,94 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
+    retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
+    wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("s3move");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    let live = dist.client(2)?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-decommission.bin")),
+        "list during decommission missed the newly written key"
+    );
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+    assert_inventory(&live, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let status = rebalance_status_json(&dist.cluster).await?;
+    if !rebalance_running_with_progress(&status, &rebalance_id)? {
+        return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
+    }
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,128 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
+    wait_until,
+};
+use crate::common::{init_logging, signed_request};
+use http::{Method, StatusCode};
+use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
+use std::time::Duration;
+
+async fn site_replication_add(
+    cluster: &crate::common::RustFSTestClusterEnvironment,
+    sites: &[PeerSite],
+) -> TestResult<ReplicateAddStatus> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &cluster.access_key,
+        &cluster.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let info = site_replication_info(cluster).await?;
+            Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
+        },
+        "site replication enabled with two synchronized sites",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+    let bucket = unique_bucket("siterepl");
+    site_a.create_bucket(&bucket).await?;
+    site_b.create_bucket(&bucket).await?;
+
+    let client_a = site_a.client(0)?;
+    let client_b = site_b.client(0)?;
+    enable_versioning(&client_a, &bucket).await?;
+    enable_versioning(&client_b, &bucket).await?;
+
+    let sites = vec![
+        PeerSite {
+            name: "site-a".to_string(),
+            endpoint: site_a.cluster.nodes[0].url.clone(),
+            access_key: site_a.cluster.access_key.clone(),
+            secret_key: site_a.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+        PeerSite {
+            name: "site-b".to_string(),
+            endpoint: site_b.cluster.nodes[0].url.clone(),
+            access_key: site_b.cluster.access_key.clone(),
+            secret_key: site_b.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+    ];
+    let add_status = site_replication_add(&site_a.cluster, &sites).await?;
+    assert!(
+        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
+        "site replication add reported failure: {add_status:?}"
+    );
+    wait_for_site_replication_enabled(&site_a.cluster).await?;
+    wait_for_site_replication_enabled(&site_b.cluster).await?;
+
+    let info_a = site_replication_info(&site_a.cluster).await?;
+    let remote = info_a
+        .sites
+        .iter()
+        .find(|site| site.name == "site-b")
+        .ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
+    assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
+    let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
+    assert!(
+        deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
+        "site peers must have two distinct non-empty deployment IDs: {info_a:?}"
+    );
+    assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
+    assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
+
+    let key = "site-object.bin";
+    let body = b"four-node-site-replication".to_vec();
+    put_object(&client_a, &bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
+
+    let peer_b = site_b.client(3)?;
+    wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
+
+    let reverse_key = "reverse/site-object.bin";
+    let reverse_body = b"site-b-to-site-a".to_vec();
+    put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
+    wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
+    Ok(())
+}
@@ -0,0 +1,345 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node upgrade coverage for historical objects and IAM AK/SK.
+//!
+//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
+//! mixed-version listing). This module pins the distributed contract the
+//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
+//! written on the previous release still read back, and IAM user credentials
+//! created before the upgrade still authenticate.
+//!
+//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
+//! release. The `e2e-distributed` workflow downloads that binary; a local run
+//! without it fails closed rather than skipping.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
+    unique_bucket, wait_until,
+};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use uuid::Uuid;
+
+const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const IAM_SECRET: &str = "UpgradeTestSecretKey1";
+const WRONG_SECRET: &str = "WrongSecretKey000000";
+const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct UpgradeSeed {
+    history_bucket: String,
+    history_key: &'static str,
+    history_body: Vec<u8>,
+    versioned_bucket: String,
+    versioned_key: &'static str,
+    version1: String,
+    version1_body: Vec<u8>,
+    version2: String,
+    version2_body: Vec<u8>,
+    iam_bucket: String,
+    iam_key: &'static str,
+    iam_body: Vec<u8>,
+    iam_user: String,
+    iam_secret: &'static str,
+}
+
+fn source_binary() -> TestResult<PathBuf> {
+    let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
+        format!(
+            "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
+        )
+    })?;
+    if !path.is_file() {
+        return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
+    }
+    Ok(path)
+}
+
+fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
+        cluster
+            .cluster
+            .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+fn iam_rw_policy(bucket: &str) -> String {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}"),
+                format!("arn:aws:s3:::{bucket}/*")
+            ]
+        }]
+    })
+    .to_string()
+}
+
+async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
+    let url = &dist.cluster.nodes[0].url;
+    let access = &dist.cluster.access_key;
+    let admin_secret = &dist.cluster.secret_key;
+    admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
+    admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
+    admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
+    Ok(())
+}
+
+async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                put_object(&client, &bucket, &key, body).await?;
+                Ok(true)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let expected = expected.to_vec();
+            async move {
+                let got = get_object_bytes(&client, &bucket, &key).await?;
+                Ok(got == expected)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
+    let history_bucket = unique_bucket("upg-hist");
+    let versioned_bucket = unique_bucket("upg-ver");
+    let iam_bucket = unique_bucket("upg-iam");
+    dist.create_bucket(&history_bucket).await?;
+    dist.create_bucket(&versioned_bucket).await?;
+    dist.create_bucket(&iam_bucket).await?;
+
+    let root = dist.client(0)?;
+    enable_versioning(&root, &versioned_bucket).await?;
+
+    let history_key = "plain-history.bin";
+    let history_body = b"written by the previous 4-node release".to_vec();
+    put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
+
+    let versioned_key = "versioned-history.txt";
+    let version1_body = b"version-one-before-upgrade".to_vec();
+    let version1 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("first versioned PUT omitted version ID")?
+        .to_string();
+    let version2_body = b"version-two-before-upgrade".to_vec();
+    let version2 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("second versioned PUT omitted version ID")?
+        .to_string();
+
+    let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
+
+    let iam_key = "iam-history.bin";
+    let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
+    let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
+    wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
+
+    Ok(UpgradeSeed {
+        history_bucket,
+        history_key,
+        history_body,
+        versioned_bucket,
+        versioned_key,
+        version1,
+        version1_body,
+        version2,
+        version2_body,
+        iam_bucket,
+        iam_key,
+        iam_body,
+        iam_user,
+        iam_secret: IAM_SECRET,
+    })
+}
+
+async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
+    let root_a = dist.client(0)?;
+    let root_b = dist.client(3)?;
+    wait_for_bytes(
+        &root_b,
+        &seed.history_bucket,
+        seed.history_key,
+        &seed.history_body,
+        &format!("{context}: root GET historical object"),
+    )
+    .await?;
+    assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
+
+    let v1 = root_b
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version1)
+        .send()
+        .await?;
+    let v1_body = v1.body.collect().await?.into_bytes();
+    if v1_body.as_ref() != seed.version1_body.as_slice() {
+        return Err(format!("{context}: version 1 bytes changed after upgrade").into());
+    }
+    let v2 = root_a
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version2)
+        .send()
+        .await?;
+    let v2_body = v2.body.collect().await?.into_bytes();
+    if v2_body.as_ref() != seed.version2_body.as_slice() {
+        return Err(format!("{context}: version 2 bytes changed after upgrade").into());
+    }
+
+    let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    if !users.contains(&seed.iam_user) {
+        return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
+    }
+
+    let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
+    let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
+    wait_for_bytes(
+        &iam_on_upgraded,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 0"),
+    )
+    .await?;
+    wait_for_bytes(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 3"),
+    )
+    .await?;
+
+    let post_key = format!("after-upgrade-{context}.txt");
+    let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
+    wait_for_put(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        &post_key,
+        post_body.clone(),
+        &format!("{context}: IAM PUT after upgrade"),
+    )
+    .await?;
+    assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
+
+    let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
+    match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
+        Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
+        Err(error) => {
+            let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+            let rejected = code == Some("SignatureDoesNotMatch")
+                || code == Some("InvalidAccessKeyId")
+                || code == Some("AccessDenied")
+                || code == Some("InvalidArgument")
+                || error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
+            if !rejected {
+                return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
+            }
+        }
+    }
+
+    let post_root_key = format!("root-after-{context}.bin");
+    let post_root_body = format!("{context}: root write after upgrade").into_bytes();
+    put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
+    assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "direct-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+    dist.restart_with_current_binary().await?;
+    assert_history_and_iam(&dist, &seed, "direct").await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+
+    dist.replace_node_with_current_binary(0).await?;
+    assert_history_and_iam(&dist, &seed, "one-current-node").await?;
+
+    for node_idx in [1, 2] {
+        dist.replace_node_with_current_binary(node_idx).await?;
+    }
+    assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
+
+    dist.replace_node_with_current_binary(3).await?;
+    assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
+    Ok(())
+}
@@ -0,0 +1,188 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+
+#[tokio::test]
+async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("version");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "versioned.txt";
+    let v1_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v1".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v1 PUT omitted version ID")?
+        .to_string();
+    let v2_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v2".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v2 PUT omitted version ID")?
+        .to_string();
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_versions: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
+    assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
+        "v1 was not the historical version: {versions:?}"
+    );
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
+        "v2 was not the latest version: {versions:?}"
+    );
+
+    let latest = get_object_bytes(&reader, &bucket, key).await?;
+    assert_eq!(latest, b"v2");
+
+    let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let older_body = older.body.collect().await?.into_bytes();
+    assert_eq!(older_body.as_ref(), b"v1");
+
+    let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(deleted.delete_marker(), Some(true));
+    let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
+    let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_markers: Vec<_> = after_delete
+        .delete_markers()
+        .iter()
+        .filter(|marker| marker.key() == Some(key))
+        .collect();
+    assert_eq!(
+        matching_markers.len(),
+        1,
+        "delete marker missing or duplicated after current-version delete: {after_delete:?}"
+    );
+    assert!(
+        matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
+        "DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
+    );
+
+    let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
+    match latest_after_delete {
+        Ok(_) => return Err("current version should be a delete marker".into()),
+        Err(error)
+            if error
+                .as_service_error()
+                .and_then(ProvideErrorMetadata::code)
+                .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
+        Err(error) => return Err(error.into()),
+    }
+
+    let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let restored_body = restored.body.collect().await?.into_bytes();
+    assert_eq!(restored_body.as_ref(), b"v1");
+
+    writer
+        .delete_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(marker_id)
+        .send()
+        .await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("suspend");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "suspended.txt";
+    let original = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"enabled-history".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("enabled PUT omitted version ID")?
+        .to_string();
+    writer
+        .put_bucket_versioning()
+        .bucket(&bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Suspended)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
+    put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
+    let null_version_count = matching
+        .iter()
+        .filter(|version| {
+            matches!(
+                version.version_id(),
+                None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
+            )
+        })
+        .count();
+    assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
+
+    let historical = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(&original)
+        .send()
+        .await?;
+    assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
+    Ok(())
+}
@@ -378,6 +378,11 @@ mod bucket_stats_regression_test;
 #[cfg(test)]
 mod distributed_startup_regression_test;

+// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
+// quota, observability, expand/decommission/rebalance, site replication, chaos).
+#[cfg(test)]
+mod distributed;
+
 // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
 #[cfg(test)]
 mod tier_transition_regression_test;
@@ -20,10 +20,9 @@
 //! journal (`count_requests`) carries the assertion in every one of them.

 use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
-use crate::fake_s3_target::{FaultAction, Operation};
+use crate::fake_s3_target::Operation;
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use bytes::Bytes;
-use futures::{StreamExt, TryStreamExt};
 use std::time::Duration;

 type TestResult = Result<(), BoxError>;
@@ -146,38 +145,14 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
    .await?;

    let body = payload(128 * 1024);
-    let blocker = "queue/blocker.bin";
-    env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
-    // The one-chunk range completes immediately; its full background pull
-    // occupies the only slot while the remaining requests fill the queue.
-    env.source.inject_for_key(
-        Operation::GetObject,
-        blocker,
-        FaultAction::SlowSendBody {
-            chunk_bytes: 1024,
-            delay: Duration::from_millis(100),
-        },
-        2,
-    );
-    let response = env
-        .raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
-        .await?;
-    assert_eq!(response.status, 206);
-    assert_eq!(response.body, body.slice(0..1024));
-    env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
-
    let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
    let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
    env.seed_source(SOURCE_BUCKET, &seeds);

-    // Bound source connections below the fixture's limit while still
-    // submitting all 100 requests to the eight-slot background queue.
-    let responses: Vec<RawResponse> = futures::stream::iter(
+    let responses: Vec<RawResponse> = futures::future::try_join_all(
        keys.iter()
            .map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
    )
-    .buffered(16)
-    .try_collect()
    .await?;
    for (key, response) in keys.iter().zip(&responses) {
        assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
@@ -193,15 +168,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
        .wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
        .await?;
    assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
-    let queue_full = usize::try_from(queue_full)?;
-    assert!(queue_full <= REQUESTS);
-    env.wait_for_status_counter(
-        bucket,
-        "/counters/pulled_objects_total/background",
-        u64::try_from(REQUESTS + 1 - queue_full)?,
-        SETTLE,
-    )
-    .await?;

    let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
    assert!(
@@ -209,6 +175,9 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
        "every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
    );
    let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
-    assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
+    assert!(
+        dropped > 0,
+        "the overflowed keys are the ones with no backfill GET, but every key got one"
+    );
    Ok(())
 }
@@ -265,13 +265,16 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
    let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
    assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");

-    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
-    assert_ne!(tampered, token, "the test must change the token version");
-    let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
-    let rejected = env.raw_list_objects_v2(bucket, &query).await?;
-    let error_body = String::from_utf8_lossy(&rejected.body);
-    assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
-    assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
+    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
+    let rejected = env
+        .raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
+        .await?;
+    assert_eq!(
+        rejected.status,
+        400,
+        "a bumped token version is a client error: {}",
+        String::from_utf8_lossy(&rejected.body)
+    );
    Ok(())
 }

@@ -12,34 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::common::{
-    RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
-};
-use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
-use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
-use crate::replication_extension_test::{
-    LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
-};
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::error::ProvideErrorMetadata;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{
-    BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
-    ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
-    ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
-    ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
+    BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
 };
-use http::{Method, StatusCode};
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 use tokio::task::JoinSet;
 use tokio::time::{Instant, sleep};

 type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
-type BoxError = Box<dyn std::error::Error + Send + Sync>;

 const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
-const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -53,32 +40,6 @@ const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
 // comfortably covers that window plus CI scheduling jitter.
 const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);

-// Bucket-configuration upgrade/rollback scenarios (rustfs#7172, #7183, #7089).
-const CONFIG_PLAIN_BUCKET: &str = "upgrade-config-plain";
-const CONFIG_ENCRYPTED_BUCKET: &str = "upgrade-config-encrypted";
-const CONFIG_REPLICATED_BUCKET: &str = "upgrade-config-replicated";
-const CONFIG_LOCKED_BUCKET: &str = "upgrade-config-locked";
-const CONFIG_REPLICA_BUCKET: &str = "upgrade-config-replica";
-const ROLLBACK_BUCKET: &str = "rollback-config-data";
-const ROLLBACK_REPLICA_BUCKET: &str = "rollback-config-replica";
-const BUCKET_QUOTA_BYTES: u64 = 64 * 1024 * 1024;
-const LIFECYCLE_RULE_ID: &str = "upgrade-expire-logs";
-const LIFECYCLE_PREFIX: &str = "logs/";
-const LIFECYCLE_DAYS: i32 = 30;
-const BUCKET_TAG_KEY: &str = "owner";
-const BUCKET_TAG_VALUE: &str = "upgrade-compatibility";
-const OBJECT_LOCK_DAYS: i32 = 1;
-// `set-bucket-quota` answers 503 until the scanner has made the bucket's usage
-// authoritative; the quota test uses the same 30s budget.
-const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30);
-// Quota admission fails closed while a freshly started server has neither
-// authoritative usage nor a persisted degraded baseline for the bucket
-// (rustfs#5716), so a write to a quota-enabled bucket is retryable-503 for that
-// window. It is a restart property, not an upgrade property — the same window
-// opens on the very first start — so the write assertions ride it out instead
-// of treating it as an upgrade failure.
-const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90);
-
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
    let path = std::env::var_os(SOURCE_BINARY_ENV)
        .map(PathBuf::from)
@@ -279,93 +240,6 @@ async fn exercise_mixed_cluster(
    Ok(())
 }

-/// Pins the published old writer's limitation and the supported recovery
-/// procedure. This is not a promise that mixed-version ODM is supported.
-/// Replace the loss assertion when ODM gains independent persistence;
-/// preserving configuration across rc.5 writes is then an improvement.
-#[tokio::test]
-#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
-async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
-    init_logging();
-    let previous_binary = source_binary()?;
-    let version = tokio::process::Command::new(&previous_binary)
-        .arg("--version")
-        .output()
-        .await?;
-    assert!(version.status.success(), "previous binary must report its version");
-    assert!(
-        String::from_utf8(version.stdout)?.contains(RC5_COMMIT),
-        "this compatibility scenario requires the published rc.5 writer"
-    );
-    let mut env = OdmTestEnv::start().await?;
-    let bucket = "odm-rc5-rollback";
-    let source_bucket = "odm-rc5-source";
-    env.source.create_bucket_with_mode(source_bucket, BucketMode::Unversioned);
-    env.seed_source(
-        source_bucket,
-        &[SeedObject::new(
-            "source-only",
-            bytes::Bytes::from_static(b"source read after recovery"),
-        )],
-    );
-    env.rustfs.create_test_bucket(bucket).await?;
-    let saved_config = env.fake_source_spec(source_bucket);
-    assert_eq!(env.configure_source(bucket, &saved_config).await?.status, 200);
-    let before = env.get_config(bucket).await?;
-    assert_eq!(before.status, 200);
-    let expected_config = before
-        .json()?
-        .get("config")
-        .cloned()
-        .ok_or("configuration response omitted config")?;
-    env.client
-        .put_object()
-        .bucket(bucket)
-        .key("local")
-        .body(ByteStream::from_static(b"local data survives rollback"))
-        .send()
-        .await?;
-    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
-    let restarted = env.get_config(bucket).await?;
-    assert_eq!(restarted.status, 200, "a current writer preserves ODM across restart");
-    assert_eq!(restarted.json()?.get("config"), Some(&expected_config));
-
-    restart_from_binary(&mut env.rustfs, &previous_binary, &[]).await?;
-    env.client
-        .put_bucket_tagging()
-        .bucket(bucket)
-        .tagging(
-            Tagging::builder()
-                .tag_set(Tag::builder().key("writer").value("rc5").build()?)
-                .build()?,
-        )
-        .send()
-        .await?;
-    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
-    let missing = env.get_config(bucket).await?;
-    assert_eq!(missing.status, 404, "rc.5 rewrites metadata without ODM keys");
-    assert!(missing.body.contains("NoSuchConfiguration"));
-    assert_eq!(read_object(&env.client, bucket, "local", None).await?.1, b"local data survives rollback");
-    let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
-    assert!(tags.tag_set().iter().any(|tag| tag.key() == "writer" && tag.value() == "rc5"));
-
-    assert_eq!(
-        env.configure_source(bucket, &saved_config).await?.status,
-        200,
-        "restore from saved full configuration"
-    );
-    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
-    let restored = env.get_config(bucket).await?;
-    assert_eq!(restored.status, 200, "restored ODM configuration persists");
-    assert_eq!(restored.json()?.get("config"), Some(&expected_config));
-    env.wait_until_source_consulted(bucket).await?;
-    assert_eq!(
-        read_object(&env.client, bucket, "source-only", None).await?.1,
-        b"source read after recovery"
-    );
-    Ok(())
-}
-
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
@@ -555,653 +429,3 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes

    Ok(())
 }
-
-/// Child-process environment shared by both bucket-configuration scenarios.
-///
-/// The replication target is an in-process fake bound to `127.0.0.1`, which
-/// `set-remote-target` rejects as an SSRF risk without the loopback opt-in, and
-/// the proxy bypass keeps a developer's `HTTP_PROXY` from intercepting the
-/// server's outbound health check.
-fn bucket_config_server_env() -> Vec<(&'static str, &'static str)> {
-    let mut env = vec![
-        (SSE_MASTER_KEY_ENV, SSE_MASTER_KEY),
-        ("NO_PROXY", "127.0.0.1,localhost"),
-        ("HTTP_PROXY", ""),
-        ("HTTPS_PROXY", ""),
-        // Shorten the scanner cycle so the bucket's usage becomes authoritative
-        // in seconds; both `set-bucket-quota` and quota admission block on it.
-        ("RUSTFS_SCANNER_CYCLE", "1"),
-        ("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
-    ];
-    env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
-    env.extend(replication_fast_env());
-    env
-}
-
-/// Restart `env` in place on the same data directory using an explicit binary.
-///
-/// [`RustFSTestEnvironment::restart_server_preserving_data`] always relaunches
-/// the workspace build, which is the upgrade direction only. The rollback
-/// scenario needs the reverse: stop the current build and bring the pinned
-/// previous release up on the metadata that build just wrote.
-async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, server_env: &[(&str, &str)]) -> TestResult {
-    env.stop_server();
-    env.start_rustfs_server_from_binary(binary, vec![], server_env).await
-}
-
-async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
-    let path = format!("/rustfs/admin/v3/quota/{bucket}");
-    let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
-    let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
-    loop {
-        let (status, response) =
-            admin_request(&env.url, Method::PUT, &path, Some(body.clone()), &env.access_key, &env.secret_key).await?;
-        if status.is_success() {
-            return Ok(());
-        }
-        if status != StatusCode::SERVICE_UNAVAILABLE || Instant::now() >= deadline {
-            return Err(format!("setting the quota of {bucket} failed: {status} {response}").into());
-        }
-        sleep(Duration::from_millis(500)).await;
-    }
-}
-
-/// PUT into a quota-enabled bucket, riding out the post-start quota-admission
-/// warm-up described on [`QUOTA_ADMISSION_WARMUP_TIMEOUT`].
-///
-/// Only `ServiceUnavailable` is retried: any other failure, and a warm-up that
-/// never ends, is a genuine regression and surfaces as an error.
-async fn put_object_through_quota_warmup(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult {
-    let deadline = Instant::now() + QUOTA_ADMISSION_WARMUP_TIMEOUT;
-    loop {
-        let result = client
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(ByteStream::from_static(body))
-            .send()
-            .await;
-        let error = match result {
-            Ok(_) => return Ok(()),
-            Err(error) => error,
-        };
-        let retryable = error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable");
-        if !retryable || Instant::now() >= deadline {
-            return Err(format!("PUT {bucket}/{key} failed after the quota warm-up window: {error}").into());
-        }
-        sleep(Duration::from_millis(500)).await;
-    }
-}
-
-async fn get_bucket_quota(env: &RustFSTestEnvironment, bucket: &str) -> Result<Option<u64>, BoxError> {
-    let path = format!("/rustfs/admin/v3/quota/{bucket}");
-    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
-    if status != StatusCode::OK {
-        return Err(format!("reading the quota of {bucket} failed: {status} {response}").into());
-    }
-    let quota: serde_json::Value = serde_json::from_str(&response)?;
-    Ok(quota.get("quota").and_then(serde_json::Value::as_u64))
-}
-
-/// `GET /rustfs/admin/v3/list-remote-targets?bucket=...`.
-///
-/// Returns an error for any non-200, because rustfs#7172 made this endpoint
-/// fail closed on a `bucket-targets.json` blob the running build cannot parse.
-/// An upgrade that misreads a blob written by the previous release therefore
-/// shows up here as an error, and a silently dropped target shows up as an
-/// empty list — the caller must distinguish the two.
-async fn list_remote_targets(env: &RustFSTestEnvironment, bucket: &str) -> Result<Vec<serde_json::Value>, BoxError> {
-    let path = format!("/rustfs/admin/v3/list-remote-targets?bucket={}", urlencoding::encode(bucket));
-    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
-    if status != StatusCode::OK {
-        return Err(format!("list-remote-targets for {bucket} failed: {status} {response}").into());
-    }
-    Ok(serde_json::from_str(&response)?)
-}
-
-/// Assert that `bucket` still carries exactly the replication target `arn`.
-async fn assert_remote_target_preserved(env: &RustFSTestEnvironment, bucket: &str, arn: &str, context: &str) -> TestResult {
-    let targets = list_remote_targets(env, bucket).await?;
-    assert_eq!(
-        targets.len(),
-        1,
-        "{context}: list-remote-targets must still report the single configured target, got {targets:?}"
-    );
-    assert_eq!(
-        targets[0].get("arn").and_then(serde_json::Value::as_str),
-        Some(arn),
-        "{context}: the target ARN changed across the restart: {targets:?}"
-    );
-    Ok(())
-}
-
-/// Configure a replication target on `bucket` pointing at the in-process fake,
-/// then attach an enabled replication rule for it. Returns the target ARN.
-async fn configure_replication(
-    env: &RustFSTestEnvironment,
-    bucket: &str,
-    target: &FakeS3Target,
-    target_bucket: &str,
-) -> Result<String, BoxError> {
-    let arn = set_replication_target_with_options(
-        env,
-        bucket,
-        ReplicationTargetOptions {
-            endpoint: &target.address(),
-            access_key: FAKE_ACCESS_KEY,
-            secret_key: FAKE_SECRET_KEY,
-            target_bucket,
-            secure: false,
-            skip_tls_verify: false,
-            ca_cert_pem: None,
-        },
-    )
-    .await?;
-    put_bucket_replication(env, bucket, &arn).await?;
-    Ok(arn)
-}
-
-async fn put_default_sse_s3_encryption(client: &Client, bucket: &str) -> TestResult {
-    let configuration = ServerSideEncryptionConfiguration::builder()
-        .rules(
-            ServerSideEncryptionRule::builder()
-                .apply_server_side_encryption_by_default(
-                    ServerSideEncryptionByDefault::builder()
-                        .sse_algorithm(ServerSideEncryption::Aes256)
-                        .build()?,
-                )
-                .build(),
-        )
-        .build()?;
-    client
-        .put_bucket_encryption()
-        .bucket(bucket)
-        .server_side_encryption_configuration(configuration)
-        .send()
-        .await?;
-    Ok(())
-}
-
-async fn assert_default_sse_s3_encryption(client: &Client, bucket: &str, context: &str) -> TestResult {
-    let response = client.get_bucket_encryption().bucket(bucket).send().await?;
-    let rules = response
-        .server_side_encryption_configuration()
-        .ok_or("GetBucketEncryption omitted the configuration")?
-        .rules();
-    assert_eq!(rules.len(), 1, "{context}: expected exactly one encryption rule, got {rules:?}");
-    assert_eq!(
-        rules[0]
-            .apply_server_side_encryption_by_default()
-            .map(ServerSideEncryptionByDefault::sse_algorithm),
-        Some(&ServerSideEncryption::Aes256),
-        "{context}: the default encryption algorithm changed"
-    );
-    Ok(())
-}
-
-async fn put_bucket_tag(client: &Client, bucket: &str) -> TestResult {
-    let tagging = Tagging::builder()
-        .tag_set(Tag::builder().key(BUCKET_TAG_KEY).value(BUCKET_TAG_VALUE).build()?)
-        .build()?;
-    client.put_bucket_tagging().bucket(bucket).tagging(tagging).send().await?;
-    Ok(())
-}
-
-async fn assert_bucket_tag(client: &Client, bucket: &str, context: &str) -> TestResult {
-    let tags = client.get_bucket_tagging().bucket(bucket).send().await?;
-    let tag_set = tags.tag_set();
-    assert_eq!(tag_set.len(), 1, "{context}: expected exactly one bucket tag, got {tag_set:?}");
-    assert_eq!(tag_set[0].key(), BUCKET_TAG_KEY, "{context}: bucket tag key changed");
-    assert_eq!(tag_set[0].value(), BUCKET_TAG_VALUE, "{context}: bucket tag value changed");
-    Ok(())
-}
-
-async fn assert_versioning_enabled(client: &Client, bucket: &str, context: &str) -> TestResult {
-    let versioning = client.get_bucket_versioning().bucket(bucket).send().await?;
-    assert_eq!(
-        versioning.status(),
-        Some(&BucketVersioningStatus::Enabled),
-        "{context}: versioning is no longer Enabled on {bucket}"
-    );
-    Ok(())
-}
-
-fn bucket_policy_document(bucket: &str) -> serde_json::Value {
-    serde_json::json!({
-        "Version": "2012-10-17",
-        "Statement": [{
-            "Sid": "UpgradePublicRead",
-            "Effect": "Allow",
-            "Principal": { "AWS": ["*"] },
-            "Action": ["s3:GetObject"],
-            "Resource": [format!("arn:aws:s3:::{bucket}/public/*")]
-        }]
-    })
-}
-
-/// `GET .../on-demand-migration/{bucket}/status`.
-///
-/// The migration module defaults on from rustfs#7089, so a bucket that never
-/// configured a source must still answer `configured: false` rather than
-/// engaging the migration path.
-async fn assert_migration_not_configured(env: &RustFSTestEnvironment, bucket: &str) -> TestResult {
-    let path = format!("/rustfs/admin/v3/on-demand-migration/{bucket}/status");
-    let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
-    assert_eq!(
-        status,
-        StatusCode::OK,
-        "the migration status endpoint must answer for an unconfigured bucket: {status} {response}"
-    );
-    let body: serde_json::Value = serde_json::from_str(&response)?;
-    assert_eq!(
-        body.get("configured"),
-        Some(&serde_json::Value::Bool(false)),
-        "a bucket upgraded from the previous release must not look migration-configured: {body}"
-    );
-    Ok(())
-}
-
-/// A GET for a key that was never written must be a plain `NoSuchKey`.
-///
-/// With the migration module on by default this is the cheap proof that an
-/// unconfigured bucket never consults a source: any migration engagement would
-/// surface as a different status or error code here.
-async fn assert_missing_key_is_no_such_key(client: &Client, bucket: &str, key: &str) -> TestResult {
-    let error = client
-        .get_object()
-        .bucket(bucket)
-        .key(key)
-        .send()
-        .await
-        .expect_err("a key that was never written must not be readable");
-    assert_eq!(
-        error.raw_response().map(|response| response.status().as_u16()),
-        Some(404),
-        "a missing key must stay a 404 on a bucket with no migration configuration"
-    );
-    assert_eq!(
-        error.as_service_error().and_then(ProvideErrorMetadata::code),
-        Some("NoSuchKey"),
-        "a missing key must stay NoSuchKey on a bucket with no migration configuration"
-    );
-    Ok(())
-}
-
-/// Bucket configuration written by the pinned previous release must survive an
-/// upgrade to the current build unchanged, and must keep working.
-///
-/// This pins the three on-disk surfaces the on-demand-migration series moved:
-///
-/// * `BucketMetadata` grew two msgpack keys (encoded map length 44 -> 46), so
-///   every configuration read below decodes a 44-key blob on 46-key code.
-/// * rustfs#7172 made an unreadable `bucket-targets.json` / encryption /
-///   public-access-block / quota blob "present but unreadable" instead of
-///   silently defaulting, and made `list-remote-targets` fail closed on it. A
-///   replication target configured by the old release must therefore still be
-///   *listed*, not dropped and not an error.
-/// * rustfs#7183 made the object write path refuse a PUT when the bucket's
-///   encryption configuration cannot be read, so a misparsed SSE config would
-///   turn every PUT to that bucket into a 500.
-///
-/// Not covered on purpose: on-demand-migration configuration itself, which the
-/// previous release has no public API for — the reverse direction is asserted
-/// instead (an upgraded bucket reports `configured: false`).
-#[tokio::test]
-#[ignore = "requires a pinned previous RustFS release binary"]
-async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -> TestResult {
-    init_logging();
-    let previous_binary = source_binary()?;
-
-    // In-process: the fake target outlives both server processes, so the
-    // replication target stays reachable across the upgrade.
-    let replication_target = FakeS3Target::start().await?;
-    replication_target.create_bucket(CONFIG_REPLICA_BUCKET);
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    let server_env = bucket_config_server_env();
-    env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
-        .await?;
-    let old_client = env.create_s3_client();
-
-    env.create_test_bucket(CONFIG_PLAIN_BUCKET).await?;
-    env.create_test_bucket(CONFIG_ENCRYPTED_BUCKET).await?;
-    env.create_test_bucket(CONFIG_REPLICATED_BUCKET).await?;
-    old_client
-        .create_bucket()
-        .bucket(CONFIG_LOCKED_BUCKET)
-        .object_lock_enabled_for_bucket(true)
-        .send()
-        .await?;
-
-    // Plain bucket: policy, tags, lifecycle, quota.
-    let policy = bucket_policy_document(CONFIG_PLAIN_BUCKET);
-    old_client
-        .put_bucket_policy()
-        .bucket(CONFIG_PLAIN_BUCKET)
-        .policy(policy.to_string())
-        .send()
-        .await?;
-    put_bucket_tag(&old_client, CONFIG_PLAIN_BUCKET).await?;
-    old_client
-        .put_bucket_lifecycle_configuration()
-        .bucket(CONFIG_PLAIN_BUCKET)
-        .lifecycle_configuration(
-            BucketLifecycleConfiguration::builder()
-                .rules(
-                    LifecycleRule::builder()
-                        .id(LIFECYCLE_RULE_ID)
-                        .status(ExpirationStatus::Enabled)
-                        .filter(LifecycleRuleFilter::builder().prefix(LIFECYCLE_PREFIX).build())
-                        .expiration(LifecycleExpiration::builder().days(LIFECYCLE_DAYS).build())
-                        .build()?,
-                )
-                .build()?,
-        )
-        .send()
-        .await?;
-    set_bucket_quota(&env, CONFIG_PLAIN_BUCKET, BUCKET_QUOTA_BYTES).await?;
-
-    // Encrypted bucket: SSE-S3 default encryption plus a fully restrictive
-    // public access block, both of which rustfs#7172 now fails closed on.
-    put_default_sse_s3_encryption(&old_client, CONFIG_ENCRYPTED_BUCKET).await?;
-    old_client
-        .put_public_access_block()
-        .bucket(CONFIG_ENCRYPTED_BUCKET)
-        .public_access_block_configuration(
-            PublicAccessBlockConfiguration::builder()
-                .block_public_acls(true)
-                .ignore_public_acls(true)
-                .block_public_policy(true)
-                .restrict_public_buckets(true)
-                .build(),
-        )
-        .send()
-        .await?;
-
-    // Replicated bucket: versioning, a validated remote target, a rule.
-    enable_versioning(&old_client, CONFIG_REPLICATED_BUCKET).await?;
-    let target_arn = configure_replication(&env, CONFIG_REPLICATED_BUCKET, &replication_target, CONFIG_REPLICA_BUCKET).await?;
-    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "before the upgrade").await?;
-
-    // Object-lock bucket: a default GOVERNANCE retention on a fresh bucket.
-    old_client
-        .put_object_lock_configuration()
-        .bucket(CONFIG_LOCKED_BUCKET)
-        .object_lock_configuration(
-            ObjectLockConfiguration::builder()
-                .object_lock_enabled(ObjectLockEnabled::Enabled)
-                .rule(
-                    ObjectLockRule::builder()
-                        .default_retention(
-                            DefaultRetention::builder()
-                                .mode(ObjectLockRetentionMode::Governance)
-                                .days(OBJECT_LOCK_DAYS)
-                                .build(),
-                        )
-                        .build(),
-                )
-                .build(),
-        )
-        .send()
-        .await?;
-
-    let plain_key = "plain/written-by-previous";
-    let plain_bytes = b"plain object written by the previous RustFS release";
-    put_object_through_quota_warmup(&old_client, CONFIG_PLAIN_BUCKET, plain_key, plain_bytes).await?;
-
-    let encrypted_key = "encrypted/written-by-previous";
-    let encrypted_bytes = b"default-encrypted object written by the previous RustFS release";
-    old_client
-        .put_object()
-        .bucket(CONFIG_ENCRYPTED_BUCKET)
-        .key(encrypted_key)
-        .body(ByteStream::from_static(encrypted_bytes))
-        .send()
-        .await?;
-    assert_eq!(
-        read_object(&old_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None)
-            .await?
-            .0,
-        Some(ServerSideEncryption::Aes256),
-        "the previous release must apply the bucket default encryption it just accepted"
-    );
-
-    // The multipart object lives in the default-encrypted bucket so the
-    // upgraded build has to reassemble parts *and* re-derive the object key.
-    let multipart_key = "encrypted/multipart-written-by-previous";
-    let multipart_parts = vec![vec![b'm'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()];
-    let multipart_bytes = multipart_parts.concat();
-    write_multipart(&old_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, &multipart_parts).await?;
-
-    let versioned_key = "versioned/written-by-previous";
-    let versioned_bytes = b"versioned object written by the previous RustFS release";
-    let versioned_id = old_client
-        .put_object()
-        .bucket(CONFIG_REPLICATED_BUCKET)
-        .key(versioned_key)
-        .body(ByteStream::from_static(versioned_bytes))
-        .send()
-        .await?
-        .version_id()
-        .ok_or("versioned PUT omitted version ID")?
-        .to_string();
-
-    env.restart_server_preserving_data(vec![], &server_env).await?;
-    let new_client = env.create_s3_client();
-
-    // Every configuration must read back unchanged on the upgraded build.
-    let upgraded_policy = new_client.get_bucket_policy().bucket(CONFIG_PLAIN_BUCKET).send().await?;
-    let upgraded_policy: serde_json::Value =
-        serde_json::from_str(upgraded_policy.policy().ok_or("GetBucketPolicy omitted the document")?)?;
-    assert_eq!(upgraded_policy, policy, "the bucket policy changed across the upgrade");
-    assert_bucket_tag(&new_client, CONFIG_PLAIN_BUCKET, "after the upgrade").await?;
-
-    let lifecycle = new_client
-        .get_bucket_lifecycle_configuration()
-        .bucket(CONFIG_PLAIN_BUCKET)
-        .send()
-        .await?;
-    let rules = lifecycle.rules();
-    assert_eq!(rules.len(), 1, "the lifecycle rule count changed across the upgrade: {rules:?}");
-    assert_eq!(rules[0].id(), Some(LIFECYCLE_RULE_ID));
-    assert_eq!(rules[0].status(), &ExpirationStatus::Enabled);
-    assert_eq!(
-        rules[0].expiration().and_then(LifecycleExpiration::days),
-        Some(LIFECYCLE_DAYS),
-        "the lifecycle expiration changed across the upgrade"
-    );
-
-    assert_eq!(
-        get_bucket_quota(&env, CONFIG_PLAIN_BUCKET).await?,
-        Some(BUCKET_QUOTA_BYTES),
-        "the bucket quota changed across the upgrade"
-    );
-
-    assert_default_sse_s3_encryption(&new_client, CONFIG_ENCRYPTED_BUCKET, "after the upgrade").await?;
-    let public_access_block = new_client
-        .get_public_access_block()
-        .bucket(CONFIG_ENCRYPTED_BUCKET)
-        .send()
-        .await?;
-    let public_access_block = public_access_block
-        .public_access_block_configuration()
-        .ok_or("GetPublicAccessBlock omitted the configuration")?;
-    assert_eq!(public_access_block.block_public_acls(), Some(true));
-    assert_eq!(public_access_block.ignore_public_acls(), Some(true));
-    assert_eq!(public_access_block.block_public_policy(), Some(true));
-    assert_eq!(public_access_block.restrict_public_buckets(), Some(true));
-
-    assert_versioning_enabled(&new_client, CONFIG_REPLICATED_BUCKET, "after the upgrade").await?;
-    // rustfs#7172: neither an empty list nor an error is acceptable here.
-    assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "after the upgrade").await?;
-    let replication = new_client
-        .get_bucket_replication()
-        .bucket(CONFIG_REPLICATED_BUCKET)
-        .send()
-        .await?;
-    let replication_rules = replication
-        .replication_configuration()
-        .ok_or("GetBucketReplication omitted the configuration")?
-        .rules();
-    assert_eq!(
-        replication_rules.len(),
-        1,
-        "the replication rule count changed across the upgrade: {replication_rules:?}"
-    );
-    assert_eq!(
-        replication_rules[0].destination().map(|destination| destination.bucket()),
-        Some(target_arn.as_str()),
-        "the replication rule no longer points at the configured target"
-    );
-
-    let object_lock = new_client
-        .get_object_lock_configuration()
-        .bucket(CONFIG_LOCKED_BUCKET)
-        .send()
-        .await?;
-    let object_lock = object_lock
-        .object_lock_configuration()
-        .ok_or("GetObjectLockConfiguration omitted the configuration")?;
-    assert_eq!(object_lock.object_lock_enabled(), Some(&ObjectLockEnabled::Enabled));
-    let retention = object_lock
-        .rule()
-        .and_then(ObjectLockRule::default_retention)
-        .ok_or("the object lock configuration lost its default retention")?;
-    assert_eq!(retention.mode(), Some(&ObjectLockRetentionMode::Governance));
-    assert_eq!(retention.days(), Some(OBJECT_LOCK_DAYS));
-
-    // rustfs#7183: a PUT into the default-encrypted bucket must still succeed
-    // and still come back encrypted.
-    let post_upgrade_encrypted_key = "encrypted/written-after-upgrade";
-    let post_upgrade_encrypted_bytes = b"default-encrypted object written by the current RustFS build";
-    new_client
-        .put_object()
-        .bucket(CONFIG_ENCRYPTED_BUCKET)
-        .key(post_upgrade_encrypted_key)
-        .body(ByteStream::from_static(post_upgrade_encrypted_bytes))
-        .send()
-        .await?;
-    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, post_upgrade_encrypted_key, None).await?;
-    assert_eq!(
-        encryption,
-        Some(ServerSideEncryption::Aes256),
-        "a PUT after the upgrade lost the bucket default encryption"
-    );
-    assert_eq!(body, post_upgrade_encrypted_bytes);
-
-    let post_upgrade_plain_key = "plain/written-after-upgrade";
-    let post_upgrade_plain_bytes = b"plain object written by the current RustFS build";
-    put_object_through_quota_warmup(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, post_upgrade_plain_bytes).await?;
-    let (encryption, body) = read_object(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, None).await?;
-    assert_eq!(encryption, None, "a bucket without default encryption must not encrypt a PUT");
-    assert_eq!(body, post_upgrade_plain_bytes);
-
-    // Every object written by the previous release reads back byte-identical.
-    assert_eq!(read_object(&new_client, CONFIG_PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes);
-    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None).await?;
-    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
-    assert_eq!(body, encrypted_bytes);
-    let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, None).await?;
-    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
-    assert_eq!(body, multipart_bytes, "the multipart object did not survive the upgrade");
-    assert_eq!(
-        read_object(&new_client, CONFIG_REPLICATED_BUCKET, versioned_key, Some(&versioned_id))
-            .await?
-            .1,
-        versioned_bytes
-    );
-
-    // rustfs#7089: the migration module is on by default, but a bucket that
-    // never configured a source behaves exactly as before.
-    assert_migration_not_configured(&env, CONFIG_PLAIN_BUCKET).await?;
-    assert_missing_key_is_no_such_key(&new_client, CONFIG_PLAIN_BUCKET, "plain/never-written").await?;
-
-    replication_target.shutdown().await;
-    Ok(())
-}
-
-/// Rolling back to the pinned previous release must still read the bucket
-/// metadata the current build wrote.
-///
-/// This is the other half of the `BucketMetadata` 44 -> 46 key change: the
-/// current build writes a 46-key msgpack map with `OnDemandMigrationConfigJSON`
-/// and `OnDemandMigrationConfigUpdatedAt`, and the previous release's decoder
-/// has to skip those two unknown keys instead of failing the whole blob. If it
-/// did not, every configuration read below would come back empty or error and
-/// the rollback would silently discard the bucket's configuration.
-#[tokio::test]
-#[ignore = "requires a pinned previous RustFS release binary"]
-async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
-    init_logging();
-    let previous_binary = source_binary()?;
-
-    let replication_target = FakeS3Target::start().await?;
-    replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
-
-    let mut env = RustFSTestEnvironment::new().await?;
-    let server_env = bucket_config_server_env();
-    env.start_rustfs_server_with_env(vec![], &server_env).await?;
-    let new_client = env.create_s3_client();
-
-    env.create_test_bucket(ROLLBACK_BUCKET).await?;
-    enable_versioning(&new_client, ROLLBACK_BUCKET).await?;
-    put_default_sse_s3_encryption(&new_client, ROLLBACK_BUCKET).await?;
-    put_bucket_tag(&new_client, ROLLBACK_BUCKET).await?;
-    let target_arn = configure_replication(&env, ROLLBACK_BUCKET, &replication_target, ROLLBACK_REPLICA_BUCKET).await?;
-    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "before the rollback").await?;
-
-    let single_key = "rollback/single";
-    let single_bytes = b"single-part object written by the current RustFS build";
-    let single_version = new_client
-        .put_object()
-        .bucket(ROLLBACK_BUCKET)
-        .key(single_key)
-        .body(ByteStream::from_static(single_bytes))
-        .send()
-        .await?
-        .version_id()
-        .ok_or("versioned PUT omitted version ID")?
-        .to_string();
-
-    let multipart_key = "rollback/multipart";
-    let multipart_parts = vec![vec![b'r'; 5 * 1024 * 1024], b"final rollback bytes".to_vec()];
-    let multipart_bytes = multipart_parts.concat();
-    write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
-
-    restart_from_binary(&mut env, &previous_binary, &server_env).await?;
-    let old_client = env.create_s3_client();
-
-    assert_versioning_enabled(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
-    assert_default_sse_s3_encryption(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
-    assert_bucket_tag(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
-    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after the rollback").await?;
-
-    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, single_key, Some(&single_version)).await?;
-    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
-    assert_eq!(body, single_bytes);
-    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, multipart_key, None).await?;
-    assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
-    assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
-
-    // A PUT on the rolled-back release must still honour the encryption
-    // configuration it decoded out of the current build's metadata blob.
-    let post_rollback_key = "rollback/written-after-rollback";
-    let post_rollback_bytes = b"object written by the previous RustFS release after the rollback";
-    old_client
-        .put_object()
-        .bucket(ROLLBACK_BUCKET)
-        .key(post_rollback_key)
-        .body(ByteStream::from_static(post_rollback_bytes))
-        .send()
-        .await?;
-    let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, post_rollback_key, None).await?;
-    assert_eq!(
-        encryption,
-        Some(ServerSideEncryption::Aes256),
-        "the rolled-back release lost the bucket default encryption"
-    );
-    assert_eq!(body, post_rollback_bytes);
-
-    replication_target.shutdown().await;
-    Ok(())
-}
@@ -31,7 +31,6 @@ workspace = true

 [features]
 default = []
-gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
 # Compiles the controlled list-objects namespace-journal chaos injector into a
 # production binary (it is always available to tests). Off by default so the
 # RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
@@ -213,9 +212,10 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
 parking_lot = { workspace = true }
 base64-simd.workspace = true
 serde_urlencoded.workspace = true
-google-cloud-storage = { workspace = true, optional = true }
-google-cloud-auth = { workspace = true, optional = true }
+google-cloud-storage = { workspace = true }
+google-cloud-auth = { workspace = true }
 faster-hex = { workspace = true }
+quick-xml = { workspace = true }
 ratelimit = { workspace = true }
 aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }

@@ -146,23 +146,69 @@ pub mod bucket {
        };
    }

+    pub mod on_demand_migration {
+        pub use crate::bucket::on_demand_migration::{
+            ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
+            Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
+            LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
+            OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
+            PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
+            SourceLatencySnapshot, source_backend_spec, source_client_spec,
+        };
+        pub use crate::bucket::on_demand_migration::{
+            AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
+            ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig,
+            Provider, RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig,
+            ValidationContext,
+        };
+        pub use crate::bucket::on_demand_migration::{
+            EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
+            PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody,
+            WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
+            idle_guarded_body,
+        };
+        pub use crate::bucket::on_demand_migration::{
+            FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
+            ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome,
+            MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
+            decode_continuation_token, source_list_plan,
+        };
+        pub mod backfill {
+            pub use crate::bucket::on_demand_migration::backfill::{
+                BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
+                BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS,
+                BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError,
+                BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState,
+                BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting,
+                StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash,
+                read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop,
+            };
+        }
+        pub mod source_client {
+            pub use crate::bucket::on_demand_migration::source_client::{
+                AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError,
+                SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceProbe, SourceProvider, SourceSse,
+                SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value, resolve_path_style,
+            };
+        }
+    }
+
    pub mod metadata_sys {
+        #[cfg(feature = "test-util")]
+        pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
        pub use crate::bucket::metadata_sys::{
-            BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
-            ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
+            BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
            acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
            capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
            get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
            get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
            get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
-            get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
-            get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
-            init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
-            update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
-            update_quota_if_incarnation, update_under_transaction_lock,
+            get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
+            get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
+            reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
+            update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
+            update_under_transaction_lock,
        };
-        #[cfg(feature = "test-util")]
-        pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
    }

    pub mod migration {
@@ -205,7 +251,7 @@ pub mod bucket {
    pub mod remote_s3_client {
        pub use crate::bucket::remote_s3_client::{
            PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_client,
-            build_remote_s3_config, validate_remote_endpoint, validate_target_ca_pem,
+            validate_remote_endpoint,
        };
    }

@@ -451,9 +497,9 @@ pub mod object {
        ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
        RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
        SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
-        ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
-        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
-        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
+        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
+        unregister_object_mutation_hook,
    };
    pub use crate::store::{
        PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
@@ -489,17 +489,28 @@ impl BucketMetadata {
        !self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
    }

-    /// Opaque application-owned configuration with its persisted update time.
-    /// Empty bytes mean absent or cleared; decoding belongs to the consumer.
-    pub fn on_demand_migration_config(&self) -> Option<(&[u8], OffsetDateTime)> {
-        (!self.on_demand_migration_config_json.is_empty()).then_some((
-            self.on_demand_migration_config_json.as_slice(),
-            self.on_demand_migration_config_updated_at,
-        ))
+    /// Parsed per-bucket durability override, if a valid one is stored.
+    ///
+    /// Absent/empty/unparsable payloads all mean "no override" (the bucket
+    /// follows the global durability mode); a parse failure is logged so a
+    /// corrupted entry cannot silently change fsync behavior.
+    /// Parsed on-demand migration config, if one is stored.
+    ///
+    /// `Ok(None)` means no config (absent or cleared). A stored payload that
+    /// does not parse is an error, never a default: the runtime must not
+    /// pull from a source it cannot describe.
+    pub fn on_demand_migration_config(
+        &self,
+    ) -> std::result::Result<
+        Option<super::on_demand_migration::OnDemandMigrationConfig>,
+        super::on_demand_migration::OnDemandMigrationConfigError,
+    > {
+        if self.on_demand_migration_config_json.is_empty() {
+            return Ok(None);
+        }
+        super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
    }

-    /// Parsed per-bucket durability override, if a valid one is stored.
-    /// Invalid payloads follow the global mode after logging a parse failure.
    pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
        if self.durability_config_json.is_empty() {
            return None;
@@ -905,6 +916,13 @@ impl BucketMetadata {
                self.durability_config_updated_at = updated;
            }
            BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
+                // Structural check only (shape, unknown fields); the
+                // deployment-relative rules run in the admin handler with a
+                // `ValidationContext`. A blob this build cannot read must not
+                // be persisted for every later reader to trip over.
+                if !data.is_empty() {
+                    super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
+                }
                self.on_demand_migration_config_json = data;
                self.on_demand_migration_config_updated_at = updated;
            }
@@ -1960,30 +1978,51 @@ mod test {

    const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;

-    /// The metadata codec preserves application-owned bytes and timestamps.
+    /// rustfs/backlog#2148: the on-demand migration config is a RustFS
+    /// extension entry that round-trips through `update_config` and the
+    /// msgpack codec, clears on delete, and never parses corruption into a
+    /// default.
    #[test]
    fn on_demand_migration_config_round_trips_and_tracks_updates() {
+        use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
+
        let mut bm = BucketMetadata::new("odm-bucket");
-        assert_eq!(bm.on_demand_migration_config(), None, "fresh metadata carries no config");
+        assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
+
+        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .expect("opaque config is accepted");
-        let stamped = bm.on_demand_migration_config_updated_at;
-        assert_ne!(stamped, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), Some((ODM_JSON, stamped)));
+            .expect("valid config is accepted");
+        assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
+        assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
+
        let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
        assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
-        assert_eq!(back.on_demand_migration_config_updated_at.unix_timestamp(), stamped.unix_timestamp());
+        assert_eq!(
+            back.on_demand_migration_config_updated_at.unix_timestamp(),
+            bm.on_demand_migration_config_updated_at.unix_timestamp()
+        );
+        assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
+
+        // A blob this build cannot read is rejected at the write boundary
+        // rather than persisted for every reader to trip over.
+        let before = bm.on_demand_migration_config_json.clone();
+        assert!(
+            bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
+                .is_err()
+        );
+        assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
+
+        // Delete clears the entry.
+        let stamped = bm.on_demand_migration_config_updated_at;
        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
        assert!(bm.on_demand_migration_config_json.is_empty());
-        assert_eq!(bm.on_demand_migration_config(), None);
+        assert_eq!(bm.on_demand_migration_config(), Ok(None));
        assert!(bm.on_demand_migration_config_updated_at >= stamped);
-        bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, b"not-json".to_vec())
-            .unwrap();
-        let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
-        assert_eq!(
-            back.on_demand_migration_config_json, b"not-json",
-            "metadata must not reinterpret application bytes"
-        );
+
+        // Corruption that bypassed `update_config` (disk, another writer)
+        // is a typed error, never a default.
+        bm.on_demand_migration_config_json = b"not-json".to_vec();
+        assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
    }

    /// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
@@ -1995,7 +2034,7 @@ mod test {
        let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
        assert!(bm.on_demand_migration_config_json.is_empty());
        assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
-        assert_eq!(bm.on_demand_migration_config(), None);
+        assert_eq!(bm.on_demand_migration_config(), Ok(None));

        bm.default_timestamps();
        assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");
@@ -19,6 +19,7 @@ use super::quota::BucketQuota;
 use super::target::BucketTargets;
 use crate::bucket::bucket_target_sys::BucketTargetSys;
 use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
+use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
 use crate::bucket::utils::is_meta_bucketname;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
@@ -48,11 +49,6 @@ use tokio_util::sync::CancellationToken;
 use tracing::{error, warn};
 use uuid::Uuid;

-/// Opaque bucket configuration notifications for application-owned services.
-/// `None` withdraws a configuration; consumers validate nonempty bytes.
-pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime, Uuid)>) + Send + Sync>;
-pub static BUCKET_CONFIG_PUBLISH_HOOK: std::sync::OnceLock<BucketConfigPublishHook> = std::sync::OnceLock::new();
-
 const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);

 #[cfg(any(test, feature = "test-util"))]
@@ -399,21 +395,39 @@ fn clear_bucket_durability(bucket: &str) {
    crate::disk::local::bucket_durability::set(bucket, None);
 }

-/// Publish application-owned bytes on every cache install path.
+/// Publish the bucket's on-demand migration config (or its absence) to the
+/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
+///
+/// Called from the same five cache-install paths as
+/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
+/// published as `None`: the runtime must stop pulling for that bucket rather
+/// than keep an older config or guess.
 fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
-    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
-        hook(
-            bucket,
-            super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG,
-            bm.on_demand_migration_config()
-                .map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
-        );
+    let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
+        return;
+    };
+    match bm.on_demand_migration_config() {
+        Ok(config) => hook(bucket, config.as_ref()),
+        Err(err) => {
+            warn!(
+                event = "bucket_metadata_parse_failed",
+                component = "ecstore",
+                subsystem = "bucket_metadata",
+                bucket = %bucket,
+                config = "on_demand_migration",
+                error = %err,
+                "Failed to parse bucket metadata config"
+            );
+            hook(bucket, None);
+        }
    }
 }

+/// Withdraw a bucket's on-demand migration config when its metadata leaves
+/// the cache.
 fn clear_on_demand_migration(bucket: &str) {
-    if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
-        hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
+    if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
+        hook(bucket, None);
    }
 }

@@ -1035,21 +1049,15 @@ pub async fn get_durability_config(
 }

 /// The bucket's on-demand migration config with its update time, or
-/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
-pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
+/// `Ok(None)` when the bucket has none. A stored payload that does not parse
+/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
+pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
    let bucket_meta_sys = bucket_meta_sys_lock.read().await;

    bucket_meta_sys.get_on_demand_migration_config(bucket).await
 }

-/// Resolve opaque configuration from the store's own metadata system.
-pub async fn get_on_demand_migration_config_in(api: &ECStore, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
-    let sys = bucket_metadata_sys_of(&api.ctx)?;
-    let lock = sys.read().await;
-    lock.get_on_demand_migration_config(bucket).await
-}
-
 pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
    let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -2571,27 +2579,29 @@ impl BucketMetadataSys {
    }

    /// See [`get_on_demand_migration_config`].
-    pub async fn get_on_demand_migration_config(&self, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
+    pub async fn get_on_demand_migration_config(
+        &self,
+        bucket: &str,
+    ) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
        let (bm, _) = self.get_config(bucket).await?;

-        Ok(bm
-            .on_demand_migration_config()
-            .map(|(bytes, updated_at)| (bytes.to_vec(), updated_at)))
+        let config = bm.on_demand_migration_config().map_err(Error::other)?;
+        Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
    }
 }

 /// Test-only fixture shared with sibling modules (e.g. the quota checker
 /// tests): a 4-disk `ECStore` on an isolated instance context, so tests
 /// exercising the metadata system never touch ambient process state.
-#[cfg(any(test, feature = "test-util"))]
-pub mod test_support {
+#[cfg(test)]
+pub(crate) mod test_support {
    use super::*;
    use crate::disk::endpoint::Endpoint;
    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
    use crate::runtime::instance::InstanceContext;
    use crate::store::init_local_disks_with_instance_ctx;

-    pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
+    pub(crate) async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
        let mut dirs = Vec::with_capacity(4);
        let mut endpoints = Vec::with_capacity(4);
        for disk_idx in 0..4 {
@@ -4375,26 +4385,19 @@ mod tests {

    const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;

-    type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
-    type RecordedOdmHookCall = (String, RecordedOdmConfig);
-
    /// Every `(bucket, config)` the recording hook has seen. Tests filter by
    /// their own bucket name; the hook is process-wide and set once.
-    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
+    static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());

    fn install_recording_odm_hook() {
-        BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
-            Box::new(|bucket, config_file, config| {
-                assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
-                ODM_HOOK_CALLS.lock().unwrap().push((
-                    bucket.to_string(),
-                    config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
-                ));
+        ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
+            Box::new(|bucket, config| {
+                ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
            })
        });
    }

-    fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
+    fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
        ODM_HOOK_CALLS
            .lock()
            .unwrap()
@@ -4404,6 +4407,54 @@ mod tests {
            .collect()
    }

+    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
+    /// stored payload it cannot parse as a typed error, never as a default
+    /// and never as `ConfigNotFound`.
+    #[tokio::test]
+    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
+        use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
+
+        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
+        let sys = BucketMetadataSys::new(ecstore);
+        let bucket = "odm-accessor";
+
+        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
+        assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
+
+        let mut corrupt = BucketMetadata::new(bucket);
+        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
+        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
+        let err = sys
+            .get_on_demand_migration_config(bucket)
+            .await
+            .expect_err("corrupt config must not read as a default");
+        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
+        let typed = match &err {
+            Error::Io(io) => io
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
+            _ => None,
+        };
+        assert!(
+            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
+            "typed parse error must survive the Result boundary, got: {err:?}"
+        );
+
+        let mut valid = BucketMetadata::new(bucket);
+        valid
+            .update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
+            .unwrap();
+        let stamped = valid.on_demand_migration_config_updated_at;
+        sys.set(bucket.to_string(), Arc::new(valid)).await;
+        let (config, updated_at) = sys
+            .get_on_demand_migration_config(bucket)
+            .await
+            .unwrap()
+            .expect("stored config is returned");
+        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
+        assert_eq!(updated_at, stamped);
+    }
+
    /// rustfs/backlog#2148: the publish hook fires on every path that
    /// installs bucket metadata into the cache (set, initial load, peer
    /// reload, refresh loop, lazy load) and withdraws on removal, mirroring
@@ -4417,22 +4468,15 @@ mod tests {
        for dir in &dirs {
            std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
        }
-
-        let incarnation = Uuid::new_v4();
+        let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
        let expect_publish = |before: usize, label: &str| {
            let calls = odm_hook_calls(bucket);
            assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
-            assert_eq!(
-                calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
-                Some(ODM_JSON),
-                "{label} must publish the stored bytes"
-            );
-            assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
+            assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
        };

        // set (via persist_new_and_set, which installs through `set`).
        let mut bm = BucketMetadata::new(bucket);
-        bm.bucket_incarnation_id = incarnation;
        bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
            .unwrap();
        let writer = BucketMetadataSys::new(ecstore.clone());
@@ -4474,18 +4518,14 @@ mod tests {
        assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
        assert_eq!(calls.last().unwrap(), &None);

-        // Opaque bytes reach the application even if they are not valid JSON.
+        // A corrupt payload is withdrawn, never published as a config.
        let mut corrupt = BucketMetadata::new(bucket);
        corrupt.on_demand_migration_config_json = b"not-json".to_vec();
        let before = odm_hook_calls(bucket).len();
        lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
        let calls = odm_hook_calls(bucket);
        assert_eq!(calls.len(), before + 1);
-        assert_eq!(
-            calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
-            Some(b"not-json".as_slice()),
-            "the application validates opaque config bytes"
-        );
+        assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
    }

    #[tokio::test]
@@ -26,6 +26,7 @@ mod metadata_test;
 pub mod migration;
 mod msgp_decode;
 pub mod object_lock;
+pub mod on_demand_migration;
 pub mod policy_sys;
 pub mod quota;
 pub mod remote_s3_client;
@@ -40,8 +40,8 @@ use super::source_client::{
    AzureAuth, AzureSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
    SourceTimeouts, range_header_value,
 };
-use super::storage_api::HTTPRangeSpec;
-use super::storage_api::remote_s3_client::RemoteS3ClientError;
+use crate::bucket::remote_s3_client::RemoteS3ClientError;
+use crate::storage_api_contracts::range::HTTPRangeSpec;
 use hmac::{Hmac, Mac, digest::KeyInit};
 use http::{HeaderMap, HeaderValue, Method};
 use quick_xml::Reader;
@@ -162,32 +162,6 @@ impl AzureSourceBackend {
        Ok(request)
    }

-    /// A missing blob is distinct from a missing container or version. Only
-    /// object reads may use BlobNotFound as positive evidence of absence.
-    async fn send_object_request(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
-        let is_head = request.method() == Method::HEAD;
-        let versioned = request
-            .url()
-            .query_pairs()
-            .any(|(name, _)| name.eq_ignore_ascii_case("versionid") || name.eq_ignore_ascii_case("snapshot"));
-        let response = self.http.execute(request).await?;
-        if response.status() == http::StatusCode::NOT_FOUND && !versioned {
-            match header(response.headers(), HEADER_ERROR_CODE) {
-                Some("BlobNotFound") => return Err(SourceError::NotFound),
-                None | Some("ResourceNotFound") if is_head => {
-                    // HEAD may omit an error code. One successful container
-                    // probe proves key absence; a failed probe keeps its error.
-                    // These are two independently timed requests, not one deadline.
-                    drop(response);
-                    self.probe().await?;
-                    return Err(SourceError::NotFound);
-                }
-                _ => {}
-            }
-        }
-        NativeHttp::check_response(response, Some(HEADER_ERROR_CODE))
-    }
-
    /// Shared mapping for Get Blob and Get Blob Properties.
    fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
        // A customer-provided key means the service holds ciphertext it cannot
@@ -214,7 +188,7 @@ impl AzureSourceBackend {
 impl SourceBackend for AzureSourceBackend {
    async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
        let request = self.request(Method::HEAD, self.blob_url(key)?, HeaderMap::new())?;
-        let response = self.send_object_request(request).await?;
+        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
        Self::head_from_response(response.headers())
    }

@@ -227,7 +201,7 @@ impl SourceBackend for AzureSourceBackend {
            );
        }
        let request = self.request(Method::GET, self.blob_url(key)?, headers)?;
-        let response = self.send_object_request(request).await?;
+        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
        let head = Self::head_from_response(response.headers())?;
        let content_range = header(response.headers(), "content-range").map(str::to_string);
        Ok(SourceGet {
@@ -265,7 +239,7 @@ impl SourceBackend for AzureSourceBackend {
        }

        let request = self.request(Method::GET, url, HeaderMap::new())?;
-        let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
+        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
        let body = read_text(response, MAX_XML_BYTES).await?;
        let listing = parse_list_blobs(&body)?;

@@ -281,7 +255,7 @@ impl SourceBackend for AzureSourceBackend {
        let mut url = self.blob_url(key)?;
        url.query_pairs_mut().append_pair("comp", "tags");
        let request = self.request(Method::GET, url, HeaderMap::new())?;
-        let response = self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
+        let response = self.http.send(request, HEADER_ERROR_CODE).await?;
        let body = read_text(response, MAX_XML_BYTES).await?;
        parse_blob_tags(&body)
    }
@@ -290,7 +264,7 @@ impl SourceBackend for AzureSourceBackend {
        let mut url = self.container_url()?;
        url.query_pairs_mut().append_pair("restype", "container");
        let request = self.request(Method::HEAD, url, HeaderMap::new())?;
-        self.http.send(request, Some(HEADER_ERROR_CODE)).await?;
+        self.http.send(request, HEADER_ERROR_CODE).await?;
        Ok(())
    }
 }
@@ -368,9 +342,9 @@ struct AzureListing {

 #[derive(Default)]
 struct BlobEntry {
-    name: Option<String>,
+    name: String,
    etag: Option<String>,
-    size: Option<u64>,
+    size: u64,
    last_modified: Option<std::time::SystemTime>,
    access_tier: Option<String>,
 }
@@ -383,7 +357,6 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
    let mut next_marker = None;
    let mut blob: Option<BlobEntry> = None;
    let mut in_blob_prefix = false;
-    let mut blob_prefix: Option<String> = None;
    // Open container elements. quick-xml reports a truncated document as a
    // plain end of input, so a non-zero depth at EOF is the only signal that
    // the page was cut short and must not be read as a complete listing.
@@ -393,9 +366,6 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
        match reader.read_event() {
            Ok(Event::Start(start)) => {
                let name = local_name(start.name().as_ref());
-                if matches!(name.as_str(), "blob" | "blobprefix") && (blob.is_some() || in_blob_prefix) {
-                    return Err(SourceError::Other("source listing entries must not be nested".to_string()));
-                }
                match name.as_str() {
                    "blob" => {
                        depth += 1;
@@ -409,30 +379,22 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
                    _ => {
                        let end = start.to_end().into_owned();
                        let text = leaf_text(&mut reader, end.name())?;
-                        apply_list_field(&name, text, &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?;
+                        apply_list_field(&name, text, &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix);
                    }
                }
            }
            Ok(Event::Empty(empty)) => {
                let name = local_name(empty.name().as_ref());
-                if matches!(name.as_str(), "blob" | "blobprefix") {
-                    return Err(SourceError::Other("source listing entry has no name".to_string()));
-                }
-                apply_list_field(&name, String::new(), &mut blob, &mut blob_prefix, &mut next_marker, in_blob_prefix)?;
+                apply_list_field(&name, String::new(), &mut blob, &mut prefixes, &mut next_marker, in_blob_prefix);
            }
            Ok(Event::End(end)) => match local_name(end.name().as_ref()).as_str() {
                "blob" => {
                    depth = depth.saturating_sub(1);
                    if let Some(entry) = blob.take() {
                        objects.push(SourceObject {
-                            key: entry
-                                .name
-                                .filter(|name| !name.is_empty())
-                                .ok_or_else(|| SourceError::Other("source listing object has no name".to_string()))?,
+                            key: entry.name,
                            etag: entry.etag,
-                            size: entry
-                                .size
-                                .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?,
+                            size: entry.size,
                            last_modified: entry.last_modified,
                            storage_class: entry.access_tier,
                            // Azure ETags carry no part count; the listing
@@ -444,12 +406,6 @@ fn parse_list_blobs(xml: &str) -> Result<AzureListing, SourceError> {
                "blobprefix" => {
                    depth = depth.saturating_sub(1);
                    in_blob_prefix = false;
-                    prefixes.push(
-                        blob_prefix
-                            .take()
-                            .filter(|name| !name.is_empty())
-                            .ok_or_else(|| SourceError::Other("source listing prefix has no name".to_string()))?,
-                    );
                }
                "properties" | "blobs" | "enumerationresults" => depth = depth.saturating_sub(1),
                _ => {}
@@ -474,22 +430,16 @@ fn apply_list_field(
    name: &str,
    text: String,
    blob: &mut Option<BlobEntry>,
-    blob_prefix: &mut Option<String>,
+    prefixes: &mut Vec<String>,
    next_marker: &mut Option<String>,
    in_blob_prefix: bool,
-) -> Result<(), SourceError> {
+) {
    match name {
        "name" => {
            if in_blob_prefix {
-                if blob_prefix.is_some() {
-                    return Err(SourceError::Other("source listing prefix has duplicate names".to_string()));
-                }
-                *blob_prefix = Some(text);
+                prefixes.push(text);
            } else if let Some(entry) = blob.as_mut() {
-                if entry.name.is_some() {
-                    return Err(SourceError::Other("source listing object has duplicate names".to_string()));
-                }
-                entry.name = Some(text);
+                entry.name = text;
            }
        }
        "nextmarker" => *next_marker = Some(text),
@@ -500,14 +450,7 @@ fn apply_list_field(
        }
        "content-length" => {
            if let Some(entry) = blob.as_mut() {
-                if entry.size.is_some() {
-                    return Err(SourceError::Other("source listing object has duplicate sizes".to_string()));
-                }
-                entry.size = Some(
-                    text.trim()
-                        .parse()
-                        .map_err(|_| SourceError::Other("source listing object has no valid size".to_string()))?,
-                );
+                entry.size = text.trim().parse().unwrap_or(0);
            }
        }
        "last-modified" => {
@@ -522,7 +465,6 @@ fn apply_list_field(
        }
        _ => {}
    }
-    Ok(())
 }

 /// Parses a `Get Blob Tags` response.
@@ -607,9 +549,9 @@ fn leaf_text(reader: &mut Reader<&[u8]>, end: quick_xml::name::QName<'_>) -> Res
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
-    use crate::on_demand_migration::source_client::SourceError;
-    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server};
+    use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
+    use crate::bucket::on_demand_migration::source_client::SourceError;
+    use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};

    const LIST_PAGE: &str = r#"<?xml version="1.0" encoding="utf-8"?>
 <EnumerationResults ServiceEndpoint="https://acct.blob.core.windows.net/" ContainerName="legacy">
@@ -691,107 +633,6 @@ mod tests {
        assert!(parse_blob_tags("<Tags><TagSet>").is_err(), "a truncated tag set must fail");
    }

-    #[tokio::test]
-    async fn native_listing_rejects_missing_or_invalid_required_object_fields() {
-        for entry in [
-            "<Blob />",
-            "<Blob><Properties><Content-Length>1</Content-Length></Properties></Blob>",
-            "<Blob><Name /><Properties><Content-Length>1</Content-Length></Properties></Blob>",
-            "<Blob><Name>broken</Name></Blob>",
-            "<Blob><Name>broken</Name><Properties><Content-Length /></Properties></Blob>",
-            "<Blob><Name>broken</Name><Properties><Content-Length>-1</Content-Length></Properties></Blob>",
-            "<Blob><Name>broken</Name><Properties><Content-Length>18446744073709551616</Content-Length></Properties></Blob>",
-            "<Blob><Name>broken</Name><Properties><Content-Length>not-a-size</Content-Length></Properties></Blob>",
-            "<BlobPrefix />",
-            "<BlobPrefix><Name /></BlobPrefix>",
-            "<BlobPrefix></BlobPrefix>",
-        ] {
-            // Reject the entire page even if a valid object precedes the bad
-            // entry, so callers cannot expose partial data or advance its cursor.
-            let body = format!(
-                "<EnumerationResults><Blobs><Blob><Name>valid</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker>next</NextMarker></EnumerationResults>"
-            );
-            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
-            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
-                .list(&SourceListRequest {
-                    prefix: Some("dir/"),
-                    delimiter: Some("/"),
-                    continuation_token: Some("opaque+/="),
-                    max_keys: 2,
-                    ..Default::default()
-                })
-                .await
-                .expect_err("malformed object must reject the complete native page");
-            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
-            assert!(!err.is_retryable());
-            assert_requests(
-                &recorded,
-                &[(
-                    "GET",
-                    "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2",
-                )],
-            );
-        }
-    }
-
-    #[tokio::test]
-    async fn native_listing_rejects_duplicate_fields_and_nested_entries() {
-        for entry in [
-            "<Blob><Name>a</Name><Name>b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>",
-            "<Blob><Name /><Name>b</Name><Properties><Content-Length>1</Content-Length></Properties></Blob>",
-            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length><Content-Length>2</Content-Length></Properties></Blob>",
-            "<BlobPrefix><Name>a/</Name><Name>b/</Name></BlobPrefix>",
-            "<BlobPrefix><Name /><Name>b/</Name></BlobPrefix>",
-            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length></Properties><Blob><Name>b</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></Blob>",
-            "<Blob><Name>a</Name><Properties><Content-Length>1</Content-Length></Properties><BlobPrefix><Name>b/</Name></BlobPrefix></Blob>",
-            "<BlobPrefix><Name>a/</Name><Blob><Name>b</Name><Properties><Content-Length>2</Content-Length></Properties></Blob></BlobPrefix>",
-            "<BlobPrefix><Name>a/</Name><BlobPrefix><Name>b/</Name></BlobPrefix></BlobPrefix>",
-        ] {
-            let body = format!(
-                "<EnumerationResults><Blobs><Blob><Name>valid</Name><Properties><Content-Length>0</Content-Length></Properties></Blob>{entry}</Blobs><NextMarker>next</NextMarker></EnumerationResults>"
-            );
-            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
-            let result = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
-                .list(&SourceListRequest {
-                    delimiter: Some("/"),
-                    continuation_token: Some("opaque+/="),
-                    max_keys: 2,
-                    ..Default::default()
-                })
-                .await;
-            let err = result.expect_err("ambiguous entries must reject the entire page and its cursor");
-            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
-            assert!(!err.is_retryable(), "{entry}: {err:?}");
-            assert_requests(
-                &recorded,
-                &[(
-                    "GET",
-                    "/legacy?restype=container&comp=list&delimiter=%2F&marker=opaque%2B%2F%3D&maxresults=2",
-                )],
-            );
-        }
-    }
-
-    #[tokio::test]
-    async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() {
-        let body = "<EnumerationResults><Blobs><Blob><Name>目录/空 &amp; file</Name><Properties><Content-Length>0</Content-Length></Properties></Blob><BlobPrefix><Name>目录/子/</Name></BlobPrefix></Blobs><NextMarker>opaque+/=</NextMarker></EnumerationResults>";
-        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
-        let page = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
-            .list(&SourceListRequest {
-                max_keys: 2,
-                ..Default::default()
-            })
-            .await
-            .expect("valid native page");
-        assert_eq!(page.objects.len(), 1);
-        assert_eq!(page.objects[0].key, "目录/空 & file");
-        assert_eq!(page.objects[0].size, 0);
-        assert_eq!(page.common_prefixes, ["目录/子/"]);
-        assert!(page.is_truncated);
-        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
-        assert_requests(&recorded, &[("GET", "/legacy?restype=container&comp=list&maxresults=2")]);
-    }
-
    #[test]
    fn blob_tags_parse_into_the_shared_tag_map() {
        let tags = parse_blob_tags(TAGS).expect("tags should parse");
@@ -1150,184 +991,6 @@ mod tests {
        ]
    }

-    #[tokio::test]
-    async fn object_not_found_requires_provider_evidence_or_one_successful_head_probe() {
-        for method in [Method::HEAD, Method::GET] {
-            for (status, code, expected) in [
-                (404, Some("BlobNotFound"), "not_found"),
-                (403, Some("BlobNotFound"), "access_denied"),
-                (404, Some("ContainerNotFound"), "other"),
-                (404, Some("BlobVersionNotFound"), "other"),
-                (404, Some("UnrecognizedError"), "other"),
-                (404, None, if method == Method::HEAD { "not_found" } else { "other" }),
-                (404, Some("ResourceNotFound"), if method == Method::HEAD { "not_found" } else { "other" }),
-            ] {
-                let probes = method == Method::HEAD && status == 404 && matches!(code, None | Some("ResourceNotFound"));
-                let headers = code
-                    .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())])
-                    .unwrap_or_default();
-                let mut responses = vec![ScriptedResponse::new(status, headers, "untrusted-error-body".to_string())];
-                if probes {
-                    responses.push(ScriptedResponse::new(200, Vec::new(), String::new()));
-                }
-                let (endpoint, recorded) = scripted_server(responses).await;
-                let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
-                let result = if method == Method::HEAD {
-                    backend.head("missing").await.map(|_| ())
-                } else {
-                    backend.get("missing", None).await.map(|_| ())
-                };
-                let err = result.expect_err("object error must remain an error");
-                assert_eq!(err.class_label(), expected, "{method} {status} {code:?}: {err:?}");
-                assert!(!err.is_retryable(), "{err:?}");
-                assert!(!err.to_string().contains("untrusted-error-body"));
-                let mut requests = vec![(method.as_str(), "/legacy/missing")];
-                if probes {
-                    requests.push(("HEAD", "/legacy?restype=container"));
-                }
-                assert_requests(&recorded, &requests);
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn s3_not_found_alias_never_proves_native_object_absence() {
-        for selector in [None, Some("versionid"), Some("snapshot")] {
-            for operation in ["head", "get", "list", "tags", "probe"] {
-                if selector.is_some() && !matches!(operation, "head" | "get") {
-                    continue;
-                }
-                for (status, expected, retryable) in [
-                    (403, "access_denied", false),
-                    (404, "other", false),
-                    (416, "other", false),
-                    (500, "server_error", true),
-                ] {
-                    let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
-                        status,
-                        vec![(HEADER_ERROR_CODE, "NoSuchKey".to_string())],
-                        "untrusted-error-body".to_string(),
-                    )])
-                    .await;
-                    let credential = selector.map_or_else(
-                        || Credential::SharedKey(vec![7_u8; 32]),
-                        |selector| Credential::Sas(vec![(selector.to_string(), "old-version".to_string())]),
-                    );
-                    let backend = backend(&endpoint, credential);
-                    let result = match operation {
-                        "head" => backend.head("missing").await.map(|_| ()),
-                        "get" => backend.get("missing", None).await.map(|_| ()),
-                        "list" => backend.list(&SourceListRequest::default()).await.map(|_| ()),
-                        "tags" => backend.tagging("missing").await.map(|_| ()),
-                        "probe" => backend.probe().await,
-                        _ => unreachable!(),
-                    };
-                    let err = result.expect_err("an S3 error alias is not Azure absence evidence");
-                    assert_eq!(err.class_label(), expected, "{operation} {selector:?} HTTP {status}: {err:?}");
-                    assert_eq!(err.is_retryable(), retryable, "{operation} {selector:?} HTTP {status}: {err:?}");
-                    if status == 500 {
-                        assert!(matches!(err, SourceError::ServerError(500)));
-                    }
-                    assert!(!err.to_string().contains("untrusted-error-body"));
-                    let (method, mut target) = match operation {
-                        "head" => ("HEAD", "/legacy/missing".to_string()),
-                        "get" => ("GET", "/legacy/missing".to_string()),
-                        "list" => ("GET", "/legacy?restype=container&comp=list".to_string()),
-                        "tags" => ("GET", "/legacy/missing?comp=tags".to_string()),
-                        "probe" => ("HEAD", "/legacy?restype=container".to_string()),
-                        _ => unreachable!(),
-                    };
-                    if let Some(selector) = selector {
-                        target.push_str(&format!("?{selector}=old-version"));
-                    }
-                    assert_requests(&recorded, &[(method, target.as_str())]);
-                }
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn ambiguous_head_preserves_the_container_probe_failure() {
-        for (status, expected, retryable) in [
-            (403, "access_denied", false),
-            (404, "other", false),
-            (429, "throttled", true),
-            (500, "server_error", true),
-            (503, "throttled", true),
-        ] {
-            let (endpoint, recorded) = scripted_server(vec![
-                ScriptedResponse::new(404, Vec::new(), String::new()),
-                // A BlobNotFound header on a container request cannot prove
-                // that the object is missing, regardless of this status.
-                ScriptedResponse::new(status, vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())], String::new()),
-            ])
-            .await;
-            let err = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]))
-                .head("missing")
-                .await
-                .expect_err("failed probe must not become object absence");
-            assert_eq!(err.class_label(), expected, "probe {status}: {err:?}");
-            assert_eq!(err.is_retryable(), retryable, "probe {status}: {err:?}");
-            if status == 500 {
-                assert!(matches!(err, SourceError::ServerError(500)));
-            }
-            assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("HEAD", "/legacy?restype=container")]);
-        }
-    }
-
-    #[tokio::test]
-    async fn version_and_snapshot_absence_are_not_missing_current_blobs() {
-        for selector in ["versionid", "snapshot"] {
-            for code in [None, Some("BlobNotFound"), Some("ResourceNotFound")] {
-                for method in [Method::HEAD, Method::GET] {
-                    let headers = code
-                        .map(|value| vec![(HEADER_ERROR_CODE, value.to_string())])
-                        .unwrap_or_default();
-                    let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(404, headers, String::new())]).await;
-                    let backend = backend(&endpoint, Credential::Sas(vec![(selector.to_string(), "old-version".to_string())]));
-                    let result = if method == Method::HEAD {
-                        backend.head("object").await.map(|_| ())
-                    } else {
-                        backend.get("object", None).await.map(|_| ())
-                    };
-                    let err = result.expect_err("missing selected version must remain a source error");
-                    assert!(matches!(err, SourceError::Other(_)), "{method} {selector} {code:?}: {err:?}");
-                    assert_requests(&recorded, &[(method.as_str(), &format!("/legacy/object?{selector}=old-version"))]);
-                }
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn blob_not_found_header_is_not_object_absence_for_list_or_tags() {
-        for tags in [false, true] {
-            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
-                404,
-                vec![(HEADER_ERROR_CODE, "BlobNotFound".to_string())],
-                String::new(),
-            )])
-            .await;
-            let backend = backend(&endpoint, Credential::SharedKey(vec![7_u8; 32]));
-            let result = if tags {
-                backend.tagging("missing").await.map(|_| ())
-            } else {
-                backend.list(&SourceListRequest::default()).await.map(|_| ())
-            };
-            assert!(matches!(result, Err(SourceError::Other(_))), "tags={tags}: {result:?}");
-            assert_requests(
-                &recorded,
-                &[(
-                    "GET",
-                    if tags {
-                        "/legacy/missing?comp=tags"
-                    } else {
-                        "/legacy?restype=container&comp=list"
-                    },
-                )],
-            );
-        }
-    }
-
    #[tokio::test]
    async fn azure_backend_satisfies_the_shared_backend_contract() {
        let mut ranged = contract_blob_headers();
@@ -1335,7 +998,7 @@ mod tests {
        // A HEAD reports the object size with no body, exactly as Azure does.
        let mut head_only = contract_blob_headers();
        head_only.push(("Content-Length", "5".to_string()));
-        let (endpoint, recorded) = scripted_server(vec![
+        let (endpoint, _) = scripted_server(vec![
            ScriptedResponse::new(200, head_only, String::new()),
            ScriptedResponse::new(200, contract_blob_headers(), "hello".to_string()),
            ScriptedResponse::new(206, ranged, "ell".to_string()),
@@ -1365,23 +1028,6 @@ mod tests {
            },
        )
        .await;
-        assert_requests(
-            &recorded,
-            &[
-                ("HEAD", "/legacy/dir/a.txt"),
-                ("GET", "/legacy/dir/a.txt"),
-                ("GET", "/legacy/dir/a.txt"),
-                ("GET", "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&maxresults=2"),
-                (
-                    "GET",
-                    "/legacy?restype=container&comp=list&prefix=dir%2F&delimiter=%2F&marker=cursor-1&maxresults=2",
-                ),
-                ("GET", "/legacy/dir/a.txt?comp=tags"),
-                ("HEAD", "/legacy?restype=container"),
-                ("HEAD", "/legacy/missing"),
-                ("HEAD", "/legacy/secret"),
-            ],
-        );
    }

    #[tokio::test]
@@ -27,7 +27,7 @@
 //! and whether the provider can resume a listing from a key.

 use super::source_client::{SourceBackend, SourceError, SourceListRequest};
-use super::storage_api::HTTPRangeSpec;
+use crate::storage_api_contracts::range::HTTPRangeSpec;
 use std::collections::HashMap;

 /// The single object every fixture serves.
@@ -45,12 +45,16 @@

 use super::pull::{EnqueueOutcome, PullReason, QueuedPullOutcome};
 use super::source_client::{SourceError, SourcePage};
-use super::storage_api::{
-    BUCKET_META_PREFIX, ECStore, HTTPPreconditions, NamespaceLocking as _, ObjectOperations as _, ObjectOptions,
-    RUSTFS_META_BUCKET, StorageError, WriteCompletion, get_lock_acquire_timeout, get_on_demand_migration_config_in,
-    local_node_name, read_config_with_metadata, save_config_with_opts,
-};
 use super::sys::{BucketOdmState, OnDemandMigrationSys};
+use crate::bucket::metadata_sys::bucket_metadata_sys_of;
+use crate::config::com::{read_config_with_metadata, save_config_with_opts};
+use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+use crate::error::Error as StorageError;
+use crate::object_api::ObjectOptions;
+use crate::runtime::sources::local_node_name;
+use crate::set_disk::get_lock_acquire_timeout;
+use crate::storage_api_contracts::{namespace::NamespaceLocking as _, object::HTTPPreconditions, object::ObjectOperations as _};
+use crate::store::ECStore;
 use async_trait::async_trait;
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
@@ -370,8 +374,6 @@ pub type PullReport = Option<super::pull::QueuedPullReport>;
 /// mock in unit tests. Production: [`BucketBackfillContext`].
 #[async_trait]
 pub trait BackfillContext: Send + Sync {
-    /// The bucket incarnation captured by this context.
-    fn incarnation_id(&self) -> Uuid;
    /// One source page in the local key namespace.
    async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError>;
    /// Whether the breaker admits source traffic right now.
@@ -413,10 +415,6 @@ impl BucketBackfillContext {

 #[async_trait]
 impl BackfillContext for BucketBackfillContext {
-    fn incarnation_id(&self) -> Uuid {
-        self.state.incarnation_id()
-    }
-
    async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError> {
        let client = self.state.client().map_err(|err| SourceError::Unsupported(err.to_string()))?;
        let started = Instant::now();
@@ -476,10 +474,12 @@ impl BackfillContext for BucketBackfillContext {
    }

    async fn config_updated_at(&self) -> Result<Option<OffsetDateTime>, StorageError> {
-        Ok(
-            super::config::decode_stored_config(get_on_demand_migration_config_in(&self.api, self.state.bucket()).await?)?
-                .map(|(_, updated_at)| updated_at),
-        )
+        let sys = bucket_metadata_sys_of(&self.api.ctx)?;
+        let guard = sys.read().await;
+        Ok(guard
+            .get_on_demand_migration_config(self.state.bucket())
+            .await?
+            .map(|(_, updated_at)| updated_at))
    }
 }

@@ -667,36 +667,8 @@ pub async fn read_checkpoint(api: &Arc<ECStore>, bucket: &str) -> Result<Option<
 async fn write_checkpoint(
    api: &Arc<ECStore>,
    bucket: &str,
-    incarnation_id: Uuid,
    checkpoint: &BackfillCheckpoint,
    expected_etag: Option<&str>,
-) -> Result<String, BackfillError> {
-    let api = Arc::clone(api);
-    let bucket = bucket.to_string();
-    let checkpoint = checkpoint.clone();
-    let expected_etag = expected_etag.map(str::to_string);
-    // The storage commit owns detached work. Keep its user-bucket fence alive
-    // even when a caller aborts its waiter before the erasure tail has drained.
-    tokio::spawn(async move {
-        let fence = api.acquire_bucket_incarnation_fence(&bucket, incarnation_id).await?;
-        let mut opts = ObjectOptions::default();
-        fence.attach_to_object_options(&mut opts);
-        let result = write_checkpoint_while_fenced(&api, &bucket, &checkpoint, expected_etag.as_deref(), opts).await;
-        drop(fence);
-        result
-    })
-    .await
-    .map_err(|err| StorageError::other(format!("backfill checkpoint task failed: {err}")))?
-}
-
-/// The caller holds the destination bucket's lifecycle fence through the CAS
-/// write and its read-back, including the drained erasure write tail.
-async fn write_checkpoint_while_fenced(
-    api: &Arc<ECStore>,
-    bucket: &str,
-    checkpoint: &BackfillCheckpoint,
-    expected_etag: Option<&str>,
-    mut opts: ObjectOptions,
 ) -> Result<String, BackfillError> {
    let data = checkpoint.to_json()?;
    let preconditions = match expected_etag {
@@ -709,9 +681,12 @@ async fn write_checkpoint_while_fenced(
            ..Default::default()
        },
    };
-    opts.max_parity = true;
-    opts.write_completion = WriteCompletion::TailDrained;
-    opts.http_preconditions = Some(preconditions);
+    let opts = ObjectOptions {
+        max_parity: true,
+        write_completion: crate::object_api::WriteCompletion::TailDrained,
+        http_preconditions: Some(preconditions),
+        ..Default::default()
+    };
    match save_config_with_opts(Arc::clone(api), &checkpoint_path(bucket), data, &opts).await {
        Ok(()) => {}
        Err(StorageError::PreconditionFailed) => return Err(BackfillError::Conflict(bucket.to_string())),
@@ -882,14 +857,7 @@ impl BackfillRunner {
            });
        }
        let checkpoint = BackfillCheckpoint::new(&request, config_updated_at, &self.node, now);
-        let etag = write_checkpoint(
-            &self.api,
-            bucket,
-            context.incarnation_id(),
-            &checkpoint,
-            stored.as_ref().map(|s| s.etag.as_str()),
-        )
-        .await?;
+        let etag = write_checkpoint(&self.api, bucket, &checkpoint, stored.as_ref().map(|s| s.etag.as_str())).await?;
        info!(
            event = EVENT_ODM_BACKFILL_STATE,
            component = LOG_COMPONENT_ECSTORE,
@@ -924,42 +892,30 @@ impl BackfillRunner {
            }
            return Ok(handle.snapshot.lock().clone());
        }
-        let incarnation_id = self.api.bucket_incarnation_id_from_disk(bucket).await?;
-        let lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?;
-        let api = Arc::clone(&self.api);
-        let bucket = bucket.to_string();
-        tokio::spawn(async move {
-            let _lock = lock;
-            let fence = api.acquire_bucket_incarnation_fence(&bucket, incarnation_id).await?;
-            let mut opts = ObjectOptions::default();
-            fence.attach_to_object_options(&mut opts);
-            let Some(stored) = read_checkpoint(&api, &bucket).await? else {
-                return Err(BackfillError::NotFound(bucket.to_string()));
-            };
-            if !stored.checkpoint.state.is_active() {
-                return Ok(stored.checkpoint);
-            }
-            let mut checkpoint = stored.checkpoint;
-            let now = OffsetDateTime::now_utc();
-            checkpoint.state = BackfillState::Cancelled;
-            checkpoint.updated_at = now;
-            write_checkpoint_while_fenced(&api, &bucket, &checkpoint, Some(&stored.etag), opts).await?;
-            info!(
-                event = EVENT_ODM_BACKFILL_STATE,
-                component = LOG_COMPONENT_ECSTORE,
-                subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
-                state = checkpoint.state.as_str(),
-                result = "cancelled",
-                bucket = %bucket,
-                job_id = %checkpoint.job_id,
-                owner = %checkpoint.owner.as_ref().map(|o| o.node.as_str()).unwrap_or_default(),
-                "On-demand migration backfill job cancelled remotely"
-            );
-            drop(fence);
-            Ok(checkpoint)
-        })
-        .await
-        .map_err(|err| StorageError::other(format!("backfill cancellation task failed: {err}")))?
+        let _lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?;
+        let Some(stored) = read_checkpoint(&self.api, bucket).await? else {
+            return Err(BackfillError::NotFound(bucket.to_string()));
+        };
+        if !stored.checkpoint.state.is_active() {
+            return Ok(stored.checkpoint);
+        }
+        let mut checkpoint = stored.checkpoint;
+        let now = OffsetDateTime::now_utc();
+        checkpoint.state = BackfillState::Cancelled;
+        checkpoint.updated_at = now;
+        write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
+        info!(
+            event = EVENT_ODM_BACKFILL_STATE,
+            component = LOG_COMPONENT_ECSTORE,
+            subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
+            state = checkpoint.state.as_str(),
+            result = "cancelled",
+            bucket = %bucket,
+            job_id = %checkpoint.job_id,
+            owner = %checkpoint.owner.as_ref().map(|o| o.node.as_str()).unwrap_or_default(),
+            "On-demand migration backfill job cancelled remotely"
+        );
+        Ok(checkpoint)
    }

    /// Latest checkpoint: the in-memory progress of a local job, else the
@@ -1052,7 +1008,7 @@ impl BackfillRunner {
            checkpoint.state = BackfillState::Cancelled;
            checkpoint.updated_at = now;
            checkpoint.record_failure("config_changed", None, now);
-            write_checkpoint(&self.api, bucket, context.incarnation_id(), &checkpoint, Some(&stored.etag)).await?;
+            write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
            info!(
                event = EVENT_ODM_BACKFILL_STATE,
                component = LOG_COMPONENT_ECSTORE,
@@ -1071,7 +1027,7 @@ impl BackfillRunner {
            node: self.node.clone(),
            lease_until: now + BACKFILL_LEASE,
        });
-        let etag = write_checkpoint(&self.api, bucket, context.incarnation_id(), &checkpoint, Some(&stored.etag)).await?;
+        let etag = write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
        warn!(
            event = EVENT_ODM_BACKFILL_LEASE_TAKEOVER,
            component = LOG_COMPONENT_ECSTORE,
@@ -1510,8 +1466,7 @@ impl Job {
                lease_until: now + BACKFILL_LEASE,
            });
        }
-        let etag =
-            write_checkpoint(&self.api, &self.bucket, self.context.incarnation_id(), &self.checkpoint, Some(&self.etag)).await?;
+        let etag = write_checkpoint(&self.api, &self.bucket, &self.checkpoint, Some(&self.etag)).await?;
        self.etag = etag;
        self.keys_since_save = 0;
        self.last_save = Instant::now();
@@ -1523,7 +1478,7 @@ impl Job {
 /// Spawns [`run_backfill_recovery_loop`] on the store's shutdown token;
 /// `false` (nothing spawned) when the store has no background token.
 pub fn spawn_backfill_recovery_loop(runner: Arc<BackfillRunner>) -> bool {
-    let Some(cancel) = runner.api.background_cancel_token() else {
+    let Some(cancel) = runner.api.ctx.background_cancel_token() else {
        return false;
    };
    tokio::spawn(run_backfill_recovery_loop(runner, cancel));
@@ -1551,13 +1506,10 @@ pub async fn run_backfill_recovery_loop(runner: Arc<BackfillRunner>, cancel: Can

 #[cfg(test)]
 mod tests {
-    use super::super::storage_api::test_support::{
-        BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations as _, PutObjectCommitBarrier, PutObjectCommitPause,
-        isolated_store_over_temp_disks,
-    };
    use super::*;
-    use crate::on_demand_migration::source_client::SourceObject;
-    use crate::on_demand_migration::sys::PullError;
+    use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
+    use crate::bucket::on_demand_migration::source_client::SourceObject;
+    use crate::bucket::on_demand_migration::sys::PullError;
    use std::collections::{BTreeSet, HashSet};
    use std::sync::atomic::AtomicBool;

@@ -1680,7 +1632,6 @@ mod tests {

    /// Scripted source + local store + queue with a controllable report path.
    struct MockContext {
-        incarnation_id: Mutex<Option<Uuid>>,
        objects: Vec<SourceObject>,
        page_size: usize,
        local: Mutex<HashMap<String, LocalBackfillObject>>,
@@ -1709,7 +1660,6 @@ mod tests {
                })
                .collect();
            Arc::new(Self {
-                incarnation_id: Mutex::new(None),
                objects,
                page_size,
                local: Mutex::new(HashMap::new()),
@@ -1744,10 +1694,6 @@ mod tests {

    #[async_trait]
    impl BackfillContext for MockContext {
-        fn incarnation_id(&self) -> Uuid {
-            self.incarnation_id.lock().expect("test bucket initialized")
-        }
-
        async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError> {
            if let Some(err) = self.list_error.lock().take() {
                return Err(err);
@@ -1840,17 +1786,12 @@ mod tests {
        context: Arc<MockContext>,
    ) -> (Vec<tempfile::TempDir>, Arc<ECStore>, Arc<BackfillRunner>) {
        let (dirs, store) = isolated_store_over_temp_disks().await;
-        super::super::storage_api::test_support::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
-        store
-            .make_bucket(bucket, &Default::default())
-            .await
-            .expect("create test bucket");
-        *context.incarnation_id.lock() = Some(
-            store
-                .bucket_incarnation_id_from_disk(bucket)
-                .await
-                .expect("test bucket identity"),
-        );
+        // The isolated store has no bucket metadata system; the checkpoint
+        // only needs the bucket's directory under the metadata volume.
+        for dir in &dirs {
+            std::fs::create_dir_all(dir.path().join(RUSTFS_META_BUCKET).join(BUCKET_META_PREFIX).join(bucket))
+                .expect("test bucket metadata directory");
+        }
        let runner = runner_on(node, bucket, context, Arc::clone(&store));
        (dirs, store, runner)
    }
@@ -1860,129 +1801,6 @@ mod tests {
        BackfillRunner::new(store, node, Arc::new(contexts))
    }

-    #[tokio::test]
-    async fn cancelled_checkpoint_waiter_keeps_bucket_fenced_until_commit_finishes() {
-        for (suffix, pause) in [
-            ("before", PutObjectCommitPause::BeforeQuotaRename),
-            ("after", PutObjectCommitPause::AfterRenameQuorum),
-        ] {
-            let bucket = format!("backfill-cancel-tail-{suffix}");
-            let context = MockContext::new(0, 1);
-            let (_dirs, store, _runner) = runner_with("node-a", &bucket, Arc::clone(&context)).await;
-            let original_incarnation = context.incarnation_id();
-            let checkpoint = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", ts(1_700_000_001));
-            let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, &checkpoint_path(&bucket), pause);
-            let writer_api = Arc::clone(&store);
-            let writer_bucket = bucket.clone();
-            let waiter = tokio::spawn(async move {
-                write_checkpoint(&writer_api, &writer_bucket, original_incarnation, &checkpoint, None).await
-            });
-            barrier.wait_until_paused().await;
-            waiter.abort();
-            assert!(waiter.await.expect_err("caller aborted").is_cancelled());
-
-            let lifecycle_lock = store
-                .new_ns_lock(&bucket, BUCKET_LIFECYCLE_LOCK_OBJECT)
-                .await
-                .expect("lifecycle lock");
-            {
-                let mut probe = Box::pin(lifecycle_lock.get_write_lock(Duration::from_secs(1)));
-                assert!(
-                    futures::poll!(probe.as_mut()).is_pending(),
-                    "lifecycle writer must first try to acquire the lock"
-                );
-                assert!(
-                    tokio::time::timeout(Duration::from_millis(100), probe.as_mut())
-                        .await
-                        .is_err(),
-                    "the checkpoint owner must retain the user bucket lifecycle read lock after caller cancellation"
-                );
-            }
-
-            let delete_api = Arc::clone(&store);
-            let delete_bucket = bucket.clone();
-            let mut deletion = tokio::spawn(async move { delete_api.delete_bucket(&delete_bucket, &Default::default()).await });
-            assert!(
-                tokio::time::timeout(Duration::from_millis(100), &mut deletion).await.is_err(),
-                "DeleteBucket must wait for the checkpoint owner after its caller aborts"
-            );
-            barrier.release();
-            tokio::time::timeout(Duration::from_secs(10), deletion)
-                .await
-                .expect("commit must drain and release its lifecycle guard")
-                .expect("delete task")
-                .expect("delete original bucket");
-            store
-                .make_bucket(&bucket, &Default::default())
-                .await
-                .expect("recreate bucket");
-            assert_ne!(
-                original_incarnation,
-                store.bucket_incarnation_id_from_disk(&bucket).await.expect("new identity")
-            );
-            assert!(
-                read_checkpoint(&store, &bucket)
-                    .await
-                    .expect("read recreated bucket")
-                    .is_none(),
-                "no old checkpoint may outlive bucket deletion"
-            );
-        }
-    }
-
-    #[tokio::test]
-    async fn stale_checkpoint_writer_cannot_resurrect_or_overwrite_a_recreated_bucket() {
-        let bucket = "backfill-incarnation";
-        let context = MockContext::new(0, 1);
-        let (_dirs, store, _runner) = runner_with("node-a", bucket, Arc::clone(&context)).await;
-        let old_incarnation = context.incarnation_id();
-        let old = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", ts(1_700_000_001));
-        let old_etag = write_checkpoint(&store, bucket, old_incarnation, &old, None)
-            .await
-            .expect("old checkpoint");
-
-        store
-            .delete_bucket(bucket, &Default::default())
-            .await
-            .expect("delete original bucket");
-        store.make_bucket(bucket, &Default::default()).await.expect("recreate bucket");
-        let current_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
-        assert_ne!(old_incarnation, current_incarnation);
-        assert!(
-            read_checkpoint(&store, bucket)
-                .await
-                .expect("read after recreation")
-                .is_none()
-        );
-        for expected_etag in [None, Some(old_etag.as_str())] {
-            let error = write_checkpoint(&store, bucket, old_incarnation, &old, expected_etag)
-                .await
-                .expect_err("stale writer rejected");
-            assert!(matches!(error, BackfillError::Storage(StorageError::BucketNotFound(_))));
-        }
-        assert!(
-            read_checkpoint(&store, bucket)
-                .await
-                .expect("stale writer left no checkpoint")
-                .is_none()
-        );
-
-        let current = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-b", ts(1_700_000_002));
-        let current_etag = write_checkpoint(&store, bucket, current_incarnation, &current, None)
-            .await
-            .expect("current checkpoint");
-        let error = write_checkpoint(&store, bucket, old_incarnation, &old, Some(&current_etag))
-            .await
-            .expect_err("old identity cannot overwrite a matching ETag");
-        assert!(matches!(error, BackfillError::Storage(StorageError::BucketNotFound(_))));
-        let stored = read_checkpoint(&store, bucket)
-            .await
-            .expect("read current checkpoint")
-            .expect("current checkpoint remains");
-        assert_eq!(stored.etag, current_etag);
-        assert_eq!(stored.checkpoint, current);
-    }
-
    #[tokio::test]
    async fn full_backfill_lists_pages_and_counts_every_key() {
        let bucket = "backfill-full";
@@ -2322,7 +2140,7 @@ mod tests {
            node: "node-a".to_string(),
            lease_until: now - Duration::from_secs(120),
        });
-        let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &crashed, None)
+        let etag = write_checkpoint(&store, bucket, &crashed, None)
            .await
            .expect("seed checkpoint");

@@ -2333,7 +2151,7 @@ mod tests {
            lease_until: now + Duration::from_secs(60),
        });
        live.updated_at = now;
-        let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &live, Some(&etag))
+        let etag = write_checkpoint(&store, bucket, &live, Some(&etag))
            .await
            .expect("live lease");
        assert_eq!(runner.recover_once().await.taken_over, 0, "unexpired lease must not be taken over");
@@ -2350,7 +2168,7 @@ mod tests {
            lease_until: now - Duration::from_secs(1),
        });
        expired.updated_at = now + Duration::from_millis(1);
-        write_checkpoint(&store, bucket, context.incarnation_id(), &expired, Some(&etag))
+        write_checkpoint(&store, bucket, &expired, Some(&etag))
            .await
            .expect("expire lease");
        let stats = runner.recover_once().await;
@@ -2389,7 +2207,7 @@ mod tests {
        crashed.continuation_token = Some("2".to_string());
        crashed.failed = 1;
        crashed.record_failure("local_write", Some("k/00002"), crashed_at);
-        write_checkpoint(&store, bucket, context.incarnation_id(), &crashed, None)
+        write_checkpoint(&store, bucket, &crashed, None)
            .await
            .expect("seed failed page with an expired lease");

@@ -2445,9 +2263,7 @@ mod tests {

        // Same node name, unexpired lease: only a restart can produce this.
        let own = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", now);
-        let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &own, None)
-            .await
-            .expect("seed");
+        let etag = write_checkpoint(&store, bucket, &own, None).await.expect("seed");
        assert_eq!(runner.recover_once().await.taken_over, 1, "own-node running job is reclaimed at once");
        runner.wait_until_idle(bucket).await;
        let cp = read_checkpoint(&store, bucket)
@@ -2465,7 +2281,7 @@ mod tests {
            node: "node-z".to_string(),
            lease_until: now - Duration::from_secs(1),
        });
-        write_checkpoint(&store, bucket, context.incarnation_id(), &stale, Some(&stored.etag))
+        write_checkpoint(&store, bucket, &stale, Some(&stored.etag))
            .await
            .expect("seed stale");
        let stats = runner.recover_once().await;
@@ -14,34 +14,16 @@

 //! Bucket-level On-Demand Migration configuration: wire model (JSON stored
 //! under `on-demand-migration.json`), pure validation, credential redaction,
-//! and persisted-config decoding (rustfs/backlog#2148).
+//! and the publish hook the runtime registers into (rustfs/backlog#2148).
 //!
 //! The persisted blob is not encrypted; it shares the trust boundary of
 //! `bucket-targets.json` and `tier-config.bin`.

 use serde::{Deserialize, Serialize};
 use std::fmt;
+use std::sync::OnceLock;
 use url::Url;

-/// Decode bytes only at the service boundary, preserving typed corruption errors.
-pub(super) fn decode_stored_config(
-    stored: Option<(Vec<u8>, time::OffsetDateTime)>,
-) -> Result<Option<(OnDemandMigrationConfig, time::OffsetDateTime)>, super::storage_api::StorageError> {
-    stored
-        .map(|(bytes, updated_at)| {
-            OnDemandMigrationConfig::from_json(&bytes)
-                .map(|config| (config, updated_at))
-                .map_err(super::storage_api::StorageError::other)
-        })
-        .transpose()
-}
-
-pub(crate) async fn get_config(
-    bucket: &str,
-) -> Result<Option<(OnDemandMigrationConfig, time::OffsetDateTime)>, super::storage_api::StorageError> {
-    decode_stored_config(super::storage_api::get_on_demand_migration_config(bucket).await?)
-}
-
 /// The only wire version this build reads and writes.
 pub const ON_DEMAND_MIGRATION_CONFIG_VERSION: u32 = 1;

@@ -106,12 +88,12 @@ pub struct SourceConfig {
    pub tls: TlsConfig,
    /// Required for [`Provider::Azure`] and rejected for every other
    /// provider.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
    pub azure: Option<AzureSourceConfig>,
    /// Required for [`Provider::GcsNative`] and rejected for every other
    /// provider. [`Provider::Gcs`] keeps using `credentials` because it
    /// speaks the S3 interoperability API.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
    pub gcs: Option<GcsSourceConfig>,
 }

@@ -804,14 +786,20 @@ impl EndpointKey {
    }
 }

+/// Signature of the runtime publish hook: called with the bucket name and
+/// its parsed config (`None` when absent, cleared, or unreadable) every time
+/// the bucket's metadata is installed into or removed from the cache.
+pub type ConfigPublishHook = Box<dyn Fn(&str, Option<&OnDemandMigrationConfig>) + Send + Sync>;
+
+/// Registration point for the runtime (`OnDemandMigrationSys`). Until it is
+/// set, metadata publishes are no-ops for ODM, so this crate carries no
+/// runtime dependency and the config layer stays inert.
+pub static ON_DEMAND_MIGRATION_CONFIG_HOOK: OnceLock<ConfigPublishHook> = OnceLock::new();
+
 #[cfg(test)]
 mod tests {
    use super::*;

-    mod before_native_sources {
-        include!("../../fixtures/on_demand_migration/source_config_e2a.rs");
-    }
-
    const FULL_JSON: &str = r#"{
  "version": 1,
  "enabled": true,
@@ -882,32 +870,6 @@ mod tests {
        assert_eq!(minimal.policy.source_timeout.first_byte_ms, 15_000);
    }

-    #[test]
-    fn s3_config_writes_remain_readable_by_the_strict_pre_native_reader() {
-        // FULL_JSON is the complete config fixture already present in e2a921bc.
-        for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] {
-            let mut old_wire: serde_json::Value = serde_json::from_str(FULL_JSON).expect("historical config fixture");
-            old_wire["source"]["provider"] = provider.into();
-            let config = OnDemandMigrationConfig::from_json(&serde_json::to_vec(&old_wire).expect("historical wire"))
-                .expect("current reader accepts the historical source");
-            let wire = config.to_json().expect("persist current config");
-            let actual: serde_json::Value = serde_json::from_slice(&wire).expect("persisted config JSON");
-            let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone())
-                .expect("an existing S3 source must remain readable by the strict e2a source consumer");
-            assert_eq!(serde_json::to_value(old_source).expect("old reader wire"), old_wire["source"]);
-            assert_eq!(actual, old_wire, "provider={provider}: no existing config field or value may change");
-
-            for field in ["azure", "gcs"] {
-                let mut rejected = old_wire["source"].clone();
-                rejected[field] = serde_json::Value::Null;
-                assert!(
-                    serde_json::from_value::<before_native_sources::SourceConfig>(rejected).is_err(),
-                    "the frozen old reader must reject {field}, even when null"
-                );
-            }
-        }
-    }
-
    #[test]
    fn unknown_fields_are_rejected_at_every_level() {
        for (label, json) in [
@@ -1110,19 +1072,6 @@ mod tests {
        for cfg in [azure_cfg(), gcs_native_cfg()] {
            let json = cfg.to_json().expect("config must serialize");
            assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg);
-            let wire: serde_json::Value = serde_json::from_slice(&json).expect("native config JSON");
-            let (present, absent, expected) = match cfg.source.provider {
-                Provider::Azure => ("azure", "gcs", serde_json::to_value(&cfg.source.azure).expect("Azure block")),
-                Provider::GcsNative => ("gcs", "azure", serde_json::to_value(&cfg.source.gcs).expect("GCS block")),
-                _ => unreachable!("native fixture"),
-            };
-            assert!(expected.is_object(), "native credentials must be present");
-            assert_eq!(wire["source"][present], expected);
-            assert!(wire["source"].get(absent).is_none());
-            assert!(
-                serde_json::from_value::<before_native_sources::SourceConfig>(wire["source"].clone()).is_err(),
-                "native providers still require upgraded readers"
-            );
        }
        // The wire labels are part of the admin contract.
        assert!(
@@ -1552,55 +1501,4 @@ mod tests {
        assert!(!rendered.contains("topsecret"), "{rendered}");
        assert!(!rendered.contains("SK"), "{rendered}");
    }
-    /// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
-    /// stored payload it cannot parse as a typed error, never as a default
-    /// and never as `ConfigNotFound`.
-    #[tokio::test]
-    async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
-        use super::super::storage_api::StorageError as Error;
-        use super::super::storage_api::test_support::{
-            BUCKET_ON_DEMAND_MIGRATION_CONFIG, BucketMetadata, BucketMetadataSys, isolated_store_over_temp_disks,
-        };
-        use std::sync::Arc;
-        const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
-
-        let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
-        let sys = BucketMetadataSys::new(ecstore);
-        let bucket = "odm-accessor";
-
-        sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
-        assert_eq!(
-            decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap()).unwrap(),
-            None
-        );
-
-        let mut corrupt = BucketMetadata::new(bucket);
-        corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
-        sys.set(bucket.to_string(), Arc::new(corrupt)).await;
-        let err = decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap())
-            .expect_err("corrupt config must not read as a default");
-        assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
-        let typed = match &err {
-            Error::Io(io) => io
-                .get_ref()
-                .and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
-            _ => None,
-        };
-        assert!(
-            matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
-            "typed parse error must survive the Result boundary, got: {err:?}"
-        );
-
-        let mut valid = BucketMetadata::new(bucket);
-        valid
-            .update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
-            .unwrap();
-        let stamped = valid.on_demand_migration_config_updated_at;
-        sys.set(bucket.to_string(), Arc::new(valid)).await;
-        let (config, updated_at) = decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap())
-            .unwrap()
-            .expect("stored config is returned");
-        assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
-        assert_eq!(updated_at, stamped);
-    }
 }
@@ -43,8 +43,8 @@ use super::source_client::{
    GcsSourceSpec, SourceBackend, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
    SourceTimeouts, range_header_value,
 };
-use super::storage_api::HTTPRangeSpec;
-use super::storage_api::remote_s3_client::RemoteS3ClientError;
+use crate::bucket::remote_s3_client::RemoteS3ClientError;
+use crate::storage_api_contracts::range::HTTPRangeSpec;
 use google_cloud_auth::credentials::service_account::{AccessSpecifier, Builder as ServiceAccountBuilder};
 use google_cloud_auth::credentials::{CacheableResource, Credentials};
 use http::{HeaderMap, HeaderValue, Method};
@@ -55,6 +55,10 @@ use url::Url;
 /// Read-only object scope: this backend never writes to the source.
 const READ_ONLY_SCOPE: &str = "https://www.googleapis.com/auth/devstorage.read_only";
 const METADATA_PREFIX: &str = "x-goog-meta-";
+/// GCS reports its error code in the response body, not a header; the shared
+/// transport takes a header name, so it is given one that never matches and
+/// classification falls back to the status.
+const NO_ERROR_CODE_HEADER: &str = "x-goog-unused-error-code";
 /// One `objects.list` page is small; refuse an unbounded document.
 const MAX_JSON_BYTES: usize = 8 * 1024 * 1024;

@@ -120,18 +124,6 @@ impl GcsNativeSourceBackend {
        Ok(request)
    }

-    async fn send_object(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
-        match self.http.send_object(request, None).await {
-            Err(SourceError::NotFound) => {
-                // An XML object URL also returns 404 when its bucket is gone.
-                // Reuse the read-only listing probe before caching a key miss.
-                self.probe().await?;
-                Err(SourceError::NotFound)
-            }
-            result => result,
-        }
-    }
-
    /// Shared mapping for the XML API's HEAD and GET responses.
    fn head_from_response(headers: &HeaderMap) -> Result<SourceHead, SourceError> {
        if header(headers, "x-goog-encryption-key-sha256").is_some() {
@@ -172,7 +164,7 @@ impl GcsNativeSourceBackend {
 impl SourceBackend for GcsNativeSourceBackend {
    async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
        let request = self.request(Method::HEAD, self.object_url(key)?, HeaderMap::new()).await?;
-        let response = self.send_object(request).await?;
+        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
        Self::head_from_response(response.headers())
    }

@@ -185,7 +177,7 @@ impl SourceBackend for GcsNativeSourceBackend {
            );
        }
        let request = self.request(Method::GET, self.object_url(key)?, headers).await?;
-        let response = self.send_object(request).await?;
+        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
        let head = Self::head_from_response(response.headers())?;
        let content_range = header(response.headers(), "content-range").map(str::to_string);
        Ok(SourceGet {
@@ -221,7 +213,7 @@ impl SourceBackend for GcsNativeSourceBackend {
        }

        let request = self.request(Method::GET, url, HeaderMap::new()).await?;
-        let response = self.http.send(request, None).await?;
+        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
        let body = read_text(response, MAX_JSON_BYTES).await?;
        parse_objects_list(&body)
    }
@@ -241,7 +233,7 @@ impl SourceBackend for GcsNativeSourceBackend {
        let mut url = self.objects_url()?;
        url.query_pairs_mut().append_pair("maxResults", "1");
        let request = self.request(Method::GET, url, HeaderMap::new()).await?;
-        let response = self.http.send(request, None).await?;
+        let response = self.http.send(request, NO_ERROR_CODE_HEADER).await?;
        read_text(response, MAX_JSON_BYTES)
            .await
            .and_then(|body| parse_objects_list(&body))?;
@@ -280,38 +272,28 @@ struct ListedObject {
 fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
    let listing: ObjectsList =
        serde_json::from_str(body).map_err(|err| SourceError::Other(format!("source listing is not valid JSON: {err}")))?;
-    if listing.prefixes.iter().any(|prefix| prefix.is_empty()) {
-        return Err(SourceError::Other("source listing prefix has no name".to_string()));
-    }
    let next_continuation_token = listing.next_page_token.filter(|token| !token.is_empty());
    let objects = listing
        .items
        .into_iter()
        .map(|item| {
-            if item.name.is_empty() {
-                return Err(SourceError::Other("source listing object has no name".to_string()));
-            }
-            let size = item
-                .size
-                .and_then(|size| size.parse::<u64>().ok())
-                .ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?;
            let etag = item
                .md5_hash
                .as_deref()
                .and_then(base64_md5_to_hex)
                .or_else(|| item.etag.map(|etag| etag.trim_matches('"').to_string()))
                .filter(|etag| !etag.is_empty());
-            Ok(SourceObject {
+            SourceObject {
                key: item.name,
                etag,
-                size,
+                size: item.size.and_then(|size| size.parse().ok()).unwrap_or(0),
                last_modified: item.updated.as_deref().and_then(parse_http_timestamp),
                storage_class: item.storage_class,
                // GCS never encodes a part count in a digest or an ETag.
                is_multipart_etag: false,
-            })
+            }
        })
-        .collect::<Result<_, SourceError>>()?;
+        .collect();

    Ok(SourcePage {
        objects,
@@ -324,8 +306,8 @@ fn parse_objects_list(body: &str) -> Result<SourcePage, SourceError> {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
-    use crate::on_demand_migration::test_http_fixture::{ScriptedResponse, assert_requests, scripted_server};
+    use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, assert_backend_contract};
+    use crate::bucket::on_demand_migration::test_http_fixture::{ScriptedResponse, scripted_server};
    use google_cloud_auth::credentials::anonymous::Builder as AnonymousBuilder;

    const LIST_PAGE_ONE: &str = r#"{
@@ -506,7 +488,6 @@ mod tests {
            // request; the probe is the next one on the wire.
            ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
            ScriptedResponse::new(404, Vec::new(), String::new()),
-            ScriptedResponse::new(200, Vec::new(), "{}".to_string()),
            ScriptedResponse::new(403, Vec::new(), String::new()),
        ])
        .await;
@@ -522,240 +503,4 @@ mod tests {
        )
        .await;
    }
-
-    #[tokio::test]
-    async fn listing_404_is_not_an_object_not_found() {
-        let (endpoint, _) = scripted_server(vec![ScriptedResponse::new(404, Vec::new(), String::new())]).await;
-        let err = backend(&endpoint)
-            .list(&SourceListRequest {
-                max_keys: 1,
-                ..Default::default()
-            })
-            .await
-            .expect_err("a failed bucket listing is not a per-object miss");
-        assert_eq!(err.class_label(), "other", "{err:?}");
-    }
-
-    #[tokio::test]
-    async fn object_404_requires_a_readable_source_bucket() {
-        for method in [Method::HEAD, Method::GET] {
-            for (probe_status, expected_class) in [
-                (200, "not_found"),
-                (404, "other"),
-                (403, "access_denied"),
-                (503, "throttled"),
-                (500, "server_error"),
-            ] {
-                let (endpoint, recorded) = scripted_server(vec![
-                    ScriptedResponse::new(404, Vec::new(), String::new()),
-                    ScriptedResponse::new(probe_status, Vec::new(), "{}".to_string()),
-                ])
-                .await;
-                let backend = backend(&endpoint);
-                let result = if method == Method::HEAD {
-                    backend.head("missing").await.map(|_| ())
-                } else {
-                    backend.get("missing", None).await.map(|_| ())
-                };
-                let error = result.expect_err("the object 404 must remain an error");
-                assert_eq!(error.class_label(), expected_class, "{method} with probe HTTP {probe_status}: {error:?}");
-                let recorded = recorded.lock().expect("recorder lock");
-                assert_eq!(recorded.len(), 2, "one bounded read-only probe per ambiguous object miss");
-                assert_eq!(recorded[0].method, method.as_str());
-                assert_eq!(recorded[1].method, "GET");
-                assert_eq!(recorded[1].target, "/storage/v1/b/legacy/o?maxResults=1");
-            }
-        }
-    }
-
-    #[tokio::test]
-    async fn native_listing_rejects_missing_or_invalid_required_object_fields() {
-        for entry in [
-            r#"{"size":"1"}"#,
-            r#"{"name":"","size":"1"}"#,
-            r#"{"name":"broken"}"#,
-            r#"{"name":"broken","size":null}"#,
-            r#"{"name":"broken","size":""}"#,
-            r#"{"name":"broken","size":"-1"}"#,
-            r#"{"name":"broken","size":"18446744073709551616"}"#,
-            r#"{"name":"broken","size":"not-a-size"}"#,
-            r#"{"name":"broken","size":1}"#,
-        ] {
-            let body = format!(r#"{{"items":[{{"name":"valid","size":"1"}},{entry}],"nextPageToken":"next"}}"#);
-            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body)]).await;
-            let err = backend(&endpoint)
-                .list(&SourceListRequest {
-                    prefix: Some("dir/"),
-                    delimiter: Some("/"),
-                    continuation_token: Some("opaque+/="),
-                    max_keys: 2,
-                    ..Default::default()
-                })
-                .await
-                .expect_err("malformed object must reject the complete native page");
-            assert!(matches!(err, SourceError::Other(_)), "{entry}: {err:?}");
-            assert!(!err.is_retryable());
-            assert_requests(
-                &recorded,
-                &[(
-                    "GET",
-                    "/storage/v1/b/legacy/o?prefix=dir%2F&delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2",
-                )],
-            );
-        }
-    }
-
-    #[tokio::test]
-    async fn native_listing_rejects_empty_prefix_entries() {
-        for body in [
-            r#"{"items":[{"name":"valid","size":"1"}],"prefixes":[""],"nextPageToken":"next"}"#,
-            r#"{"prefixes":[""],"nextPageToken":"next"}"#,
-            r#"{"prefixes":["目录/子/",""],"nextPageToken":"next"}"#,
-        ] {
-            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
-            let result = backend(&endpoint)
-                .list(&SourceListRequest {
-                    delimiter: Some("/"),
-                    continuation_token: Some("opaque+/="),
-                    max_keys: 2,
-                    ..Default::default()
-                })
-                .await;
-            let err = result.expect_err("an empty prefix must reject the entire page and its cursor");
-            assert!(matches!(err, SourceError::Other(_)), "{body}: {err:?}");
-            assert!(!err.is_retryable());
-            assert_requests(
-                &recorded,
-                &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&pageToken=opaque%2B%2F%3D&maxResults=2")],
-            );
-        }
-
-        let body = r#"{"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#;
-        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
-        let page = backend(&endpoint)
-            .list(&SourceListRequest {
-                delimiter: Some("/"),
-                max_keys: 1,
-                ..Default::default()
-            })
-            .await
-            .expect("a valid prefix-only page must remain usable");
-        assert!(page.objects.is_empty());
-        assert_eq!(page.common_prefixes, ["目录/子/"]);
-        assert!(page.is_truncated);
-        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
-        assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?delimiter=%2F&maxResults=1")]);
-    }
-
-    #[tokio::test]
-    async fn native_listing_preserves_zero_size_unicode_prefixes_and_opaque_cursors() {
-        let body = r#"{"items":[{"name":"目录/空 & file","size":"0"}],"prefixes":["目录/子/"],"nextPageToken":"opaque+/="}"#;
-        let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(200, Vec::new(), body.to_string())]).await;
-        let page = backend(&endpoint)
-            .list(&SourceListRequest {
-                max_keys: 2,
-                ..Default::default()
-            })
-            .await
-            .expect("valid native page");
-        assert_eq!(page.objects.len(), 1);
-        assert_eq!(page.objects[0].key, "目录/空 & file");
-        assert_eq!(page.objects[0].size, 0);
-        assert_eq!(page.common_prefixes, ["目录/子/"]);
-        assert!(page.is_truncated);
-        assert_eq!(page.next_continuation_token.as_deref(), Some("opaque+/="));
-        assert_requests(&recorded, &[("GET", "/storage/v1/b/legacy/o?maxResults=2")]);
-    }
-
-    #[tokio::test]
-    async fn missing_object_head_requires_one_successful_bucket_probe() {
-        for (status, body, expected, retryable) in [
-            (200, "{}", "not_found", false),
-            (403, "", "access_denied", false),
-            (404, "", "other", false),
-            (429, "", "throttled", true),
-            (500, "", "server_error", true),
-            (503, "", "throttled", true),
-            (200, "not JSON", "other", false),
-        ] {
-            let (endpoint, recorded) = scripted_server(vec![
-                ScriptedResponse::new(404, Vec::new(), String::new()),
-                ScriptedResponse::new(status, Vec::new(), body.to_string()),
-            ])
-            .await;
-            let err = backend(&endpoint).head("missing").await.expect_err("missing HEAD must fail");
-            assert_eq!(err.class_label(), expected, "probe {status} {body:?}: {err:?}");
-            assert_eq!(err.is_retryable(), retryable, "probe {status} {body:?}: {err:?}");
-            if status == 500 {
-                assert!(matches!(err, SourceError::ServerError(500)));
-            }
-            assert_requests(&recorded, &[("HEAD", "/legacy/missing"), ("GET", "/storage/v1/b/legacy/o?maxResults=1")]);
-        }
-    }
-
-    #[tokio::test]
-    async fn denied_object_reads_do_not_probe_or_become_object_absence() {
-        for method in [Method::HEAD, Method::GET] {
-            let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
-                403,
-                vec![("x-goog-unused-error-code", "NoSuchKey".to_string())],
-                "untrusted-error-body".to_string(),
-            )])
-            .await;
-            let backend = backend(&endpoint);
-            let result = if method == Method::HEAD {
-                backend.head("missing").await.map(|_| ())
-            } else {
-                backend.get("missing", None).await.map(|_| ())
-            };
-            let err = result.expect_err("denied object read must remain a failure");
-            assert_eq!(err.class_label(), "access_denied");
-            assert!(!err.is_retryable());
-            assert!(!err.to_string().contains("untrusted-error-body"));
-            assert_requests(&recorded, &[(method.as_str(), "/legacy/missing")]);
-        }
-    }
-    #[tokio::test]
-    async fn non_object_errors_ignore_untrusted_error_code_headers() {
-        for probe in [false, true] {
-            for (status, expected, retryable) in [(403, "access_denied", false), (500, "server_error", true)] {
-                let (endpoint, recorded) = scripted_server(vec![ScriptedResponse::new(
-                    status,
-                    vec![("x-goog-unused-error-code", "NoSuchKey".to_string())],
-                    "untrusted-error-body".to_string(),
-                )])
-                .await;
-                let backend = backend(&endpoint);
-                let result = if probe {
-                    backend.probe().await
-                } else {
-                    backend
-                        .list(&SourceListRequest {
-                            max_keys: 2,
-                            ..Default::default()
-                        })
-                        .await
-                        .map(|_| ())
-                };
-                let err = result.expect_err("a synthetic provider header cannot change the source status");
-                assert_eq!(err.class_label(), expected, "probe={probe} status={status}: {err:?}");
-                assert_eq!(err.is_retryable(), retryable);
-                assert!(!err.to_string().contains("untrusted-error-body"));
-                if status == 500 {
-                    assert!(matches!(err, SourceError::ServerError(500)));
-                }
-                assert_requests(
-                    &recorded,
-                    &[(
-                        "GET",
-                        if probe {
-                            "/storage/v1/b/legacy/o?maxResults=1"
-                        } else {
-                            "/storage/v1/b/legacy/o?maxResults=2"
-                        },
-                    )],
-                );
-            }
-        }
-    }
 }
@@ -94,16 +94,13 @@ pub struct MergePick {
 }

 /// The continuation-token envelope. Opaque to clients: it is serialized as
-/// JSON, optionally framed, then base64-encoded like a local marker.
+/// framed JSON and then base64-encoded by the same helper as a local marker.
 ///
 /// A `null` cursor with `done = false` means "list that side from the start";
 /// `done = true` means the side is finished and must not be listed again.
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct ListThroughToken {
-    /// Transport framing observed by the decoder, never an envelope field.
-    #[serde(skip)]
-    pub framed: bool,
    /// Envelope marker, always [`LIST_THROUGH_TOKEN_TAG`].
    pub t: String,
    pub v: u32,
@@ -130,7 +127,6 @@ pub struct ListThroughToken {
 impl ListThroughToken {
    fn new(local: SideCursor, source: SideCursor, last_key: Option<String>) -> Self {
        Self {
-            framed: false,
            t: LIST_THROUGH_TOKEN_TAG.to_string(),
            v: LIST_THROUGH_TOKEN_VERSION,
            local: local.token,
@@ -145,12 +141,7 @@ impl ListThroughToken {
    pub fn encode(&self) -> String {
        // The envelope is built here from owned strings, so serialization
        // cannot fail; the fallback keeps the signature infallible.
-        let json = serde_json::to_string(self).unwrap_or_default();
-        if self.framed {
-            format!("{LIST_THROUGH_TOKEN_PREFIX}{json}")
-        } else {
-            json
-        }
+        format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
    }
 }

@@ -174,30 +165,16 @@ pub enum ListThroughTokenError {

 /// Classifies an already base64-decoded continuation token.
 ///
-/// Framed envelopes and complete historical writer envelopes are merged tokens.
-/// Partial JSON-shaped keys remain local markers. A key identical to a complete
-/// historical envelope is inherently ambiguous and retains merged semantics.
-/// Recognized envelopes share the same version, count and field validation.
+/// Only a framed JSON object is read as a merged token;
+/// anything else is a local marker, so a bucket that turns `list_through` off
+/// keeps paginating with the tokens it handed out. A token that *is* an
+/// envelope but was tampered with (unknown version, unknown field, truncated
+/// JSON) is an error, never a silent fallback.
 pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
-    let (payload, framed) = match decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) {
-        Some(payload) => (payload, true),
-        None if decoded.starts_with('{') => (decoded, false),
-        None => return Ok(ListThroughCursor::Local(decoded.to_string())),
-    };
-    let value = match serde_json::from_str::<serde_json::Value>(payload) {
-        Ok(value) => value,
-        Err(_) if framed => return Err(ListThroughTokenError::Malformed),
-        Err(_) => return Ok(ListThroughCursor::Local(decoded.to_string())),
-    };
-    // RUSTFS_COMPAT_TODO(odm-list-bare-envelope): old writers issued bare JSON. Remove after all supported readers understand framing and outstanding bare listings have drained or explicitly restarted.
-    if !framed
-        && (value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG)
-            || ["v", "local", "local_done", "source", "source_done", "last_key"]
-                .iter()
-                .any(|field| value.get(field).is_none()))
-    {
+    let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
        return Ok(ListThroughCursor::Local(decoded.to_string()));
-    }
+    };
+    let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
    if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
        return Err(ListThroughTokenError::Malformed);
    }
@@ -221,10 +198,7 @@ pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, Lis
        None => return Err(ListThroughTokenError::Malformed),
    }
    serde_json::from_value::<ListThroughToken>(value)
-        .map(|mut token| {
-            token.framed = framed;
-            ListThroughCursor::Merged(Box::new(token))
-        })
+        .map(|token| ListThroughCursor::Merged(Box::new(token)))
        .map_err(|_| ListThroughTokenError::Malformed)
 }

@@ -823,7 +797,6 @@ mod tests {
    #[test]
    fn a_degraded_page_keeps_the_source_cursor_for_the_next_one() {
        let resume = ListThroughToken {
-            framed: false,
            t: LIST_THROUGH_TOKEN_TAG.to_string(),
            v: LIST_THROUGH_TOKEN_VERSION,
            local: Some("local-1".to_string()),
@@ -1060,7 +1033,7 @@ mod tests {

    #[test]
    fn token_round_trips_and_rejects_tampering() {
-        let mut token = ListThroughToken::new(
+        let token = ListThroughToken::new(
            SideCursor {
                token: Some("l".to_string()),
                done: false,
@@ -1068,7 +1041,6 @@ mod tests {
            SideCursor { token: None, done: true },
            Some("k".to_string()),
        );
-        token.framed = true;
        let encoded = token.encode();
        assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));

@@ -1122,103 +1094,22 @@ mod tests {
            token.encode(),
            r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
        );
-        for framed in [false, true] {
-            let prefix = if framed { LIST_THROUGH_TOKEN_PREFIX } else { "" };
-            for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
-                let mut token = progress_token(Some(count), true, false);
-                token.framed = framed;
-                assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
-            }
-            // Bare recognition requires the complete shape emitted by old writers;
-            // partial JSON objects are also valid local keys.
-            for version in [1, 2] {
-                for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
-                    let encoded = format!(
-                        r#"{prefix}{{"t":"odm-list","v":{version},"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":{value}}}"#
-                    );
-                    assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
-                }
-            }
-            for encoded in [
-                r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1}"#,
-                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#,
-                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"extra":true}"#,
-                r#"{"t":"odm-list","v":2,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key","no_progress":1,"framed":true}"#,
-            ] {
-                let encoded = format!("{prefix}{encoded}");
+        for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
+            let token = progress_token(Some(count), true, false);
+            assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
+        }
+        for version in [1, 2] {
+            for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
+                let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#);
                assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
            }
-            let bumped = format!("{prefix}{}", token.encode().replace("\"v\":1", "\"v\":9"));
-            assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(9)));
        }
-    }
-
-    // Frozen decoder from 447f3c704, before framing was introduced. Keeping this
-    // independent of the current decoder catches a default-writer rollout break.
-    fn decode_before_framing(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
-        if !decoded.starts_with('{') {
-            return Ok(ListThroughCursor::Local(decoded.to_string()));
-        }
-        let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
-            // Not JSON at all: an object key may legitimately start with '{'.
-            return Ok(ListThroughCursor::Local(decoded.to_string()));
-        };
-        if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
-            return Ok(ListThroughCursor::Local(decoded.to_string()));
-        }
-        match value.get("v").and_then(serde_json::Value::as_u64) {
-            Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
-                // v1 readers reject this field even when it is null or zero.
-                if value.get("no_progress").is_some() {
-                    return Err(ListThroughTokenError::Malformed);
-                }
-            }
-            Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
-                if !value
-                    .get("no_progress")
-                    .and_then(serde_json::Value::as_u64)
-                    .is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
-                {
-                    return Err(ListThroughTokenError::Malformed);
-                }
-            }
-            Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
-            None => return Err(ListThroughTokenError::Malformed),
-        }
-        serde_json::from_value::<ListThroughToken>(value)
-            .map(|token| ListThroughCursor::Merged(Box::new(token)))
-            .map_err(|_| ListThroughTokenError::Malformed)
-    }
-
-    #[test]
-    fn historical_writer_fixtures_and_default_output_remain_readable() {
-        for (wire, version, count) in [
-            (
-                r#"{"t":"odm-list","v":1,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k"}"#,
-                1,
-                None,
-            ),
-            (
-                r#"{"t":"odm-list","v":2,"local":"local-2","local_done":false,"source":"source-2","source_done":false,"last_key":"k","no_progress":15}"#,
-                2,
-                Some(15),
-            ),
+        for encoded in [
+            r#"{"t":"odm-list","v":1,"no_progress":1}"#,
+            r#"{"t":"odm-list","v":2}"#,
+            r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
        ] {
-            let ListThroughCursor::Merged(mut token) = decode_continuation_token(wire).expect("historical issued token") else {
-                panic!("a historical cursor must not silently become a local marker, even if a key has identical JSON");
-            };
-            assert_eq!(token.local.as_deref(), Some("local-2"));
-            assert_eq!(token.source.as_deref(), Some("source-2"));
-            assert_eq!(token.last_key.as_deref(), Some("k"));
-            assert_eq!(token.v, version);
-            assert_eq!(token.no_progress, count);
-            assert!(!token.framed);
-            assert_eq!(token.encode(), wire, "bare output retains the historical bytes");
-            assert_eq!(decode_before_framing(&token.encode()), Ok(ListThroughCursor::Merged(token.clone())));
-            token.framed = true;
-            let framed = format!("\0odm-list:{wire}");
-            assert_eq!(token.encode(), framed, "framing leaves the JSON payload unchanged");
-            assert_eq!(decode_continuation_token(&framed), Ok(ListThroughCursor::Merged(token)));
+            assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
        }
    }

@@ -30,16 +30,13 @@ mod backend_contract;
 pub mod backfill;
 pub mod breaker;
 pub mod config;
-#[cfg(feature = "gcs")]
 pub mod gcs;
 pub mod list_through;
-mod metrics;
 mod native_http;
 pub mod negative_cache;
 pub mod pull;
 pub mod source_client;
 pub mod stats;
-mod storage_api;
 pub mod sys;
 #[cfg(test)]
 mod test_http_fixture;
@@ -49,9 +46,9 @@ pub use breaker::{
    BreakerState, BreakerTransition, BreakerVerdict,
 };
 pub use config::{
-    AzureSourceConfig, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig,
-    OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig, SourceCredentials,
-    SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
+    AzureSourceConfig, ConfigPublishHook, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK,
+    ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider,
+    RangeGetPolicy, SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
 };
 pub use list_through::{
    FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
@@ -65,9 +62,6 @@ pub use pull::{
    PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody, WriteBackError,
    WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, idle_guarded_body,
 };
-pub use source_client::{
-    SourceClient, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceSse, is_multipart_etag,
-};
 pub use stats::{
    GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
    PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
@@ -77,7 +71,3 @@ pub use sys::{
    OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec,
    source_client_spec,
 };
-
-pub(crate) fn register_metrics() {
-    metrics::register();
-}
@@ -26,7 +26,7 @@
 //! the log line and the admin response.

 use super::source_client::{SourceError, SourceHead, SourceTimeouts, USER_AGENT_SUFFIX, classify_status, is_multipart_etag};
-use super::storage_api::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
+use crate::bucket::remote_s3_client::{RemoteS3ClientError, validate_remote_endpoint, validate_target_ca_pem};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_smithy_types::body::SdkBody;
 use futures::StreamExt;
@@ -99,7 +99,6 @@ impl NativeHttp {
    pub(super) fn for_test(endpoint: Url) -> Self {
        Self {
            client: reqwest::Client::builder()
-                .no_proxy()
                .redirect(reqwest::redirect::Policy::none())
                .build()
                .expect("test http client should build"),
@@ -123,66 +122,32 @@ impl NativeHttp {
    }

    /// Sends the request and returns the response only for a 2xx status.
-    /// Non-2xx statuses are classified from the status and an optional provider
+    /// Non-2xx statuses are classified from the status and the provider's own
    /// error-code header; response bodies are not read, so no provider message
    /// can smuggle credentials or markup into a log line.
    pub(super) async fn send(
        &self,
        request: reqwest::Request,
-        error_code_header: Option<&str>,
-    ) -> Result<reqwest::Response, SourceError> {
-        self.send_classified(request, error_code_header, false).await
-    }
-
-    #[cfg(feature = "gcs")]
-    pub(super) async fn send_object(
-        &self,
-        request: reqwest::Request,
-        error_code_header: Option<&str>,
-    ) -> Result<reqwest::Response, SourceError> {
-        self.send_classified(request, error_code_header, true).await
-    }
-
-    async fn send_classified(
-        &self,
-        request: reqwest::Request,
-        error_code_header: Option<&str>,
-        not_found_on_404_without_code: bool,
-    ) -> Result<reqwest::Response, SourceError> {
-        let response = self.execute(request).await?;
-        let status = response.status();
-        match Self::check_response(response, error_code_header) {
-            Err(SourceError::Other(_)) if not_found_on_404_without_code && status.as_u16() == 404 => Err(SourceError::NotFound),
-            result => result,
-        }
-    }
-
-    pub(super) async fn execute(&self, request: reqwest::Request) -> Result<reqwest::Response, SourceError> {
-        self.client.execute(request).await.map_err(classify_transport_error)
-    }
-
-    pub(super) fn check_response(
-        response: reqwest::Response,
-        error_code_header: Option<&str>,
+        error_code_header: &str,
    ) -> Result<reqwest::Response, SourceError> {
+        let response = self.client.execute(request).await.map_err(classify_transport_error)?;
        let status = response.status();
        if status.is_success() {
            return Ok(response);
        }
-        let code = error_code_header
-            .and_then(|header| response.headers().get(header))
+        let code = response
+            .headers()
+            .get(error_code_header)
            .and_then(|value| value.to_str().ok())
            .map(str::to_string);
-        let message = match &code {
-            Some(code) => format!("source returned HTTP {status} ({code})"),
-            None => format!("source returned HTTP {status}"),
-        };
-        match classify_status(status.as_u16(), code.as_deref(), message.clone()) {
-            // Native object absence needs provider-specific evidence or a
-            // successful bucket probe, never an alias from the S3 classifier.
-            SourceError::NotFound => Err(classify_status(status.as_u16(), None, message)),
-            error => Err(error),
-        }
+        Err(classify_status(
+            status.as_u16(),
+            None,
+            match &code {
+                Some(code) => format!("source returned HTTP {status} ({code})"),
+                None => format!("source returned HTTP {status}"),
+            },
+        ))
    }
 }

@@ -228,7 +193,6 @@ pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) ->
 /// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
 /// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
 /// an MD5.
-#[cfg(any(test, feature = "gcs"))]
 pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
    let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
    (raw.len() == 16).then(|| faster_hex::hex_string(&raw))
@@ -243,8 +243,6 @@ impl PullSource for SourceClient {
 #[derive(Clone, Debug)]
 pub struct WriteBackRequest {
    pub bucket: String,
-    /// Identity captured with the source configuration, retained through cleanup.
-    pub bucket_incarnation_id: uuid::Uuid,
    pub key: String,
    /// Source HEAD/GET of the whole object.
    pub head: SourceHead,
@@ -265,7 +263,6 @@ impl WriteBackRequest {
        let config = state.config();
        Self {
            bucket: state.bucket().to_string(),
-            bucket_incarnation_id: state.incarnation_id(),
            key: key.to_string(),
            head,
            source_label: format!("{}:{}", config.source.provider.as_str(), config.source.bucket),
@@ -360,7 +357,7 @@ pub trait OdmWriteBack: Send + Sync {
        parts: Vec<WriteBackPart>,
    ) -> Result<WriteBackOutcome, WriteBackError>;

-    async fn abort_multipart_upload(&self, request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError>;
+    async fn abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), WriteBackError>;
 }

 /// Why the pump stopped feeding the write-back before EOF.
@@ -665,7 +662,9 @@ async fn write_multipart(
        Err(err) => Err(err),
    };
    if completed.is_err()
-        && let Err(abort_err) = write_back.abort_multipart_upload(request, &upload_id).await
+        && let Err(abort_err) = write_back
+            .abort_multipart_upload(&request.bucket, &request.key, &upload_id)
+            .await
    {
        debug!(
            event = EVENT_ODM_PULL_FAILED,
@@ -1094,7 +1093,7 @@ impl OnDemandMigrationSys {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::on_demand_migration::config::{
+    use crate::bucket::on_demand_migration::config::{
        FilterConfig, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, PolicyConfig, Provider, SourceConfig,
        SourceCredentials, TlsConfig,
    };
@@ -1341,7 +1340,7 @@ mod tests {
            })
        }

-        async fn abort_multipart_upload(&self, _request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError> {
+        async fn abort_multipart_upload(&self, _bucket: &str, _key: &str, upload_id: &str) -> Result<(), WriteBackError> {
            self.aborted.lock().push(upload_id.to_string());
            Ok(())
        }
@@ -1500,7 +1499,7 @@ mod tests {
        assert_eq!(failures(&state).get("queue_full"), Some(&1));
        assert!(!queue.is_stopped());

-        assert_eq!(sys.remove(BUCKET), crate::on_demand_migration::ApplyOutcome::Removed);
+        assert_eq!(sys.remove(BUCKET), crate::bucket::on_demand_migration::ApplyOutcome::Removed);
        tokio::time::timeout(Duration::from_secs(5), queue.wait_until_stopped())
            .await
            .expect("dispatcher and in-flight job must exit after cancel");
@@ -26,13 +26,12 @@
 //! forwarded: v1 rejects SSE-C source objects outright.

 use super::azure::AzureSourceBackend;
-#[cfg(feature = "gcs")]
 use super::gcs::GcsNativeSourceBackend;
 use super::list_through::{ListPageError, validate_list_page};
-use super::storage_api::HTTPRangeSpec;
-use super::storage_api::remote_s3_client::{
+use crate::bucket::remote_s3_client::{
    PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
 };
+use crate::storage_api_contracts::range::HTTPRangeSpec;
 use aws_sdk_s3::Client as S3Client;
 use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
 use aws_sdk_s3::operation::get_object::GetObjectOutput;
@@ -335,7 +334,6 @@ const THROTTLE_CODES: &[&str] = &[
    "RequestLimitExceeded",
    "TooManyRequests",
    "RequestThrottled",
-    "ServerBusy",
 ];
 const NOT_FOUND_CODES: &[&str] = &["NoSuchKey"];
 const ACCESS_DENIED_CODES: &[&str] = &[
@@ -345,7 +343,6 @@ const ACCESS_DENIED_CODES: &[&str] = &[
    "AllAccessDisabled",
    "ExpiredToken",
    "InvalidToken",
-    "AuthorizationPermissionMismatch",
 ];

 pub(super) fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError {
@@ -723,9 +720,6 @@ impl SourceClient {
                )?;
                Ok(Self::from_backend(Box::new(backend), spec))
            }
-            #[cfg(not(feature = "gcs"))]
-            SourceBackendSpec::Gcs(_) => Err(RemoteS3ClientError::BackendNotCompiled("gcs_native")),
-            #[cfg(feature = "gcs")]
            SourceBackendSpec::Gcs(gcs) => {
                let backend = GcsNativeSourceBackend::new(
                    &spec.endpoint,
@@ -995,7 +989,7 @@ fn s3_source_object(object: SdkObject) -> Result<SourceObject, SourceError> {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract};
+    use crate::bucket::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract};
    use aws_smithy_runtime_api::client::http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn};
    use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
    use aws_smithy_runtime_api::client::result::ConnectorError;
@@ -1117,28 +1111,6 @@ mod tests {
        }
    }

-    #[cfg(not(feature = "gcs"))]
-    #[tokio::test]
-    async fn gcs_backend_not_compiled_keeps_hmac_s3_available() {
-        let mut native = spec(None);
-        native.provider = SourceProvider::GcsNative;
-        native.credentials = None;
-        native.backend = SourceBackendSpec::Gcs(GcsSourceSpec {
-            service_account_json: "{}".to_string(),
-        });
-        assert!(matches!(
-            SourceClient::new(&native).await,
-            Err(RemoteS3ClientError::BackendNotCompiled("gcs_native"))
-        ));
-
-        let mut hmac = spec(None);
-        hmac.provider = SourceProvider::Gcs;
-        hmac.endpoint = "https://storage.googleapis.com".to_string();
-        SourceClient::new(&hmac)
-            .await
-            .expect("GCS HMAC uses the always-available S3 backend");
-    }
-
    async fn scripted_client(spec: &SourceClientSpec, responses: Vec<Scripted>) -> (SourceClient, Recorded) {
        let requests: Recorded = Arc::new(Mutex::new(Vec::new()));
        let connector = SharedHttpConnector::new(ScriptedConnector {
@@ -1813,11 +1785,10 @@ mod tests {

    /// The S3 backend behind the scripted connector, without the prefix-mapping
    /// client on top: the contract is a property of the backend itself.
-    async fn scripted_s3_backend(responses: Vec<Scripted>) -> (S3SourceBackend, Recorded) {
+    async fn scripted_s3_backend(responses: Vec<Scripted>) -> S3SourceBackend {
        let spec = spec(None);
-        let requests: Recorded = Arc::new(Mutex::new(Vec::new()));
        let connector = SharedHttpConnector::new(ScriptedConnector {
-            requests: Arc::clone(&requests),
+            requests: Arc::new(Mutex::new(Vec::new())),
            responses: Arc::new(Mutex::new(responses.into_iter().collect())),
        });
        let http_client = http_client_fn(move |_settings, _components| connector.clone());
@@ -1827,20 +1798,17 @@ mod tests {
            .expect("test spec should build")
            .http_client(http_client)
            .interceptor(SourceProxyMarkerInterceptor::new());
-        (
-            S3SourceBackend {
-                client: S3Client::from_conf(config.build()),
-                bucket: spec.bucket.clone(),
-            },
-            requests,
-        )
+        S3SourceBackend {
+            client: S3Client::from_conf(config.build()),
+            bucket: spec.bucket.clone(),
+        }
    }

    #[tokio::test]
    async fn s3_backend_satisfies_the_shared_backend_contract() {
        let mut ranged = contract_object_headers(3);
        ranged.push(("content-range", "bytes 1-3/5".to_string()));
-        let (backend, requests) = scripted_s3_backend(vec![
+        let backend = scripted_s3_backend(vec![
            ok(contract_object_headers(5), ""),
            ok(contract_object_headers(5), "hello"),
            ok(ranged, "ell"),
@@ -1849,8 +1817,6 @@ mod tests {
            ok(Vec::new(), CONTRACT_TAGGING),
            ok(Vec::new(), ""),
            status(404, ""),
-            // An object HEAD 404 requires the existing S3 bucket HEAD probe.
-            ok(Vec::new(), ""),
            status(403, ACCESS_DENIED_BODY),
        ])
        .await;
@@ -1864,32 +1830,6 @@ mod tests {
            },
        )
        .await;
-        let requests = recorded(&requests);
-        let actual: Vec<_> = requests
-            .iter()
-            .map(|request| {
-                (
-                    request.method.as_str(),
-                    url::Url::parse(&request.uri).expect("recorded S3 URL").path().to_string(),
-                )
-            })
-            .collect();
-        let expected = [
-            ("HEAD", "/source-bucket/dir/a.txt"),
-            ("GET", "/source-bucket/dir/a.txt"),
-            ("GET", "/source-bucket/dir/a.txt"),
-            ("GET", "/source-bucket/"),
-            ("GET", "/source-bucket/"),
-            ("GET", "/source-bucket/dir/a.txt"),
-            ("HEAD", "/source-bucket/"),
-            ("HEAD", "/source-bucket/missing"),
-            ("HEAD", "/source-bucket/"),
-            ("HEAD", "/source-bucket/secret"),
-        ];
-        assert_eq!(actual, expected.map(|(method, path)| (method, path.to_string())));
-        for request in &requests {
-            assert_outbound_markers(request);
-        }
    }

    fn prefix_client(prefix: Option<String>) -> SourceClient {
@@ -19,12 +19,13 @@
 //! [`SourceClient`], a circuit breaker, a negative cache, a per-key
 //! singleflight table, a pull concurrency limit and counters. Its lifecycle
 //! follows the bucket metadata cache through the publish hook registered in
-//! [`BUCKET_CONFIG_PUBLISH_HOOK`]; the hook fires on every cache install
+//! [`ON_DEMAND_MIGRATION_CONFIG_HOOK`]; the hook fires on every cache install
 //! path (initial load, admin update, peer reload, refresh loop, lazy load).
 //!
 //! Change detection compares the config by value (`PartialEq`) rather than
-//! by `updated_at`. The bucket incarnation is part of this comparison:
-//! recreating a bucket must cancel old work even with identical configuration.
+//! by `updated_at`: the hook does not carry the timestamp, fetching it would
+//! re-enter the metadata system from inside its own publish path, and a
+//! byte-identical config never needs a new client anyway.
 //!
 //! Client construction is async (TLS material may be read from disk), so
 //! the hook does not build inline: `publish` removes state synchronously and
@@ -40,7 +41,9 @@

 use super::backfill::{PriorityPullPermits, PullPermit, PullPriority};
 use super::breaker::{Breaker, BreakerState, BreakerTransition, BreakerVerdict};
-use super::config::{OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig};
+use super::config::{
+    ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig,
+};
 use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter};
 use super::negative_cache::NegativeCache;
 use super::pull::{OdmWriteBack, PullQueue};
@@ -49,10 +52,9 @@ use super::source_client::{
    SourceTimeouts,
 };
 use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason};
-use super::storage_api::remote_s3_client::{
+use crate::bucket::remote_s3_client::{
    PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
 };
-use super::storage_api::{BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
 use parking_lot::{Mutex, RwLock};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -83,8 +85,6 @@ pub static GLOBAL_ON_DEMAND_MIGRATION_SYS: OnceLock<OnDemandMigrationSys> = Once
 /// `resolve` as [`OdmLookup::Unavailable`] and through status snapshots.
 #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
 pub enum OdmStateError {
-    #[error("the {0} backend is not included in this build")]
-    BackendNotCompiled(&'static str),
    /// `source.credentials` is `null`; the shared client builder has no
    /// anonymous mode yet (rustfs/backlog#2149 follow-up).
    #[error("anonymous source access is not supported yet; configure source credentials")]
@@ -269,7 +269,6 @@ impl Drop for InflightEntryGuard<'_> {
 /// config change (counters excepted), removed when the config goes away.
 pub struct BucketOdmState {
    bucket: String,
-    incarnation_id: uuid::Uuid,
    config: OnDemandMigrationConfig,
    applied_at: OffsetDateTime,
    endpoint_host: String,
@@ -307,24 +306,21 @@ impl BucketOdmState {
    async fn build(
        bucket: &str,
        config: &OnDemandMigrationConfig,
-        incarnation_id: uuid::Uuid,
        stats: Arc<OdmStats>,
        write_back: Option<Arc<dyn OdmWriteBack>>,
    ) -> Arc<Self> {
        let spec = source_client_spec(config);
-        let client = if config.source.credentials.is_none() && !config.source.provider.is_native() {
+        let client = if config.source.credentials.is_none() {
            Err(OdmStateError::AnonymousUnsupported)
        } else {
            SourceClient::new(&spec).await.map(Arc::new).map_err(|err| match err {
                RemoteS3ClientError::MissingCredentials => OdmStateError::AnonymousUnsupported,
-                RemoteS3ClientError::BackendNotCompiled(provider) => OdmStateError::BackendNotCompiled(provider),
                other => OdmStateError::ClientBuild(other.to_string()),
            })
        };
        let policy = &config.policy;
        Arc::new(Self {
            bucket: bucket.to_string(),
-            incarnation_id,
            endpoint_host: endpoint_host(&config.source),
            config: config.clone(),
            applied_at: OffsetDateTime::now_utc(),
@@ -342,18 +338,10 @@ impl BucketOdmState {
        })
    }

-    pub fn filter_incarnation(self: Arc<Self>, incarnation_id: uuid::Uuid) -> Option<Arc<Self>> {
-        (self.incarnation_id == incarnation_id && !self.is_cancelled()).then_some(self)
-    }
-
    pub fn bucket(&self) -> &str {
        &self.bucket
    }

-    pub fn incarnation_id(&self) -> uuid::Uuid {
-        self.incarnation_id
-    }
-
    pub fn config(&self) -> &OnDemandMigrationConfig {
        &self.config
    }
@@ -749,51 +737,21 @@ impl OnDemandMigrationSys {
    /// Registers `publish` as the bucket-metadata publish hook. Returns
    /// `false` when a hook was already registered.
    pub fn register_config_hook(&'static self) -> bool {
-        BUCKET_CONFIG_PUBLISH_HOOK
-            .set(Box::new(move |bucket, config_file, stored| {
-                if config_file == BUCKET_ON_DEMAND_MIGRATION_CONFIG {
-                    self.publish_stored(bucket, stored.map(|(bytes, _, incarnation)| (bytes, incarnation)));
-                }
-            }))
+        ON_DEMAND_MIGRATION_CONFIG_HOOK
+            .set(Box::new(move |bucket, config| self.publish(bucket, config)))
            .is_ok()
    }

-    /// Corrupt persisted bytes withdraw state synchronously, just like deletion.
-    fn publish_stored(&'static self, bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
-        let incarnation_id = stored.map(|(_, id)| id).unwrap_or_default();
-        match stored.map(|(bytes, _)| OnDemandMigrationConfig::from_json(bytes)).transpose() {
-            Ok(config) => self.publish_for_incarnation(bucket, incarnation_id, config.as_ref()),
-            Err(err) => {
-                warn!(
-                    event = EVENT_ODM_BUCKET_STATE_APPLIED,
-                    component = LOG_COMPONENT_ECSTORE,
-                    subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
-                    result = "invalid",
-                    bucket = %bucket,
-                    error = %err,
-                    "Failed to parse on-demand migration config"
-                );
-                self.publish_for_incarnation(bucket, incarnation_id, None);
-            }
-        }
-    }
-
    /// Hook entry point: removals apply immediately, installs are spawned
    /// (client construction is async). Requires a Tokio runtime for the
    /// install path; without one the config is logged and skipped.
-    pub fn publish_for_incarnation(
-        &'static self,
-        bucket: &str,
-        incarnation_id: uuid::Uuid,
-        config: Option<&OnDemandMigrationConfig>,
-    ) {
-        let config = self.desired(config).filter(|_| !incarnation_id.is_nil());
-        let generation = self.reserve_generation(bucket, config.is_some());
-        let Some(config) = config else {
+    pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) {
+        let generation = self.next_generation();
+        let Some(config) = self.desired(config) else {
            self.remove_with_generation(bucket, generation);
            return;
        };
-        if self.is_unchanged(bucket, incarnation_id, config, generation) {
+        if self.is_unchanged(bucket, config, generation) {
            return;
        }
        let Ok(handle) = tokio::runtime::Handle::try_current() else {
@@ -812,49 +770,31 @@ impl OnDemandMigrationSys {
        let bucket = bucket.to_string();
        let config = config.clone();
        handle.spawn(async move {
-            self.apply_with_generation(&bucket, incarnation_id, Some(&config), generation)
-                .await;
+            self.apply_with_generation(&bucket, Some(&config), generation).await;
        });
    }

    /// Installs, rebuilds, or removes the bucket state for `config`.
    /// Idempotent: the same config on an installed bucket is a no-op.
-    #[cfg(test)]
    pub async fn apply(&self, bucket: &str, config: Option<&OnDemandMigrationConfig>) -> ApplyOutcome {
-        self.apply_for_incarnation(bucket, uuid::Uuid::from_u128(1), config).await
-    }
-
-    #[cfg(test)]
-    pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) {
-        self.publish_for_incarnation(bucket, uuid::Uuid::from_u128(1), config);
-    }
-
-    pub async fn apply_for_incarnation(
-        &self,
-        bucket: &str,
-        incarnation_id: uuid::Uuid,
-        config: Option<&OnDemandMigrationConfig>,
-    ) -> ApplyOutcome {
-        let config = self.desired(config).filter(|_| !incarnation_id.is_nil());
-        let generation = self.reserve_generation(bucket, config.is_some());
-        self.apply_with_generation(bucket, incarnation_id, config, generation).await
+        let generation = self.next_generation();
+        self.apply_with_generation(bucket, config, generation).await
    }

    async fn apply_with_generation(
        &self,
        bucket: &str,
-        incarnation_id: uuid::Uuid,
        config: Option<&OnDemandMigrationConfig>,
        generation: u64,
    ) -> ApplyOutcome {
        let Some(config) = self.desired(config) else {
            return self.remove_with_generation(bucket, generation);
        };
-        if self.is_unchanged(bucket, incarnation_id, config, generation) {
+        if self.is_unchanged(bucket, config, generation) {
            return ApplyOutcome::Unchanged;
        }
        let stats = self.state(bucket).map(|state| Arc::clone(&state.stats)).unwrap_or_default();
-        let state = BucketOdmState::build(bucket, config, incarnation_id, stats, self.write_back()).await;
+        let state = BucketOdmState::build(bucket, config, stats, self.write_back()).await;

        let (outcome, previous) = {
            let mut buckets = self.buckets.write();
@@ -898,13 +838,12 @@ impl OnDemandMigrationSys {

    /// Removes a bucket's state (idempotent), cancelling its token.
    pub fn remove(&self, bucket: &str) -> ApplyOutcome {
-        let generation = self.reserve_generation(bucket, false);
+        let generation = self.next_generation();
        self.remove_with_generation(bucket, generation)
    }

    /// One-shot lookup: module switch, bucket state, prefix filter,
    /// client availability, negative cache, breaker, in that order.
-    #[cfg(test)]
    pub fn resolve(&self, bucket: &str, key: &str) -> Option<OdmLookup> {
        if !self.is_module_enabled() {
            return None;
@@ -912,13 +851,6 @@ impl OnDemandMigrationSys {
        self.state(bucket)?.resolve_key(key)
    }

-    pub fn resolve_for_incarnation(&self, bucket: &str, key: &str, incarnation_id: uuid::Uuid) -> Option<OdmLookup> {
-        if !self.is_module_enabled() {
-            return None;
-        }
-        self.state(bucket)?.filter_incarnation(incarnation_id)?.resolve_key(key)
-    }
-
    pub fn state(&self, bucket: &str) -> Option<Arc<BucketOdmState>> {
        self.buckets.read().get(bucket).and_then(|slot| slot.state.clone())
    }
@@ -947,17 +879,8 @@ impl OnDemandMigrationSys {
        snapshots
    }

-    fn reserve_generation(&self, bucket: &str, installing: bool) -> u64 {
-        // Reserve a desired install before its async client build, under the
-        // same lock that orders removals. Unconfigured buckets need no slot.
-        let mut buckets = self.buckets.write();
-        let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
-        if installing {
-            buckets.entry(bucket.to_string()).or_default().generation = generation;
-        } else if let Some(slot) = buckets.get_mut(bucket) {
-            slot.generation = generation;
-        }
-        generation
+    fn next_generation(&self) -> u64 {
+        self.generation.fetch_add(1, Ordering::Relaxed) + 1
    }

    fn desired<'c>(&self, config: Option<&'c OnDemandMigrationConfig>) -> Option<&'c OnDemandMigrationConfig> {
@@ -966,26 +889,15 @@ impl OnDemandMigrationSys {

    /// Claims `generation` for the bucket when the installed state already
    /// matches `config` and has a usable client.
-    fn is_unchanged(&self, bucket: &str, incarnation_id: uuid::Uuid, config: &OnDemandMigrationConfig, generation: u64) -> bool {
+    fn is_unchanged(&self, bucket: &str, config: &OnDemandMigrationConfig, generation: u64) -> bool {
        let mut buckets = self.buckets.write();
        let Some(slot) = buckets.get_mut(bucket) else {
            return false;
        };
-        if slot.generation > generation {
-            return false;
-        }
-        if slot
-            .state
-            .as_ref()
-            .is_some_and(|state| state.incarnation_id != incarnation_id)
-            && let Some(previous) = slot.state.take()
-        {
-            previous.cancel.cancel();
-        }
        let unchanged = slot
            .state
            .as_ref()
-            .is_some_and(|state| state.client.is_ok() && state.incarnation_id == incarnation_id && state.config == *config);
+            .is_some_and(|state| state.client.is_ok() && state.config == *config);
        if unchanged && slot.generation < generation {
            slot.generation = generation;
        }
@@ -1025,8 +937,8 @@ impl OnDemandMigrationSys {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::on_demand_migration::breaker::BREAKER_FAILURE_THRESHOLD;
-    use crate::on_demand_migration::config::{FilterConfig, PolicyConfig, SourceCredentials, SourceTimeout, TlsConfig};
+    use crate::bucket::on_demand_migration::breaker::BREAKER_FAILURE_THRESHOLD;
+    use crate::bucket::on_demand_migration::config::{FilterConfig, PolicyConfig, SourceCredentials, SourceTimeout, TlsConfig};
    use std::sync::atomic::AtomicUsize;
    use tokio::sync::Barrier;

@@ -1164,45 +1076,6 @@ mod tests {
        assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Rebuilt);
    }

-    #[tokio::test]
-    async fn native_azure_uses_provider_credentials_without_s3_credentials() {
-        let sys = enabled_sys();
-        let mut cfg = config(None);
-        cfg.source.provider = Provider::Azure;
-        cfg.source.endpoint = None;
-        cfg.source.credentials = None;
-        cfg.source.azure = Some(super::super::config::AzureSourceConfig {
-            account: "legacyaccount".to_string(),
-            account_key: Some("c2VjcmV0LWtleQ==".to_string()),
-            sas_token: None,
-        });
-        assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
-        let state = ready_state(sys.resolve("b", "k"));
-        assert!(state.client().is_ok(), "native credentials must not be classified as anonymous S3");
-    }
-
-    #[cfg(not(feature = "gcs"))]
-    #[tokio::test]
-    async fn gcs_backend_not_compiled_is_unavailable_not_anonymous() {
-        let sys = enabled_sys();
-        let mut cfg = config(None);
-        cfg.source.provider = Provider::GcsNative;
-        cfg.source.credentials = None;
-        cfg.source.gcs = Some(super::super::config::GcsSourceConfig {
-            service_account_json: "{}".to_string(),
-        });
-        let encoded = cfg.to_json().expect("GCS config is serializable without the backend");
-        let restored: OnDemandMigrationConfig = serde_json::from_slice(&encoded).expect("GCS config stays readable");
-        assert_eq!(restored, cfg);
-        assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
-        match sys.resolve("b", "k") {
-            Some(OdmLookup::Unavailable { error, .. }) => {
-                assert_eq!(error, OdmStateError::BackendNotCompiled("gcs_native"));
-            }
-            other => panic!("expected unavailable backend, got {other:?}"),
-        }
-    }
-
    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
    async fn singleflight_admits_one_leader_per_key() {
        let sys = enabled_sys();
@@ -1418,128 +1291,24 @@ mod tests {
        assert!(state.is_cancelled());
    }

-    #[tokio::test]
-    async fn identical_config_on_recreated_bucket_cancels_old_state() {
-        let sys = enabled_sys();
-        let cfg = config(None);
-        let old_id = uuid::Uuid::new_v4();
-        let new_id = uuid::Uuid::new_v4();
-        sys.apply_for_incarnation("recreated", old_id, Some(&cfg)).await;
-        let old = sys.state("recreated").expect("old state installed");
-        assert!(sys.resolve_for_incarnation("recreated", "key", new_id).is_none());
-        sys.apply_for_incarnation("recreated", new_id, Some(&cfg)).await;
-        let replacement = sys.state("recreated").expect("replacement state installed");
-        assert!(old.is_cancelled());
-        assert!(!Arc::ptr_eq(&old, &replacement));
-        assert_eq!(replacement.incarnation_id(), new_id);
-        assert!(sys.resolve_for_incarnation("recreated", "key", old_id).is_none());
-        assert!(sys.resolve_for_incarnation("recreated", "key", new_id).is_some());
-    }
-
-    #[tokio::test]
-    async fn changed_delete_marker_policy_withdraws_the_captured_lookup() {
-        let sys = enabled_sys();
-        let incarnation = uuid::Uuid::new_v4();
-        let mut cfg = config(None);
-        cfg.policy.respect_local_delete_marker = false;
-        sys.apply_for_incarnation("policy-snapshot", incarnation, Some(&cfg)).await;
-        let captured = sys.state("policy-snapshot").expect("policy A installed");
-        assert!(!captured.config().policy.respect_local_delete_marker);
-
-        cfg.policy.respect_local_delete_marker = true;
-        sys.apply_for_incarnation("policy-snapshot", incarnation, Some(&cfg)).await;
-        let replacement = sys.state("policy-snapshot").expect("policy B installed");
-        assert!(replacement.config().policy.respect_local_delete_marker);
-        assert!(captured.is_cancelled());
-        assert!(
-            captured
-                .filter_incarnation(incarnation)
-                .and_then(|state| state.resolve_key("key"))
-                .is_none(),
-            "a request that evaluated policy A cannot continue through policy B"
-        );
-        assert!(
-            replacement
-                .clone()
-                .filter_incarnation(incarnation)
-                .and_then(|state| state.resolve_key("key"))
-                .is_some()
-        );
-        assert_eq!(
-            replacement
-                .stats()
-                .snapshot(replacement.breaker().state())
-                .source_latency
-                .count,
-            0
-        );
-    }
-
-    #[tokio::test]
-    async fn missing_incarnation_cannot_install_or_retain_source_state() {
-        let sys: &'static OnDemandMigrationSys = Box::leak(Box::new(enabled_sys()));
-        let cfg = config(None);
-        assert_eq!(
-            sys.apply_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg)).await,
-            ApplyOutcome::NotDesired
-        );
-        sys.publish_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg));
-        assert!(sys.state("missing").is_none());
-
-        sys.apply_for_incarnation("missing", uuid::Uuid::new_v4(), Some(&cfg)).await;
-        let state = sys.state("missing").expect("valid identity installed");
-        sys.publish_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg));
-        assert!(sys.state("missing").is_none());
-        assert!(state.is_cancelled());
-    }
-
-    #[tokio::test]
-    async fn corrupt_stored_config_withdraws_runtime_state() {
-        let sys: &'static OnDemandMigrationSys = Box::leak(Box::new(enabled_sys()));
-        let cfg = config(None);
-        assert_eq!(sys.apply("corrupt", Some(&cfg)).await, ApplyOutcome::Installed);
-        let state = sys.state("corrupt").expect("state installed");
-        sys.publish_stored("corrupt", Some((b"not-json", uuid::Uuid::from_u128(1))));
-        assert!(sys.state("corrupt").is_none(), "corruption cannot keep an older source active");
-        assert!(state.is_cancelled(), "corruption cancels in-flight work");
-    }
-
-    #[tokio::test]
-    async fn absent_config_updates_do_not_allocate_bucket_slots() {
-        let sys = enabled_sys();
-        for index in 0..1000 {
-            let bucket = format!("unconfigured-{index}");
-            assert_eq!(sys.apply(&bucket, None).await, ApplyOutcome::NotDesired);
-            assert_eq!(sys.remove(&bucket), ApplyOutcome::NotDesired);
-        }
-        assert!(sys.buckets.read().is_empty(), "unconfigured buckets must not accumulate tombstones");
-    }
-
    #[tokio::test]
    async fn stale_install_cannot_overwrite_a_later_removal() {
        let sys = enabled_sys();
        let cfg = config(None);
-        let older = sys.reserve_generation("b", true);
-        let newer = sys.reserve_generation("b", false);
+        let older = sys.next_generation();
+        let newer = sys.next_generation();
        assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::NotDesired);
-        assert_eq!(
-            sys.apply_with_generation("b", uuid::Uuid::from_u128(1), Some(&cfg), older)
-                .await,
-            ApplyOutcome::Superseded
-        );
-        assert!(sys.state("b").is_none(), "removal must supersede an in-flight first install");
+        // The removal above did not create a slot; simulate an install that
+        // started before it and finishes after.
+        sys.apply_with_generation("b", Some(&cfg), older).await;
+        assert!(sys.state("b").is_some(), "no slot yet, so the older install lands");

-        assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
        let installed = sys.state("b").unwrap();
-        let older = sys.reserve_generation("b", true);
-        let newer = sys.reserve_generation("b", false);
+        let older = sys.next_generation();
+        let newer = sys.next_generation();
        assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::Removed);
        assert!(installed.is_cancelled());
-        assert_eq!(
-            sys.apply_with_generation("b", uuid::Uuid::from_u128(1), Some(&cfg), older)
-                .await,
-            ApplyOutcome::Superseded
-        );
+        assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded);
        assert!(sys.state("b").is_none(), "the stale install is discarded");
    }

@@ -56,16 +56,6 @@ impl RecordedRequest {

 pub(super) type Recorder = Arc<Mutex<Vec<RecordedRequest>>>;

-/// Checks the full request sequence, including the absence of extra probes.
-pub(super) fn assert_requests(recorder: &Recorder, expected: &[(&str, &str)]) {
-    let recorded = recorder.lock().expect("recorder lock");
-    let actual: Vec<_> = recorded
-        .iter()
-        .map(|request| (request.method.as_str(), request.target.as_str()))
-        .collect();
-    assert_eq!(actual, expected, "unexpected native source request sequence");
-}
-
 /// Binds a loopback listener that answers `responses` in order and returns its
 /// origin plus the recorder. The task ends once the script is exhausted.
 pub(super) async fn scripted_server(responses: Vec<ScriptedResponse>) -> (Url, Recorder) {
@@ -180,8 +180,6 @@ impl RemoteS3EndpointSpec {

 #[derive(Debug, thiserror::Error)]
 pub enum RemoteS3ClientError {
-    #[error("the {0} backend is not included in this build")]
-    BackendNotCompiled(&'static str),
    #[error("remote endpoint requires credentials")]
    MissingCredentials,
    #[error("{0}")]
@@ -283,7 +281,9 @@ impl Intercept for UserAgentSuffixInterceptor {

 /// Builds the SDK config for `spec` without finalizing it, so callers can add
 /// interceptors or (in tests) swap the HTTP client before `build()`.
-pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
+pub(crate) async fn build_remote_s3_config(
+    spec: &RemoteS3EndpointSpec,
+) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
    let Some(credentials) = &spec.credentials else {
        return Err(RemoteS3ClientError::MissingCredentials);
    };
@@ -523,7 +523,7 @@ fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
    Ok(())
 }

-pub fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
+pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
    validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
 }

@@ -25,7 +25,6 @@ pub(crate) mod tier_probe_intent;
 pub mod warm_backend;
 pub mod warm_backend_aliyun;
 pub mod warm_backend_azure;
-#[cfg(feature = "gcs")]
 pub mod warm_backend_gcs;
 pub mod warm_backend_huaweicloud;
 pub mod warm_backend_minio;
@@ -19,14 +19,13 @@
 #![allow(clippy::all)]

 use crate::error::is_err_bucket_not_found;
-#[cfg(feature = "gcs")]
-use crate::services::tier::warm_backend_gcs::WarmBackendGCS;
 use crate::services::tier::{
    tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
    tier_config::{TierConfig, TierType},
    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
    warm_backend_aliyun::WarmBackendAliyun,
    warm_backend_azure::WarmBackendAzure,
+    warm_backend_gcs::WarmBackendGCS,
    warm_backend_huaweicloud::WarmBackendHuaweicloud,
    warm_backend_minio::WarmBackendMinIO,
    warm_backend_r2::WarmBackendR2,
@@ -913,15 +912,6 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
                });
            }
        }
-        #[cfg(not(feature = "gcs"))]
-        TierType::GCS => {
-            return Err(AdminError {
-                code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(),
-                message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(),
-                status_code: StatusCode::NOT_IMPLEMENTED,
-            });
-        }
-        #[cfg(feature = "gcs")]
        TierType::GCS => {
            if let Some(gcs_config) = tier.gcs.as_ref() {
                let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
@@ -1038,27 +1028,6 @@ mod tests {

    const PROBE_VERSION: &str = "remote-v2";

-    #[cfg(not(feature = "gcs"))]
-    #[tokio::test]
-    async fn gcs_backend_not_compiled_preserves_config() {
-        let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#;
-        let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend");
-        assert_eq!(tier.tier_type, TierType::GCS);
-        let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable");
-        let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips");
-        assert_eq!(restored.tier_type, TierType::GCS);
-        let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved");
-        assert_eq!(restored_gcs.bucket, "archive");
-        assert_eq!(restored_gcs.creds, "secret");
-        assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED");
-        let error = match new_warm_backend(&tier, false).await {
-            Ok(_) => panic!("an excluded GCS backend cannot be constructed"),
-            Err(error) => error,
-        };
-        assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code);
-        assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED);
-    }
-
    struct CountingBackend {
        put_result: fn() -> Result<String, std::io::Error>,
        removes: Arc<AtomicUsize>,
@@ -329,11 +329,11 @@ impl ECStore {
    /// reuse its result, which is sound because bucket deletion/recreation
    /// requires the lifecycle WRITE lock and therefore cannot have run while
    /// any read guard was continuously held.
-    pub async fn acquire_bucket_incarnation_fence(
+    pub(crate) async fn acquire_bucket_incarnation_fence(
        &self,
        bucket: &str,
        expected: uuid::Uuid,
-    ) -> Result<super::BucketIncarnationFenceGuard> {
+    ) -> Result<super::bucket_fence::BucketIncarnationFenceGuard> {
        let inner = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
        let pieces = super::bucket_fence::FencePieces {
            registry: self.bucket_fence_registry.clone(),
@@ -150,7 +150,7 @@ impl BucketFenceRegistry {
 /// A held bucket lifecycle read lock plus its registration in the fence
 /// registry. Dropping the guard deregisters it; the memo is cleared when the
 /// last guard for the bucket drops (or a lost lock is observed).
-pub struct BucketIncarnationFenceGuard {
+pub(crate) struct BucketIncarnationFenceGuard {
    inner: Option<NamespaceLockGuard>,
    registry: Arc<BucketFenceRegistry>,
    bucket: String,
@@ -158,14 +158,6 @@ pub struct BucketIncarnationFenceGuard {
 }

 impl BucketIncarnationFenceGuard {
-    /// Propagate lifecycle lock loss into the storage commit checks.
-    /// The caller still owns this guard until the complete write tail drains.
-    pub fn attach_to_object_options(&self, opts: &mut crate::object_api::ObjectOptions) {
-        if let Some(guard) = self.namespace_lock_guard() {
-            opts.add_bucket_lifecycle_lock_guard(guard);
-        }
-    }
-
    pub(crate) fn is_lock_lost(&self) -> bool {
        self.inner.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
    }
@@ -354,36 +346,6 @@ mod tests {
        first_pieces.abandon("b", first.token);
    }

-    #[tokio::test]
-    async fn checkpoint_options_inherit_bucket_fence_lock_loss() {
-        let lock = NamespaceLock::new("bucket-fence-options".to_string(), Arc::new(LocalClient::new()));
-        let inner = lock
-            .acquire_guard(&lock_request("options"))
-            .await
-            .expect("acquire")
-            .expect("quorum");
-        let pieces = FencePieces {
-            registry: Arc::default(),
-            inner,
-        };
-        let registration = pieces.enter("b");
-        let fence = pieces.into_guard("b", registration.token);
-        let mut opts = crate::object_api::ObjectOptions::default();
-        fence.attach_to_object_options(&mut opts);
-        let inherited = opts
-            .bucket_lifecycle_lock_fence
-            .as_ref()
-            .expect("checkpoint inherits lifecycle guard");
-        assert!(!inherited.is_lock_lost());
-        tokio::time::timeout(
-            Duration::from_secs(2),
-            fence.namespace_lock_guard().expect("held guard").lock_lost_notified(),
-        )
-        .await
-        .expect("distributed guard expires");
-        assert!(inherited.is_lock_lost(), "the actual pre-rename options must observe lifecycle lock loss");
-    }
-
    #[test]
    fn buckets_are_isolated() {
        let reg = BucketFenceRegistry::default();
@@ -353,11 +353,6 @@ async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken)
 }

 impl ECStore {
-    /// Shutdown token owned by this store instance.
-    pub fn background_cancel_token(&self) -> Option<CancellationToken> {
-        self.ctx.background_cancel_token()
-    }
-
    /// Validate topology and process storage-class overrides before any disk is opened.
    pub fn validate_startup_storage_class(endpoint_pools: &EndpointServerPools) -> Result<()> {
        let drive_counts = startup_pool_drive_counts(endpoint_pools);
@@ -417,7 +417,6 @@ const MAX_UPLOADS_LIST: usize = 10000;
 mod bucket;
 mod bucket_fence;
 pub(crate) use bucket::await_bucket_namespace_operation;
-pub use bucket_fence::BucketIncarnationFenceGuard;
 mod heal;
 mod heal_walk;
 pub use heal_walk::HealWalkVersion;
@@ -1 +1 @@
-{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
+{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
@@ -1 +1 @@
-{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
+{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
@@ -1 +1 @@
-{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
+{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
@@ -1,80 +0,0 @@
-// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01.
-// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted.
-use serde::{Deserialize, Serialize};
-use std::fmt;
-
-const REDACTED: &str = "REDACTED";
-
-/// The external S3-compatible source bucket.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct SourceConfig {
-    pub provider: Provider,
-    /// `http(s)://host[:port]` with no path or query. Optional only for
-    /// [`Provider::Aws`], where it is derived from `region`.
-    #[serde(default)]
-    pub endpoint: Option<String>,
-    pub region: String,
-    pub bucket: String,
-    #[serde(default)]
-    pub path_style: PathStyle,
-    /// `None` means anonymous access to a public source bucket.
-    #[serde(default)]
-    pub credentials: Option<SourceCredentials>,
-    #[serde(default)]
-    pub tls: TlsConfig,
-}
-
-/// Source vendor family. `azure` is deliberately absent from this version.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum Provider {
-    /// Generic S3-compatible endpoint.
-    S3,
-    Aws,
-    Minio,
-    Rustfs,
-    R2,
-    /// GCS XML interoperability API with HMAC keys.
-    Gcs,
-}
-
-/// Bucket addressing style. `auto` is resolved by the source client builder.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum PathStyle {
-    #[default]
-    Auto,
-    Path,
-    Virtual,
-}
-
-/// Static credentials for the source. `Debug` never prints the secret or
-/// the session token.
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct SourceCredentials {
-    pub access_key: String,
-    pub secret_key: String,
-    #[serde(default)]
-    pub session_token: Option<String>,
-}
-
-impl fmt::Debug for SourceCredentials {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("SourceCredentials")
-            .field("access_key", &self.access_key)
-            .field("secret_key", &REDACTED)
-            .field("session_token", &self.session_token.as_ref().map(|_| REDACTED))
-            .finish()
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct TlsConfig {
-    #[serde(default)]
-    pub skip_verify: bool,
-    #[serde(default)]
-    pub ca_cert_pem: Option<String>,
-}
@@ -17,7 +17,7 @@
 //! Wire types for `PUT`/`GET`/`DELETE /v3/on-demand-migration/{bucket}`,
 //! `GET .../status`, `POST .../backfill?op=start|cancel` and
 //! `GET .../backfill` (ODM-12), mirroring the server's config model
-//! (`rustfs/src/on_demand_migration/config.rs`) and handler
+//! (`crates/ecstore/src/bucket/on_demand_migration/config.rs`) and handler
 //! responses (`rustfs/src/admin/handlers/on_demand_migration.rs`). The SDK
 //! owns its own copies, madmin-go style; the fixtures under
 //! `fixtures/on_demand_migration/` are the contract both sides pin
@@ -85,10 +85,10 @@ pub struct OnDemandMigrationSource {
    #[serde(default)]
    pub tls: OnDemandMigrationTls,
    /// Required for `azure` and rejected for every other provider.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
    pub azure: Option<OnDemandMigrationAzure>,
    /// Required for `gcs_native` and rejected for every other provider.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
+    #[serde(default)]
    pub gcs: Option<OnDemandMigrationGcs>,
 }

@@ -651,10 +651,6 @@ mod tests {
    use super::*;
    use crate::test_support::TestServer;

-    mod before_native_sources {
-        include!("../fixtures/on_demand_migration/source_config_e2a.rs");
-    }
-
    const SET_REQUEST_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_request.json");
    const SET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_response.json");
    const GET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/get_response.json");
@@ -688,20 +684,6 @@ mod tests {
        assert_eq!(config.source.tls, OnDemandMigrationTls::default());
    }

-    #[test]
-    fn s3_admin_writes_remain_readable_by_the_strict_pre_native_server() {
-        for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] {
-            let historical = SET_REQUEST_FIXTURE.replace("\"provider\":\"minio\"", &format!("\"provider\":\"{provider}\""));
-            let config: OnDemandMigrationConfig = serde_json::from_str(&historical).expect("historical set request");
-            let wire = serde_json::to_string(&config).expect("current admin set request");
-            let actual: serde_json::Value = serde_json::from_str(&wire).expect("admin request JSON");
-            let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone())
-                .expect("the strict e2a server must accept an ordinary S3 source from the new admin client");
-            assert_eq!(serde_json::to_value(old_source).expect("old source wire"), actual["source"]);
-            assert_eq!(wire, historical.trim(), "provider={provider}: preserve the historical request bytes");
-        }
-    }
-
    #[test]
    fn set_response_fixture_round_trips_and_is_redacted() {
        let response: OnDemandMigrationSetResponse = round_trip(SET_RESPONSE_FIXTURE);
@@ -896,11 +878,11 @@ mod tests {
        for (label, json) in [
            (
                "azure",
-                r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"}}"#,
+                r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"},"gcs":null}"#,
            ),
            (
                "gcs_native",
-                r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
+                r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
            ),
        ] {
            let source: OnDemandMigrationSource = serde_json::from_str(json).unwrap_or_else(|err| panic!("{label}: {err}"));
@@ -909,16 +891,6 @@ mod tests {
                json,
                "{label} must reproduce the server wire shape byte for byte"
            );
-            let mut wire: serde_json::Value = serde_json::from_str(json).expect("native wire fixture");
-            assert!(
-                serde_json::from_value::<before_native_sources::SourceConfig>(wire.clone()).is_err(),
-                "native provider names and fields still require an upgraded server"
-            );
-            wire[if label == "azure" { "gcs" } else { "azure" }] = serde_json::Value::Null;
-            assert_eq!(
-                serde_json::from_value::<OnDemandMigrationSource>(wire).expect("the prior explicit-null wire still decodes"),
-                source
-            );
        }

        let azure = OnDemandMigrationAzure {
@@ -973,10 +945,6 @@ mod tests {
                .is_some_and(|auth| auth.starts_with("AWS4-HMAC-SHA256"))
        );
        assert_eq!(request.body, SET_REQUEST_FIXTURE.trim(), "the body is the canonical config document");
-        let body: serde_json::Value = serde_json::from_str(&request.body).expect("signed admin request JSON");
-        let old_source: before_native_sources::SourceConfig = serde_json::from_value(body["source"].clone())
-            .expect("the strict pre-native server must accept the actual signed PUT source");
-        assert_eq!(old_source.provider, before_native_sources::Provider::Minio);
    }

    #[tokio::test]
@@ -38,4 +38,3 @@ pub(crate) use storage_api::metrics::{
    obs_on_demand_migration_snapshot, obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
    obs_transition_state_handle,
 };
-pub use storage_api::register_on_demand_migration_metrics_source;
@@ -17,6 +17,13 @@ use std::time::Duration;

 pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
+use rustfs_ecstore::api::bucket::on_demand_migration::backfill::{
+    BackfillCheckpoint as SourceBackfillCheckpoint, global_backfill_runner as source_global_backfill_runner,
+};
+use rustfs_ecstore::api::bucket::on_demand_migration::{
+    BreakerState as SourceOdmBreakerState, OdmBucketSnapshot as SourceOdmBucketSnapshot,
+    OnDemandMigrationSys as SourceOnDemandMigrationSys,
+};
 use rustfs_ecstore::api::bucket::replication::{
    BucketReplicationStats as SourceBucketReplicationStats, DurableMrfBucketBacklog, DurableMrfTargetBacklog,
    MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog, durable_mrf_backlog_summary_snapshot,
@@ -37,7 +44,9 @@ pub(crate) use rustfs_ecstore::api::runtime::{
 pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
 use rustfs_storage_api as storage_contracts;

-use crate::metrics::collectors::{OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBucketStats};
+use crate::metrics::collectors::{
+    OdmBackfillBucketStats, OdmBackfillRuntimeStats, OnDemandMigrationBreakerState, OnDemandMigrationBucketStats,
+};

 #[derive(Debug, Clone, PartialEq)]
 pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
@@ -456,37 +465,70 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
    buckets
 }

-struct OnDemandMigrationMetricsSource {
-    snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
-    backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
-}
-
-static ON_DEMAND_MIGRATION_METRICS_SOURCE: std::sync::OnceLock<OnDemandMigrationMetricsSource> = std::sync::OnceLock::new();
-
-/// Register the application-owned ODM snapshots before starting the collector.
-pub fn register_on_demand_migration_metrics_source(
-    snapshot: fn() -> Vec<OnDemandMigrationBucketStats>,
-    backfill_snapshot: fn() -> Vec<OdmBackfillBucketStats>,
-) -> bool {
-    ON_DEMAND_MIGRATION_METRICS_SOURCE
-        .set(OnDemandMigrationMetricsSource {
-            snapshot,
-            backfill_snapshot,
-        })
-        .is_ok()
+fn on_demand_migration_stats_from_snapshot(snapshot: SourceOdmBucketSnapshot) -> OnDemandMigrationBucketStats {
+    let stats = snapshot.stats;
+    OnDemandMigrationBucketStats {
+        bucket: snapshot.bucket,
+        requests_total: stats.requests_total,
+        pulled_bytes_total: stats.pulled_bytes_total,
+        pulled_objects_total: stats.pulled_objects_total,
+        pull_failures_total: stats.pull_failures_total,
+        inflight_pulls: stats.inflight_pulls,
+        queue_depth: stats.queue_depth,
+        source_latency_buckets: stats
+            .source_latency
+            .buckets
+            .into_iter()
+            .map(|bucket| (bucket.le_ms, bucket.count))
+            .collect(),
+        source_latency_count: stats.source_latency.count,
+        source_latency_sum_ms: stats.source_latency.sum_ms,
+        breaker_state: match stats.breaker_state {
+            SourceOdmBreakerState::Closed => OnDemandMigrationBreakerState::Closed,
+            SourceOdmBreakerState::HalfOpen => OnDemandMigrationBreakerState::HalfOpen,
+            SourceOdmBreakerState::Open => OnDemandMigrationBreakerState::Open,
+        },
+    }
 }

+/// Every bucket with live on-demand migration state on this node, sorted by
+/// name. Empty while the module switch is off.
 pub(crate) fn obs_on_demand_migration_snapshot() -> Vec<OnDemandMigrationBucketStats> {
-    ON_DEMAND_MIGRATION_METRICS_SOURCE
-        .get()
-        .map(|source| (source.snapshot)())
-        .unwrap_or_default()
+    SourceOnDemandMigrationSys::get()
+        .snapshot()
+        .into_iter()
+        .map(on_demand_migration_stats_from_snapshot)
+        .collect()
 }

+fn on_demand_migration_backfill_stats_from_checkpoint(
+    bucket: String,
+    checkpoint: SourceBackfillCheckpoint,
+) -> OdmBackfillBucketStats {
+    OdmBackfillBucketStats {
+        bucket,
+        state: checkpoint.state.as_str().to_string(),
+        listed: checkpoint.listed,
+        enqueued: checkpoint.enqueued,
+        pulled: checkpoint.pulled,
+        skipped_existing: checkpoint.skipped_existing,
+        failed: checkpoint.failed,
+        bytes: checkpoint.bytes,
+    }
+}
+
+/// Backfill jobs running on this node, sorted by bucket. Empty until the
+/// runner is installed, and empty again once a job finishes: the series are
+/// per-node job progress, not a cluster-wide history.
 pub(crate) fn obs_on_demand_migration_backfill_snapshot(server: String) -> OdmBackfillRuntimeStats {
-    let buckets = ON_DEMAND_MIGRATION_METRICS_SOURCE
-        .get()
-        .map(|source| (source.backfill_snapshot)())
+    let buckets = source_global_backfill_runner()
+        .map(|runner| {
+            runner
+                .local_job_snapshots()
+                .into_iter()
+                .map(|(bucket, checkpoint)| on_demand_migration_backfill_stats_from_checkpoint(bucket, checkpoint))
+                .collect()
+        })
        .unwrap_or_default();
    OdmBackfillRuntimeStats { server, buckets }
 }
@@ -538,31 +580,6 @@ pub(crate) async fn obs_replication_site_stats_snapshot(current_data_transfer_ra
 mod tests {
    use super::*;

-    #[test]
-    fn on_demand_migration_callbacks_supply_runtime_snapshots() {
-        assert!(register_on_demand_migration_metrics_source(
-            || vec![OnDemandMigrationBucketStats {
-                bucket: "configured".into(),
-                pulled_bytes_total: 4096,
-                ..Default::default()
-            }],
-            || vec![OdmBackfillBucketStats {
-                bucket: "backfill".into(),
-                pulled: 3,
-                ..Default::default()
-            }],
-        ));
-        let snapshot = obs_on_demand_migration_snapshot();
-        assert_eq!(snapshot.len(), 1);
-        assert_eq!(snapshot[0].bucket, "configured");
-        assert_eq!(snapshot[0].pulled_bytes_total, 4096);
-        let backfill = obs_on_demand_migration_backfill_snapshot("node-a".into());
-        assert_eq!(backfill.server, "node-a");
-        assert_eq!(backfill.buckets.len(), 1);
-        assert_eq!(backfill.buckets[0].bucket, "backfill");
-        assert_eq!(backfill.buckets[0].pulled, 3);
-    }
-
    #[test]
    fn obs_replication_numeric_conversions_floor_negative_values() {
        assert_eq!(i64_to_u64_floor_zero(-1), 0);
@@ -755,6 +772,51 @@ mod tests {
        assert_eq!(snapshot.mrf_last_flush_duration_millis, 4);
    }

+    #[test]
+    fn on_demand_migration_snapshot_projects_counters_and_breaker_state() {
+        // Built from JSON: the snapshot's timestamps use `time`, which obs does not depend on.
+        let snapshot: SourceOdmBucketSnapshot = serde_json::from_value(serde_json::json!({
+            "bucket": "photos",
+            "provider": "minio",
+            "endpoint_host": "source.example.com",
+            "applied_at": "2026-09-02T10:00:00Z",
+            "client_error": null,
+            "negative_cache_entries": 0,
+            "inflight_keys": 1,
+            "max_concurrent_pulls": 8,
+            "stats": {
+                "requests_total": {"get": {"source_hit": 2}},
+                "pulled_bytes_total": 4096,
+                "pulled_objects_total": {"inline": 1},
+                "pull_failures_total": {"source_timeout": 1},
+                "inflight_pulls": 1,
+                "queue_depth": 2,
+                "source_latency": {
+                    "buckets": [{"le_ms": 5, "count": 1}, {"le_ms": 10, "count": 2}],
+                    "count": 3,
+                    "sum_ms": 90753
+                },
+                "last_source_error": {"class": "server_error", "at": "2026-09-02T10:00:00Z"},
+                "breaker_state": "open"
+            }
+        }))
+        .expect("runtime snapshot decodes");
+
+        let stats = on_demand_migration_stats_from_snapshot(snapshot);
+
+        assert_eq!(stats.bucket, "photos");
+        assert_eq!(stats.requests_total["get"]["source_hit"], 2);
+        assert_eq!(stats.pulled_bytes_total, 4096);
+        assert_eq!(stats.pulled_objects_total["inline"], 1);
+        assert_eq!(stats.pull_failures_total["source_timeout"], 1);
+        assert_eq!(stats.inflight_pulls, 1);
+        assert_eq!(stats.queue_depth, 2);
+        assert_eq!(stats.source_latency_buckets, vec![(5, 1), (10, 2)]);
+        assert_eq!(stats.source_latency_count, 3);
+        assert_eq!(stats.source_latency_sum_ms, 90_753);
+        assert_eq!(stats.breaker_state, OnDemandMigrationBreakerState::Open);
+    }
+
    #[test]
    fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
        let snapshot = bucket_replication_stats_snapshot_from_parts(
@@ -13,8 +13,8 @@ Operator-facing behaviour, configuration, and troubleshooting for these services

 | Service | Desired source | Current-status inputs | Status surface | Side effects |
 |---|---|---|---|---|
-| Write-back pull pipeline (`rustfs/src/on_demand_migration/pull.rs`; the local write is delegated to the app layer in `rustfs/src/app/object/on_demand_migration_put.rs`) | The bucket's `on-demand-migration.json` (`enabled`, `policy.max_concurrent_pulls`, `pull_queue_capacity`, `multipart_part_size_bytes`, `bandwidth_limit_bytes_per_sec`) together with the process switch `RUSTFS_ON_DEMAND_MIGRATION_ENABLED` | Per-bucket runtime state in `rustfs/src/on_demand_migration/sys.rs`: whether a state is installed, whether its source client built, its cancellation token, queue depth, in-flight pull permits | `GET /rustfs/admin/v3/on-demand-migration/{bucket}/status` (`inflight_pulls`, `queue_depth`, `counters.pulled_*`, `counters.pull_failures_total`) and the `rustfs_on_demand_migration_*` series | Source GET/HEAD/GetObjectTagging traffic; local object writes through the internal put path, hence quota consumption, bucket default SSE, versioning, Object Lock defaults, `ObjectCreated` notifications, and outbound replication scheduling |
-| Backfill job (module under `rustfs/src/on_demand_migration/`, rustfs/backlog#2159) | An admin `start` request plus the bucket config; invalidated when the config's `updated_at` changes or the config is deleted | The persisted checkpoint under the bucket's metadata prefix, its `state` field, and the owner lease | The backfill section of the bucket status endpoint and the `rustfs_on_demand_migration_backfill_*` series | Source `ListObjectsV2` paging; queue admission into the write-back pipeline (and therefore all of its side effects); checkpoint writes |
-| Backfill recovery loop (registered from `rustfs/src/startup_background.rs`, rustfs/backlog#2159) | The set of persisted checkpoints in `state = running`; runs on every node | Checkpoint owner lease expiry | Takeover is reported through the same backfill status; a takeover emits a warn-level lease event | Claims the lease and resumes the backfill job, inheriting its side effects. Scanning checkpoints is read-only |
+| Write-back pull pipeline (`crates/ecstore/src/bucket/on_demand_migration/pull.rs`; the local write is delegated to the app layer in `rustfs/src/app/object/on_demand_migration_put.rs`) | The bucket's `on-demand-migration.json` (`enabled`, `policy.max_concurrent_pulls`, `pull_queue_capacity`, `multipart_part_size_bytes`, `bandwidth_limit_bytes_per_sec`) together with the process switch `RUSTFS_ON_DEMAND_MIGRATION_ENABLED` | Per-bucket runtime state in `crates/ecstore/src/bucket/on_demand_migration/sys.rs`: whether a state is installed, whether its source client built, its cancellation token, queue depth, in-flight pull permits | `GET /rustfs/admin/v3/on-demand-migration/{bucket}/status` (`inflight_pulls`, `queue_depth`, `counters.pulled_*`, `counters.pull_failures_total`) and the `rustfs_on_demand_migration_*` series | Source GET/HEAD/GetObjectTagging traffic; local object writes through the internal put path, hence quota consumption, bucket default SSE, versioning, Object Lock defaults, `ObjectCreated` notifications, and outbound replication scheduling |
+| Backfill job (module under `crates/ecstore/src/bucket/on_demand_migration/`, rustfs/backlog#2159 — not yet in the tree) | An admin `start` request plus the bucket config; invalidated when the config's `updated_at` changes or the config is deleted | The persisted checkpoint under the bucket's metadata prefix, its `state` field, and the owner lease | The backfill section of the bucket status endpoint and the `rustfs_on_demand_migration_backfill_*` series | Source `ListObjectsV2` paging; queue admission into the write-back pipeline (and therefore all of its side effects); checkpoint writes |
+| Backfill recovery loop (registered from `rustfs/src/startup_background.rs`, rustfs/backlog#2159 — not yet in the tree) | The set of persisted checkpoints in `state = running`; runs on every node | Checkpoint owner lease expiry | Takeover is reported through the same backfill status; a takeover emits a warn-level lease event | Claims the lease and resumes the backfill job, inheriting its side effects. Scanning checkpoints is read-only |

 The pull pipeline has no separate loop of its own: a bucket's queue dispatcher starts lazily on the first background pull and is cancelled when the bucket's state is rebuilt or removed, and each inline pull commits in a task that outlives its request so a client disconnect cannot truncate the stored object. Neither the switch nor the config is re-read by the workers: the bucket-metadata publish hook rebuilds the state, which is the only desired-state path.
@@ -11,7 +11,6 @@

 ## Open Items

- `odm-list-bare-envelope` historical ODM continuation tokens: preserve complete bare v1/v2 envelopes and default bare issuance while framed readers deploy. Remove the legacy classifier and default-off framing issuance gate only after every supported reader accepts framing and outstanding bare listings have drained or clients explicitly restarted them; tokens have no automatic expiry. Exact full-envelope object keys remain intrinsically ambiguous during this compatibility period.
 - `backlog-2263` legacy heal MRF inspection: retained per-record journals remain readable while committed-snapshot ownership and writer activation are staged. Remove legacy import only after all supported direct-upgrade and rollback readers understand committed snapshots and migration tooling confirms that no retained or restorable legacy journal requires it. This does not enable a new writer or change the automatic legacy consumer.
 - `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
 - `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.
@@ -84,41 +84,3 @@ The server-config model (`Config`, `KV`, `KVS`) and the global server-config sna
 ## Required Architecture Documents

 The guard requires the documents and section headings listed in its `require_source_contains` entries (`scripts/check_architecture_migration_rules.sh`); the directory index is [README.md](README.md).
-
-## On-Demand Migration Service
-
-Read-through, backfill and external pull orchestration belong in an application
-service under `rustfs/src/<service>/`. ECStore owns the storage primitives they
-need, including atomic commits, lifecycle locks and on-disk metadata. A service
-may use these primitives without moving its provider clients or scheduling
-policy into the engine.
-
-`rustfs/src/on_demand_migration/` owns source clients, pull scheduling, list
-merging, runtime state and backfill orchestration. Its `storage_api.rs` is the
-only ECStore facade boundary. Object write-back still enters the application's
-internal PUT and multipart use cases, including the atomic create-only commit,
-delete-marker protection, encryption, quota and notification rules.
-
-ECStore stores the existing ODM bytes and update timestamp without interpreting
-the JSON. Every metadata cache install or removal publishes those bytes through
-`BUCKET_CONFIG_PUBLISH_HOOK`; the application decodes them and synchronously
-withdraws corrupt configurations. Configuration writes validate structure and
-deployment constraints in the admin use case before the incarnation-fenced
-metadata update. Backfill reads metadata from its store's instance context and
-preserves the checkpoint ETag compare-and-set, lease and tail-drained writes.
-
-An ODM runtime is bound to the bucket incarnation published with its metadata,
-not just its name. Source reads and write-back reject a different incarnation.
-Checkpoint writes hold the user bucket's lifecycle fence through their complete
-commit and read-back, even if their caller stops waiting; the storage commit
-also observes lock loss. Deleting and recreating a bucket must not let work for
-its previous incarnation repopulate objects or checkpoints.
-
-Observability owns its metric DTOs and accepts application snapshot callbacks;
-it does not depend on the ODM runtime. The application registers both bucket
-and backfill snapshots during startup, before metadata and metric collection.
-
-This boundary does not change `.metadata.bin`, the ODM wire format or the
-backfill checkpoint format. An older binary may still discard unknown metadata
-fields when it rewrites a bucket; service relocation does not make mixed-version
-configuration writes or rollback preserve ODM configuration.
@@ -63,10 +63,3 @@ Lifecycle, replication, and `SetDisks` split blockers, extracted contracts, and
 4. Do not replace `SetDisks` with multiple runtime structs in one change; move one operation family only after contracts and focused tests exist.
 5. Remove or narrow one facade group per change so rollback preserves object IO, quorum, lifecycle/replication queues, scanner repair, notification/audit events, and metadata compatibility.
 6. Keep `api::bucket`, `api::config`, `api::disk`, and `api::tier` on explicit submodules and symbol lists; do not restore `pub use crate::<owner>::{...}` whole-module passthroughs for those groups.
-
-### On-Demand Migration
-
-`rustfs/src/on_demand_migration/storage_api.rs` owns the service's storage facade
-imports: opaque bucket configuration, shared remote S3 client construction,
-namespace locking, object options and metadata-object persistence. ODM types
-are owned by the application and are no longer exported through ECStore.
@@ -1,7 +1,7 @@
 # Remote Credential Sealing ADR

 **Use this when:** you add, read, or persist a stored remote credential — a replication target, a remote tier, or an on-demand migration source — or you need the sealed-envelope format, the mixed-version rules, or the reason this is worth doing in one deployment and not in another.
-**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `rustfs/src/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).
+**Source of truth:** the three stores that hold remote credentials — `BUCKET_TARGETS_FILE` and `BUCKET_ON_DEMAND_MIGRATION_CONFIG` in `crates/ecstore/src/bucket/metadata.rs`, `TIER_CONFIG_FILE` in `crates/ecstore/src/services/tier/tier.rs` — the shared envelope in `crates/ecstore/src/bucket/sealed_credentials.rs`, the consumers `crates/ecstore/src/bucket/bucket_target_sys.rs`, `crates/ecstore/src/services/tier/tier_config.rs` and `crates/ecstore/src/bucket/on_demand_migration/config.rs`, and the backend properties in [../operations/kms-backend-security.md](../operations/kms-backend-security.md).

 ## Recommendation

@@ -37,7 +37,7 @@ Two of the three are not files at all. `bucket-targets.json` and `on-demand-migr
 | Store | Reached as | Actually persisted at | Written by | Container |
 |---|---|---|---|---|
 | Replication and ILM targets | `BUCKET_TARGETS_FILE` | `BucketMetadata::bucket_targets_config_json`, msgpack field `BucketTargetsConfigJSON` | `BucketMetadata::update_config`, then `BucketMetadata::save_with_store`; `crates/ecstore/src/bucket/metadata_sys.rs` serializes the update under a transaction lock | `{BUCKET_META_PREFIX}/{bucket}/{BUCKET_METADATA_FILE}` in `RUSTFS_META_BUCKET` (`crates/ecstore/src/disk/mod.rs`) |
-| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; the application validates structure and deployment constraints before persistence | same blob as above |
+| On-demand migration source | `BUCKET_ON_DEMAND_MIGRATION_CONFIG` | `BucketMetadata::on_demand_migration_config_json`, msgpack field `OnDemandMigrationConfigJSON` | same path; `update_config` additionally refuses a blob this build cannot parse | same blob as above |
 | Remote tiers | `TIER_CONFIG_FILE` | its own object, a four-byte `TIER_CONFIG_FORMAT` / `TIER_CONFIG_VERSION` header followed by an `rmp_serde` payload of `ExternalTierConfigMgr` | `TierConfigMgr` through `encode_external_tiering_config_blob`, under `tier_config_lock_path` | `tier_config_path` under `CONFIG_PREFIX` in `RUSTFS_META_BUCKET` |

 The consequence of the first two sharing a blob is that any change to how that blob parses has a blast radius covering policy, lifecycle, versioning, object lock and everything else in `BucketMetadata` — not just credentials.
@@ -48,7 +48,7 @@ Three things hold the line today, and all three keep working whether or not seal

 - **The reserved bucket.** `RUSTFS_META_BUCKET` is `.rustfs.sys`; `is_reserved_or_invalid_bucket` keeps it off the S3 surface, and the admin inspect archive in `rustfs/src/admin/handlers/inspect_archive.rs` runs its request through a strict bucket-name check that a dot-prefixed reserved name does not pass.
 - **Admin authorization** on every route that can read or write one of the three configurations.
- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `rustfs/src/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.
+- **Redaction on every read path.** `BucketTarget::redacted_credentials` and the `Debug` for `Credentials` in `crates/ecstore/src/bucket/target/bucket_target.rs`, used by the remote-target listing in `rustfs/src/admin/handlers/replication.rs` and by the bucket-metadata export in `rustfs/src/admin/handlers/bucket_meta.rs`; `TierConfig::redacted` in `crates/ecstore/src/services/tier/tier_config.rs`, which is also what that type's `Clone` and `Debug` do; and `SourceCredentials::redacted` in `crates/ecstore/src/bucket/on_demand_migration/config.rs`, used by `rustfs/src/admin/handlers/on_demand_migration.rs`.

 So no API returns a stored secret. The bytes are reachable by reading the drives, and that is the boundary sealing is proposed to move.

@@ -74,7 +74,7 @@ The envelope deliberately does **not** carry its own scope. A scope read out of

 ## Why a hook instead of a dependency

-`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `BUCKET_CONFIG_PUBLISH_HOOK` in `crates/ecstore/src/bucket/metadata_sys.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.
+`crates/ecstore/Cargo.toml` has no `rustfs-kms` dependency, and adding one would invert the crate layering described in [crate-boundaries.md](crate-boundaries.md). The established shape is an `OnceLock` hook that ECStore defines and the binary installs at startup, as `EVENT_DISPATCH_HOOK` in `crates/ecstore/src/services/event_notification.rs` and `ON_DEMAND_MIGRATION_CONFIG_HOOK` in `crates/ecstore/src/bucket/on_demand_migration/config.rs` already do. `install_credential_sealer` follows it, and the binary supplies an implementation backed by `crates/kms/src/service_manager.rs`.

 ## Compatibility, per store, because the three differ

@@ -1,22 +0,0 @@
-# Bucket metadata diagnostics and recovery
-
-`GET /rustfs/admin/v3/export-bucket-metadata` keeps its strict behavior: an unreadable configuration fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets.
-
-To inspect readable configurations while identifying failures, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has:
-
- Filename `bucket-meta-diagnostic.zip` and header `x-rustfs-bucket-metadata-export: diagnostic`.
- Readable entries under `_diagnostic/<bucket>/<config>`; target credentials remain redacted.
- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details.
-
-`complete` reports whether all supported configuration reads succeeded. A diagnostic archive is never a restorable backup, including when `complete` is true. Import rejects the manifest or reserved directory before any bucket creation or configuration write. The reserved directory is not a valid bucket name, so older importers cannot restore diagnostic entries as ordinary bucket configurations.
-
-## Recover unreadable replication targets
-
-RustFS currently accepts the documented `{"targets": [...]}` object format. It cannot decrypt MinIO KMS-encrypted target metadata. Unreadable target payloads remain failures instead of being interpreted as an empty target set; diagnostic export and replacement import do not add MinIO KMS decryption support.
-
-1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery.
-2. Prepare a ZIP containing `<bucket>/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets.
-3. Submit the ZIP to the existing authenticated `PUT /rustfs/admin/v3/import-bucket-metadata` endpoint with `ImportBucketMetadataAction` permission. Import validates the replacement and persists it against the bucket incarnation; it does not need to parse the old target payload successfully.
-4. Verify target listing and the intended replication configuration. Retry the ordinary strict metadata export to confirm the unreadable configuration no longer blocks it.
-
-Do not submit the diagnostic archive itself to the import endpoint. Copy only reviewed replacement entries into an ordinary import archive.
@@ -1,42 +1,21 @@
 # On-Demand Migration

 **Use this when:** you are moving an existing S3-compatible bucket into RustFS without a stop-the-world copy, or you are debugging a bucket that serves reads from an external source (424 `SourceUnavailable`, an open circuit breaker, missing pulled objects, a source 403).
-**Source of truth:** `rustfs/src/on_demand_migration/` (`config.rs` for the wire model and its bounds, `sys.rs` for the per-node runtime, `source_client.rs` for the outbound client, `pull.rs` for the write-back pipeline, `breaker.rs` and `negative_cache.rs` for the protections), `rustfs/src/app/object/get.rs` and `head.rs` for the read paths, `rustfs/src/app/object/on_demand_migration_put.rs` for the local write, `rustfs/src/admin/handlers/on_demand_migration.rs` for the admin API, and `crates/obs/src/metrics/schema/on_demand_migration.rs` for the metric contract.
+**Source of truth:** `crates/ecstore/src/bucket/on_demand_migration/` (`config.rs` for the wire model and its bounds, `sys.rs` for the per-node runtime, `source_client.rs` for the outbound client, `pull.rs` for the write-back pipeline, `breaker.rs` and `negative_cache.rs` for the protections), `rustfs/src/app/object/get.rs` and `head.rs` for the read paths, `rustfs/src/app/object/on_demand_migration_put.rs` for the local write, `rustfs/src/admin/handlers/on_demand_migration.rs` for the admin API, and `crates/obs/src/metrics/schema/on_demand_migration.rs` for the metric contract.

 On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** to a local RustFS bucket. When a client GETs a key that does not exist locally, RustFS fetches it from the source, streams it to the client, and stores it locally in the same pass; every later read is served locally. It is a pull-style, lazy migration path — the RustFS equivalent of Cloudflare R2 Sippy, Tigris shadow buckets, and Alibaba Cloud OSS / Tencent COS mirror-back-to-origin.

 The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured.

-## Upgrade and rollback compatibility
-
-ODM configuration is stored in two additional keys in the existing bucket metadata map. The metadata format version remains `1` for MinIO compatibility. RustFS `1.0.0-rc.5` only re-encodes its 44 known keys: a bucket configuration write through an rc.5 node discards the ODM configuration and timestamp, even if another node originally wrote them. Restarting a newer binary cannot recover the discarded values. This also means ODM is not supported during a rolling upgrade that still allows rc.5 nodes to write bucket metadata.
-
-Upgrade every node before enabling ODM. Before any rollback to rc.5, stop new migration work, retain a secure copy of the original full configuration and credentials, and disable ODM on every bucket and node. The redacted configuration GET and metadata export are not credential backups. Objects still present only at the source cannot be read through RustFS while ODM is disabled or rc.5 is running; finish migration first, redirect those reads to the source, or plan a maintenance window. After all nodes return to a compatible release, reapply and validate the saved configuration; already stored local objects remain local. Turning the global module switch off alone does not make an old metadata writer preserve these keys.
-
-The ignored `upgrade_compatibility_test::rc5_rollback_requires_restoring_odm_configuration` test pins release commit `40a2470feb567201165a5b809b7598bb4b1f68f5`, restarts against the same data directory, writes bucket tags through rc.5, and verifies configuration recovery after returning to the current binary. Set `RUSTFS_UPGRADE_SOURCE_BINARY` to that release's executable and run `cargo test -p e2e_test rc5_rollback_requires_restoring_odm_configuration -- --ignored --test-threads=1`. The test records a known old-writer limitation; it does not certify mixed-version ODM operation.
-
-## Optional Google dependencies
-
-The default and `full` server builds include the `gcs` Cargo feature to preserve native GCS migration and existing GCS tier support. For a server without Google SDK dependencies, build with `cargo build -p rustfs --no-default-features --features ftps,webdav`. Add `gcs` to that feature list to restore native GCS support. The ECStore library has no default Google dependency; library users that need GCS tiers must enable its `gcs` feature.
-
-Both builds can read, redact and preserve GCS configuration. A build without `gcs` rejects native ODM client construction with `OnDemandMigrationBackendNotCompiled` (HTTP 501); persisted native sources report an unavailable client. GCS tier initialization returns `XRustFSAdminTierTypeUnsupported` (HTTP 501). Do not deploy that build to a cluster with GCS tiers containing transitioned objects: the configuration remains intact, but reading their remote data requires a GCS-capable binary. The `gcs` provider using HMAC credentials and the S3 interoperability API remains available in every build; only `gcs_native` and native GCS tier clients need the feature.
-
 ## List continuation token rollout

-Two independent rollout switches default to `false`; unset or invalid boolean values also keep them off. Both are node environment variables, not bucket settings:
+`RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape.

- `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` allows a v1 listing to first issue a v2 token after an empty truncated merged page. Existing v2 tokens keep their budget even on reader-only nodes. Consuming an object/common prefix or reaching a new EOF resets the budget to v1 without changing the chain's framing.
- `RUSTFS_ON_DEMAND_MIGRATION_LIST_FRAMED_TOKENS` allows a bare/new merged listing to first issue a NUL-prefixed JSON envelope inside the existing base64 encoding. Existing framed chains stay framed even with this switch off, including a reset to v1 and local continuation after list-through is disabled. With framing issuance off, new bare v1 output keeps its historical bytes; ordinary local listings remain unchanged. This switch does not enable the v2 budget.
+Leave the switch off while deploying v2 reader support to every node that can receive a continuation request, including nodes behind other load-balancer routes. Then set it to `true` in each node's environment and restart those nodes to enable issuance. A v1-only binary rejects v2 with `400 InvalidArgument` before the source-error policy runs; neither `not_found` nor turning off list-through makes that old reader compatible. With issuance still off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. The default rollout does not claim to fix that chain until issuance is enabled.

-Deploy readers before enabling either writer switch. Every node serving continuation requests must understand the selected token version and framing, including nodes behind other load-balancer routes. This build reads both complete historical bare envelopes and framed v1/v2 envelopes with the same strict version/count validation. A bare-v1-only binary rejects bare v2 with `400 InvalidArgument`; an old bare reader mistakes framed input for a local marker, while a framed-only reader mistakes bare input for one. Neither format mismatch is safe: it can restart a merged scan and lose its budget rather than returning an error. The interim framed-only build from #7187 must be replaced on every serving node before a mixed-format rollout. Enable framing only after all readers support both formats; enable the budget only after all readers support v2. Restart nodes after updating their environment.
+An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64 JSON, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries.

-Partial JSON-shaped object keys remain local markers. To retain already issued cursors, a bare JSON object with the ODM tag and every historical writer field (`v`, `local`, `local_done`, `source`, `source_done`, `last_key`) is treated as an envelope, then strictly validated. A valid object key can be identical to that complete envelope: the two byte strings are indistinguishable, so legacy compatibility necessarily gives the envelope interpretation precedence. Framing identifies new merged tokens unambiguously, but dual-format readers do not eliminate this old full-envelope key collision. There is no signature, session store, or automatic format negotiation.
-
-An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. A zero-sized request does not spend an existing budget. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64-encoded JSON, optionally framed, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries.
-
-With budget issuance off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. Default rollout does not fix that chain until the v2 switch is enabled. Framing alone does not impose the budget.
-
-For rollback, first turn both issuance switches off on every node. Keep readers compatible with outstanding framed and v2 chains: neither switch rewrites existing tokens, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring older binaries. Restarting a listing is a new scan and can repeat entries. Do not assume switching issuance off makes outstanding framed or v2 tokens disappear.
+For rollback, first turn issuance off on every node. Keep v2-capable readers available for outstanding v2 chains: switching issuance off does not erase their budgets, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring v1-only binaries. Restarting a listing is a new scan and can repeat entries. Do not roll back readers while assuming the issuance switch makes existing v2 tokens disappear.

 ## Positioning

@@ -118,7 +97,7 @@ Read-through only migrates what clients touch. The background backfill job walks

 ## Configuration reference

-The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unknown fields are rejected rather than dropped, so a config written by a newer build fails loudly on an older one. Every default and bound below comes from `rustfs/src/on_demand_migration/config.rs`.
+The persisted blob is `on-demand-migration.json` in the bucket's metadata. Unknown fields are rejected rather than dropped, so a config written by a newer build fails loudly on an older one. Every default and bound below comes from `crates/ecstore/src/bucket/on_demand_migration/config.rs`.

 | Field | Type | Default | Bounds / rules |
 |---|---|---|---|
@@ -175,7 +154,7 @@ Validation also rejects two shapes outright: a source whose endpoint and bucket
 | `azure` | Optional; derived as `https://<account>.blob.core.windows.net` | Native Blob REST, not S3 | Unused; write `auto` | Needs `source.azure`; the container is `source.bucket`. Reads need `Read` on the blob and `List` on the container, plus `Tags` when `policy.copy_tags` is on | None yet: no interop job covers Azure |
 | `gcs_native` | Optional; derived as `https://storage.googleapis.com` | Native GCS API, not S3 | Unused; write `auto` | Needs `source.gcs`. Reads use the XML API for objects and `objects.list` for listings, both with an OAuth token minted from the service-account key; the key needs `storage.objects.get` and `storage.objects.list` | None yet: no interop job covers native GCS |

-Every backend answers the same trait contract, pinned by `backend_contract.rs` in `rustfs/src/on_demand_migration/`, and the three differences that contract allows are the ones documented here.
+Every backend answers the same trait contract, pinned by `backend_contract.rs` in `crates/ecstore/src/bucket/on_demand_migration/`, and the three differences that contract allows are the ones documented here.

 `azure` differs in two of them. Its ETag is a concurrency token rather than a digest of the bytes, so it is stored as `odm-source-etag` provenance and never used as the expected MD5 of a pulled object — the write-back integrity check falls back to the local digest. And its listing paginates only with an opaque marker: there is no "start after this key" form, so a caller that asks for one gets `Unsupported` instead of a listing that silently starts over.

@@ -195,9 +174,9 @@ No write, delete, ACL or versioning permission is required or used. Scope the po

 Behaviour a client can observe. The "Test" column names the case that pins it: `*_test.rs` files live under `crates/e2e_test/src/on_demand_migration/`, and the unit tests live next to the code in `rustfs/src/app/object/get.rs`, `head.rs` and `shared.rs`.

-ODM merged continuation tokens use bare or NUL-prefixed JSON inside the existing base64 encoding. The default writer preserves bare output; compatible readers accept both formats and retain existing budgets. See [List continuation token rollout](#list-continuation-token-rollout) for the independent issuance switches, rolling-upgrade requirements, and the unavoidable ambiguity between a complete historical envelope and an identically named local key.
+ODM merged continuation tokens use a NUL-prefixed JSON envelope inside the existing base64 encoding. NUL is not valid in a local object key, so a legitimate JSON-shaped key can never be mistaken for a merged cursor. Upgrade every node before using list-through, and restart any in-progress ODM listing issued by an older build: its unframed JSON tokens cannot be distinguished from legitimate local keys. Ordinary local listing tokens remain unchanged. Tokens issued by this build can still resume the local side after list-through is disabled.

-Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket or a missing source version is not proof that the requested key is absent. Native GCS verifies the bucket after either HEAD or GET returns 404 and preserves a failed probe as a source error. Azure accepts explicit `BlobNotFound` only on an unversioned object read with status 404; an ambiguous HEAD may make one container probe, while an ambiguous GET remains a source error. Native probes add at most one request and retain the existing per-request timeouts, rather than a single deadline for the pair. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object.
+Source `HEAD` responses with status 404 require a successful bucket probe before being negative-cached. The source credential therefore needs permission for `HeadBucket` (S3 `ListBucket`); a prefix-restricted ListBucket policy can deny that probe, in which case the response is a source failure rather than a cached miss. A missing/inaccessible source bucket, a missing source version, or an ambiguous GET 404 is not proof that the requested key is absent. Conditional GET validators are checked against the actual source GET metadata as well as the advisory HEAD; a missing required validator fails with 424. Source LIST entries without a key or a non-negative size fail the page rather than fabricating an empty object.

 Write-back currently requires namespace locking enabled and exactly one pool with one erasure set. Other topologies fail write-back explicitly as `unsupported`: source reads remain available, but backfill cannot complete successfully or certify cutover. This restriction avoids relying on a set-local condition across distinct pool or lock domains; it does not restrict ordinary S3 writes. Full cross-pool migration requires a globally fenced commit protocol.

@@ -12,10 +12,10 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
 |---|---|---|---|
 | Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p <crate>`); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) |
 | ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) |
-| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) |
+| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`, `e2e-distributed`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); 4-node 4-disk map: [distributed-e2e.md](distributed-e2e.md) |
 | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` | PR, report-only (second half of the `End-to-End Tests` job) |
 | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only |
-| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree |
+| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` |
 | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus |
 | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p <crate>` | On demand; never a gate |

@@ -63,6 +63,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the
 | `e2e-full` | Merge-queue / main-push single-node e2e lane |
 | `e2e-repl-nightly` | Nightly slow / cross-process replication lane |
 | `e2e-nightly` | Nightly serial multi-process cluster fault lane |
+| `e2e-distributed` | Storage-sensitive PR and nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
 | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` |

 Membership of each e2e profile is pinned by a digest in `.config/e2e-<profile>-selection.txt` and checked by `scripts/check_test_wiring.py --check-profile <profile>` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only):
@@ -49,7 +49,7 @@ Promotion rule: never promote a report-only lane to required from one green run.
 | PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
 | PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows |
 | PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
-| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
+| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from rc.2` | `e2e-upgrade.yml` `direct-upgrade` | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release |
 | PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
 | PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
 | PR limited to main-CI-excluded paths | `Quick Checks`, `Test and Lint` | `ci-docs-only.yml` `quick-checks`, `test-and-lint` | Required | `git diff --check`; `make doc-paths-check`; `scripts/check_no_planning_docs.sh` |
@@ -74,6 +74,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` |
 | `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
+| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` |
 | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
 | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
 | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |
@@ -84,7 +85,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
 | `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` |
 | `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
-| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
+| `e2e-upgrade.yml` (weekly) | `direct-upgrade` | upgrade gate; server logs | no | see the PR row |
 | `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row |
 | `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row |
 | `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on a never-created or stale schedule | n/a | dispatch |
@@ -0,0 +1,80 @@
+# Distributed 4-node 4-disk e2e
+
+**Use this when:** adding or diagnosing GitHub Actions coverage for a 4-node cluster, or deciding whether a behaviour belongs in `e2e-distributed` versus the single-node `e2e-full` lane, the nightly cluster-fault lane, or the hardware functional chain.
+**Source of truth:** `crates/e2e_test/src/distributed/`, `[profile.e2e-distributed]` in `.config/nextest.toml`, `.github/workflows/e2e-distributed.yml`.
+
+## Topology
+
+The in-tree harness runs every node on `127.0.0.1` with a distinct port. That matches `RustFSTestClusterEnvironment` in `crates/e2e_test/src/common.rs`:
+
+| Layout | Constructor | Use |
+|---|---|---|
+| 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos |
+| 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release |
+| 1 single-node pool × 4 drives, then `append_single_node_pool` three times | expansion seed | Pool expand, then decommission / rebalance / integrity |
+
+A multi-pool layout in which any pool spans several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped expansion pools remain the hardware functional-chain / backlog #1313 / #1314 lane.
+
+Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran.
+
+The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated 1 GiB tmpfs filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use ext4 loop devices: the `sm-standard-4` ARC pods have no `/dev/loop-control`, so `mount -o loop` fails with `No such file or directory`. Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata.
+
+The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing.
+
+## What this lane covers
+
+`cargo nextest run --profile e2e-distributed -p e2e_test` selects `distributed::*`:
+
+- S3 put / get / head / list / copy / rename / delete / presign, range and conditional reads, special keys, metadata, tags, pagination, empty objects, multipart complete and abort
+- Object Lock COMPLIANCE, GOVERNANCE and bypass, legal hold, bucket default retention, and non-lock bucket rejection
+- Versioning, exact historical reads, delete-marker removal, and suspended null-version overwrite semantics
+- Bucket replication between two 4-node clusters, including metadata/tags and target-outage retry; hard quota admission and absence of rejected keys
+- Ready/live probes on every node, exact 4-server/16-disk inventory, realtime metrics on every node, and correlated audit-webhook delivery
+- Pool expand, decommission, rebalance, checksum integrity, versioned and multipart data, and S3 during active movement
+- Bidirectional site-replication convergence plus enabled/synchronized peer state on both sites
+- A 24-worker mixed PUT/HEAD/GET/COPY/DELETE workload; concurrent PUT during active decommission
+- Node kill/restart, full process restart, node-facing TCP blackhole/recovery, in-flight streaming GETs across a peer kill, and fresh-drive replacement verified by physical `xl.meta`/part-shard census
+- Multipart, cross-node listing, list-buckets agreement
+- Direct and rolling upgrade from the pinned previous release: historical objects, versioned history, and IAM user AK/SK still work afterwards
+
+## Existing Actions gaps this lane does not replace
+
+Those suites stay in place; this lane fills the in-tree 4×4 hole they leave.
+
+| Existing lane | Gap |
+|---|---|
+| `rustfs-*-test.yml` functional chain | Clones private `rustfs/auto-testing`, runs on three shared VMs (`vm000`–`vm002`), `continue-on-error: true`, not a merge signal, not 4 nodes. Hardware `rustfs-upgrade-test.yml` stays there |
+| `e2e-upgrade.yml` | Single-node SSE/multipart/delete-marker contracts plus mixed-version listing; does not pin IAM user AK/SK on a 4-node cluster |
+| `e2e-smoke` / `e2e-full` | Most selected cases are single-node; distributed modules are intentionally owned by this serialized lane |
+| `e2e-nightly` | 4-node cluster faults and heal, not S3/lock/versioning/quota/decommission matrix |
+| `e2e-repl-nightly` | Site and bucket replication on 1–3 *single-node* processes |
+| `e2e-s3tests.yml` `multi` | Weekly ceph/s3-tests against Docker 4-node; not lock/WORM, decommission, chaos, or checksum integrity |
+| `crates/e2e_test/src/chaos.rs` | Single-node disk faults only |
+
+Hardware power-loss, physical NIC pull, authenticated inter-node partition, firmware/media errors, and replacement-server provisioning still belong on the hardware validation VMs. This lane provides deterministic process kill, fresh local-volume replacement, and node-facing TCP blackhole analogues; it does not claim physical fault certification.
+
+## Run
+
+```bash
+cargo build -p rustfs --bins
+# Expansion/decommission/rebalance cases require four paths on distinct filesystems.
+# If you do not already have four disks, sized tmpfs is enough:
+#   for p in 0 1 2 3; do
+#     sudo mkdir -p /mnt/rustfs-pool-$p
+#     sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs /mnt/rustfs-pool-$p
+#   done
+export RUSTFS_E2E_POOL_ROOTS=/mnt/rustfs-pool-0:/mnt/rustfs-pool-1:/mnt/rustfs-pool-2:/mnt/rustfs-pool-3
+# Upgrade cases require the pinned previous binary (CI downloads it).
+export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2
+cargo nextest run --profile e2e-distributed -p e2e_test
+```
+
+Without `RUSTFS_UPGRADE_SOURCE_BINARY` the two `distributed::upgrade_test::*` cases fail closed. Without four distinct `RUSTFS_E2E_POOL_ROOTS`, the expansion and data-movement cases fail closed. Filter upgrades out for a local run that is not checking upgrade:
+
+```bash
+cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distributed::upgrade_test::/)'
+```
+
+The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check.
+
+Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linux and Darwin entries with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed <listing.json> <platform>` after adding or renaming a case.
@@ -57,8 +57,7 @@ name = "swift_object_integration_test"
 required-features = ["swift"]

 [features]
-default = ["ftps", "webdav", "gcs"]
-gcs = ["rustfs-ecstore/gcs", "dep:google-cloud-auth"]
+default = ["ftps", "webdav"]
 metrics-gpu = ["rustfs-obs/gpu"]
 ftps = ["rustfs-protocols/ftps"]
 swift = ["rustfs-protocols/swift"]
@@ -67,7 +66,7 @@ sftp = ["rustfs-protocols/sftp"]
 license = []
 io-scheduler-debug = []  # Enable debug information in I/O scheduler
 tracing-chunk-debug = []  # Enable per-chunk tracing in data plane (high noise, for debugging only)
-full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
+full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope"]
 e2e-test-hooks = []
 # Shortens Connect credentials only in debug E2E builds.
 connect-e2e-short-credentials = []
@@ -276,7 +275,6 @@ rcgen = { workspace = true }
 # Async Runtime and Networking
 async-trait = { workspace = true }
 axum.workspace = true
-faster-hex.workspace = true
 futures.workspace = true
 futures-lite.workspace = true
 futures-util.workspace = true
@@ -289,11 +287,6 @@ reqwest = { workspace = true, features = ["json", "stream"] }
 socket2 = { workspace = true, features = ["all"] }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "signal", "process", "io-util", "fs"] }
 tokio-rustls = { workspace = true, default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
-aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
-aws-smithy-types = { workspace = true }
-google-cloud-auth = { workspace = true, optional = true }
-moka = { workspace = true, features = ["sync"] }
-xxhash-rust = { workspace = true, features = ["xxh3"] }
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 tokio-stream.workspace = true
 tokio-util = { workspace = true, features = ["io", "compat", "time"] }
@@ -1,80 +0,0 @@
-// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01.
-// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted.
-use serde::{Deserialize, Serialize};
-use std::fmt;
-
-const REDACTED: &str = "REDACTED";
-
-/// The external S3-compatible source bucket.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct SourceConfig {
-    pub provider: Provider,
-    /// `http(s)://host[:port]` with no path or query. Optional only for
-    /// [`Provider::Aws`], where it is derived from `region`.
-    #[serde(default)]
-    pub endpoint: Option<String>,
-    pub region: String,
-    pub bucket: String,
-    #[serde(default)]
-    pub path_style: PathStyle,
-    /// `None` means anonymous access to a public source bucket.
-    #[serde(default)]
-    pub credentials: Option<SourceCredentials>,
-    #[serde(default)]
-    pub tls: TlsConfig,
-}
-
-/// Source vendor family. `azure` is deliberately absent from this version.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum Provider {
-    /// Generic S3-compatible endpoint.
-    S3,
-    Aws,
-    Minio,
-    Rustfs,
-    R2,
-    /// GCS XML interoperability API with HMAC keys.
-    Gcs,
-}
-
-/// Bucket addressing style. `auto` is resolved by the source client builder.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-pub enum PathStyle {
-    #[default]
-    Auto,
-    Path,
-    Virtual,
-}
-
-/// Static credentials for the source. `Debug` never prints the secret or
-/// the session token.
-#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct SourceCredentials {
-    pub access_key: String,
-    pub secret_key: String,
-    #[serde(default)]
-    pub session_token: Option<String>,
-}
-
-impl fmt::Debug for SourceCredentials {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("SourceCredentials")
-            .field("access_key", &self.access_key)
-            .field("secret_key", &REDACTED)
-            .field("session_token", &self.session_token.as_ref().map(|_| REDACTED))
-            .finish()
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct TlsConfig {
-    #[serde(default)]
-    pub skip_verify: bool,
-    #[serde(default)]
-    pub ca_cert_pem: Option<String>,
-}
@@ -64,9 +64,6 @@ use time::OffsetDateTime;
 use tracing::warn;
 use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};

-const DIAGNOSTIC_EXPORT_PREFIX: &str = "_diagnostic";
-const DIAGNOSTIC_EXPORT_MANIFEST: &str = "_diagnostic-manifest.json";
-
 const LOG_COMPONENT_ADMIN: &str = "admin";
 const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta";
 const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state";
@@ -100,198 +97,9 @@ fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec<u8>) ->
    checked_raw_xml(validated, raw, deserialize::<VersioningConfiguration>)
 }

-async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result<Option<Vec<u8>>> {
-    match conf {
-        BUCKET_POLICY_CONFIG => {
-            let config: BucketPolicy = match metadata_sys::get_bucket_policy(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                }
-            };
-            let config_json =
-                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
-            Ok(Some(config_json))
-        }
-        BUCKET_NOTIFICATION_CONFIG => {
-            let config: s3s::dto::NotificationConfiguration = match metadata_sys::get_notification_config(bucket).await {
-                Ok(Some(res)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                }
-                Ok(None) => return Ok(None),
-            };
-
-            let raw_config = metadata_sys::get(bucket)
-                .await
-                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                .notification_config_xml
-                .clone();
-            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
-
-            Ok(Some(config_xml))
-        }
-        BUCKET_LIFECYCLE_CONFIG => {
-            let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                }
-            };
-            let raw_config = metadata_sys::get(bucket)
-                .await
-                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
-                .lifecycle_config_xml
-                .clone();
-            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
-
-            Ok(Some(config_xml))
-        }
-        BUCKET_TAGGING_CONFIG => {
-            let config: Tagging = match metadata_sys::get_tagging_config(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
-                }
-            };
-            let raw_config = metadata_sys::get(bucket)
-                .await
-                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
-                .tagging_config_xml
-                .clone();
-            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
-
-            Ok(Some(config_xml))
-        }
-        BUCKET_QUOTA_CONFIG_FILE => {
-            let config: BucketQuota = match metadata_sys::get_quota_config(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                }
-            };
-            let config_json =
-                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
-
-            Ok(Some(config_json))
-        }
-        OBJECT_LOCK_CONFIG => {
-            let config = match metadata_sys::get_object_lock_config(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                }
-            };
-            let raw_config = metadata_sys::get(bucket)
-                .await
-                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                .object_lock_config_xml
-                .clone();
-            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
-
-            Ok(Some(config_xml))
-        }
-        BUCKET_SSECONFIG => {
-            let config = match metadata_sys::get_sse_config(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                }
-            };
-            let raw_config = metadata_sys::get(bucket)
-                .await
-                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                .encryption_config_xml
-                .clone();
-            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
-
-            Ok(Some(config_xml))
-        }
-        BUCKET_VERSIONING_CONFIG => {
-            let config = match metadata_sys::get_versioning_config(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                }
-            };
-            let raw_config = metadata_sys::get(bucket)
-                .await
-                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                .versioning_config_xml
-                .clone();
-            let config_xml = checked_versioning_xml(&config, raw_config)?;
-
-            Ok(Some(config_xml))
-        }
-        BUCKET_REPLICATION_CONFIG => {
-            let config = match metadata_sys::get_replication_config(bucket).await {
-                Ok((res, _)) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                }
-            };
-            let raw_config = metadata_sys::get(bucket)
-                .await
-                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
-                .replication_config_xml
-                .clone();
-            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
-
-            Ok(Some(config_xml))
-        }
-        BUCKET_TARGETS_FILE => {
-            let config: BucketTargets = match metadata_sys::get_bucket_targets_config(bucket).await {
-                Ok(res) => res,
-                Err(e) => {
-                    if e == StorageError::ConfigNotFound {
-                        return Ok(None);
-                    }
-                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
-                }
-            };
-
-            let config_json = serde_json::to_vec(&config.redacted_credentials())
-                .map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
-
-            Ok(Some(config_json))
-        }
-        _ => Ok(None),
-    }
-}
-
 #[derive(Debug, Default, serde::Deserialize)]
 pub struct ExportBucketMetadataQuery {
-    #[serde(default)]
    pub bucket: String,
-    #[serde(default)]
-    pub diagnostic: bool,
 }

 pub struct ExportBucketMetadata {}
@@ -361,7 +169,6 @@ impl Operation for ExportBucketMetadata {
        };

        let mut zip_writer = ZipWriter::new(Cursor::new(Vec::new()));
-        let mut errors = Vec::new();

        let confs = [
            BUCKET_POLICY_CONFIG,
@@ -379,47 +186,242 @@ impl Operation for ExportBucketMetadata {
        for bucket in buckets {
            for &conf in confs.iter() {
                let conf_path = path_join_buf(&[bucket.name.as_str(), conf]);
-                let config = match exported_bucket_config(&bucket.name, conf).await {
-                    Ok(Some(config)) => config,
-                    Ok(None) => continue,
-                    Err(error) if !query.diagnostic => return Err(error),
-                    Err(_) => {
-                        errors.push(serde_json::json!({
-                            "bucket": bucket.name,
-                            "config": conf,
-                            "code": "configuration_unavailable",
-                        }));
-                        continue;
+                match conf {
+                    BUCKET_POLICY_CONFIG => {
+                        let config: BucketPolicy = match metadata_sys::get_bucket_policy(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                            }
+                        };
+                        let config_json = serde_json::to_vec(&config)
+                            .map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
+                        zip_writer
+                            .write_all(&config_json)
+                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
                    }
-                };
-                let conf_path = if query.diagnostic {
-                    path_join_buf(&[DIAGNOSTIC_EXPORT_PREFIX, &conf_path])
-                } else {
-                    conf_path
-                };
-                zip_writer
-                    .start_file(conf_path, SimpleFileOptions::default())
-                    .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                zip_writer
-                    .write_all(&config)
-                    .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
-            }
-        }
+                    BUCKET_NOTIFICATION_CONFIG => {
+                        let config: s3s::dto::NotificationConfiguration =
+                            match metadata_sys::get_notification_config(&bucket.name).await {
+                                Ok(Some(res)) => res,
+                                Err(e) => {
+                                    if e == StorageError::ConfigNotFound {
+                                        continue;
+                                    }
+                                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                                }
+                                Ok(None) => continue,
+                            };

-        if query.diagnostic {
-            let manifest = serde_json::to_vec(&serde_json::json!({
-                "version": 1,
-                "mode": "diagnostic",
-                "complete": errors.is_empty(),
-                "errors": errors,
-            }))
-            .map_err(|e| s3_error!(InternalError, "failed to serialize diagnostic manifest: {e}"))?;
-            zip_writer
-                .start_file(DIAGNOSTIC_EXPORT_MANIFEST, SimpleFileOptions::default())
-                .map_err(|e| s3_error!(InternalError, "failed to start diagnostic manifest: {e}"))?;
-            zip_writer
-                .write_all(&manifest)
-                .map_err(|e| s3_error!(InternalError, "failed to write diagnostic manifest: {e}"))?;
+                        let raw_config = metadata_sys::get(&bucket.name)
+                            .await
+                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                            .notification_config_xml
+                            .clone();
+                        let config_xml =
+                            checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
+                        zip_writer
+                            .write_all(&config_xml)
+                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
+                    }
+                    BUCKET_LIFECYCLE_CONFIG => {
+                        let config: BucketLifecycleConfiguration = match metadata_sys::get_lifecycle_config(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                            }
+                        };
+                        let raw_config = metadata_sys::get(&bucket.name)
+                            .await
+                            .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
+                            .lifecycle_config_xml
+                            .clone();
+                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
+                        zip_writer
+                            .write_all(&config_xml)
+                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
+                    }
+                    BUCKET_TAGGING_CONFIG => {
+                        let config: Tagging = match metadata_sys::get_tagging_config(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
+                            }
+                        };
+                        let raw_config = metadata_sys::get(&bucket.name)
+                            .await
+                            .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
+                            .tagging_config_xml
+                            .clone();
+                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
+                        zip_writer
+                            .write_all(&config_xml)
+                            .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
+                    }
+                    BUCKET_QUOTA_CONFIG_FILE => {
+                        let config: BucketQuota = match metadata_sys::get_quota_config(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                            }
+                        };
+                        let config_json =
+                            serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
+                        zip_writer
+                            .write_all(&config_json)
+                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
+                    }
+                    OBJECT_LOCK_CONFIG => {
+                        let config = match metadata_sys::get_object_lock_config(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                            }
+                        };
+                        let raw_config = metadata_sys::get(&bucket.name)
+                            .await
+                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                            .object_lock_config_xml
+                            .clone();
+                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
+                        zip_writer
+                            .write_all(&config_xml)
+                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
+                    }
+                    BUCKET_SSECONFIG => {
+                        let config = match metadata_sys::get_sse_config(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                            }
+                        };
+                        let raw_config = metadata_sys::get(&bucket.name)
+                            .await
+                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                            .encryption_config_xml
+                            .clone();
+                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
+                        zip_writer
+                            .write_all(&config_xml)
+                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
+                    }
+                    BUCKET_VERSIONING_CONFIG => {
+                        let config = match metadata_sys::get_versioning_config(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                            }
+                        };
+                        let raw_config = metadata_sys::get(&bucket.name)
+                            .await
+                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                            .versioning_config_xml
+                            .clone();
+                        let config_xml = checked_versioning_xml(&config, raw_config)?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
+                        zip_writer
+                            .write_all(&config_xml)
+                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
+                    }
+                    BUCKET_REPLICATION_CONFIG => {
+                        let config = match metadata_sys::get_replication_config(&bucket.name).await {
+                            Ok((res, _)) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                            }
+                        };
+                        let raw_config = metadata_sys::get(&bucket.name)
+                            .await
+                            .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
+                            .replication_config_xml
+                            .clone();
+                        let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
+                        zip_writer
+                            .write_all(&config_xml)
+                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
+                    }
+                    BUCKET_TARGETS_FILE => {
+                        let config: BucketTargets = match metadata_sys::get_bucket_targets_config(&bucket.name).await {
+                            Ok(res) => res,
+                            Err(e) => {
+                                if e == StorageError::ConfigNotFound {
+                                    continue;
+                                }
+                                return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
+                            }
+                        };
+
+                        let config_json = serde_json::to_vec(&config.redacted_credentials())
+                            .map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;
+
+                        zip_writer
+                            .start_file(conf_path, SimpleFileOptions::default())
+                            .map_err(|e| s3_error!(InternalError, "start file failed: {e}"))?;
+                        zip_writer
+                            .write_all(&config_json)
+                            .map_err(|e| s3_error!(InternalError, "write file failed: {e}"))?;
+                    }
+                    _ => {}
+                }
+            }
        }

        let zip_bytes = zip_writer
@@ -429,17 +431,8 @@ impl Operation for ExportBucketMetadata {
        header.insert(CONTENT_TYPE, "application/zip".parse().expect("valid header value"));
        header.insert(
            CONTENT_DISPOSITION,
-            if query.diagnostic {
-                "attachment; filename=bucket-meta-diagnostic.zip"
-            } else {
-                "attachment; filename=bucket-meta.zip"
-            }
-            .parse()
-            .expect("valid header value"),
+            "attachment; filename=bucket-meta.zip".parse().expect("valid header value"),
        );
-        if query.diagnostic {
-            header.insert("x-rustfs-bucket-metadata-export", "diagnostic".parse().expect("valid header value"));
-        }
        header.insert(CONTENT_LENGTH, zip_bytes.get_ref().len().to_string().parse().expect("valid header value"));
        Ok(S3Response::with_headers((StatusCode::OK, Body::from(zip_bytes.into_inner())), header))
    }
@@ -506,18 +499,6 @@ impl Operation for ImportBucketMetadata {
            file_contents.push((file_path, content));
        }

-        // Reject the whole archive before creating buckets or writing configs,
-        // even when the marker is malformed or follows ordinary config entries.
-        if file_contents.iter().any(|(path, _)| {
-            path == DIAGNOSTIC_EXPORT_MANIFEST
-                || path == DIAGNOSTIC_EXPORT_PREFIX
-                || path
-                    .strip_prefix(DIAGNOSTIC_EXPORT_PREFIX)
-                    .is_some_and(|suffix| suffix.starts_with('/'))
-        }) {
-            return Err(s3_error!(InvalidRequest, "diagnostic bucket metadata archives cannot be imported"));
-        }
-
        let durable_quota_import = imported_quota_requires_fleet_proof(&file_contents)?;
        let quota_fleet_proof =
            if durable_quota_import {
@@ -1438,246 +1419,6 @@ mod backup_zip_compatibility_tests {
        assert_eq!(response.output.0, StatusCode::OK);
    }

-    #[tokio::test]
-    #[serial_test::serial]
-    async fn diagnostic_export_isolated_errors_and_import_recovers_unreadable_targets() {
-        const HEALTHY: &str = "diagnostic-healthy";
-        const UNREADABLE: &str = "diagnostic-unreadable";
-        const REPLACEMENT_TARGETS: &[u8] = br#"{"targets":[]}"#;
-        const SECRET: &str = "diagnostic-must-not-expose-this-secret";
-
-        let _ = rustfs_credentials::init_global_action_credentials(
-            Some(ROOT_ACCESS_KEY.to_string()),
-            Some(ROOT_SECRET_KEY.to_string()),
-        );
-        let temp = tempfile::tempdir().expect("create diagnostic export test root");
-        let env = rustfs_test_utils::TestECStoreEnv::builder()
-            .base_dir(temp.path())
-            .disk_count(1)
-            .build()
-            .await;
-        env.make_bucket(HEALTHY, false).await;
-        env.make_bucket(UNREADABLE, false).await;
-        rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore))
-            .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
-            .await
-            .expect("seed IAM format");
-        let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore))
-            .await
-            .expect("build test IAM");
-        publish_test_app_context(Arc::new(AppContext::with_default_interfaces(
-            Arc::clone(&env.ecstore),
-            iam,
-            Arc::new(rustfs_kms::KmsServiceManager::new()),
-        )));
-        metadata_sys::update(HEALTHY, BUCKET_VERSIONING_CONFIG, VERSIONING_XML.to_vec())
-            .await
-            .expect("seed healthy bucket config");
-
-        let minio_blob = hex_simd::decode_to_vec(
-            include_str!("../../../../crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex").trim(),
-        )
-        .expect("decode real MinIO metadata fixture");
-        let minio_metadata = BucketMetadata::unmarshal(&minio_blob[4..]).expect("read MinIO metadata fixture");
-        let targets_array = format!(r#"[{{"credentials":{{"secretKey":"{SECRET}"}}}}]"#).into_bytes();
-        for unreadable_targets in [targets_array, minio_metadata.bucket_targets_config_json] {
-            let mut metadata = metadata_sys::get_config_from_disk(UNREADABLE)
-                .await
-                .expect("load bucket before simulating MinIO targets");
-            let incarnation = metadata.bucket_incarnation_id;
-            metadata.bucket_targets_config_json = unreadable_targets.clone();
-            metadata
-                .save_with_store(Arc::clone(&env.ecstore))
-                .await
-                .expect("persist unreadable targets fixture");
-            crate::storage::storage_api::set_bucket_metadata(UNREADABLE.to_string(), metadata)
-                .await
-                .expect("publish unreadable targets fixture");
-            assert!(metadata_sys::get_bucket_targets_config(UNREADABLE).await.is_err());
-
-            let strict_error = ExportBucketMetadata {}
-                .call(
-                    admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
-                    Params::new(),
-                )
-                .await
-                .expect_err("a complete export must fail closed on unreadable targets");
-            assert_eq!(*strict_error.code(), s3s::S3ErrorCode::InternalError);
-
-            let response = ExportBucketMetadata {}
-                .call(
-                    admin_request(
-                        Method::GET,
-                        Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
-                        Vec::new(),
-                    ),
-                    Params::new(),
-                )
-                .await
-                .expect("one unreadable bucket must not abort diagnostic export");
-            assert_eq!(response.output.0, StatusCode::OK);
-            assert_eq!(response.headers["x-rustfs-bucket-metadata-export"], "diagnostic");
-            assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta-diagnostic.zip");
-            let bytes = response.output.1.collect().await.expect("read diagnostic archive").to_bytes();
-            let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open diagnostic archive");
-            let mut files = HashMap::new();
-            for index in 0..archive.len() {
-                let mut file = archive.by_index(index).expect("read diagnostic entry");
-                let mut content = Vec::new();
-                file.read_to_end(&mut content).expect("read diagnostic config");
-                assert!(!content.windows(SECRET.len()).any(|window| window == SECRET.as_bytes()));
-                assert!(file.name() == DIAGNOSTIC_EXPORT_MANIFEST || file.name().starts_with("_diagnostic/"));
-                files.insert(file.name().to_string(), content);
-            }
-            assert_eq!(files[&format!("_diagnostic/{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")], VERSIONING_XML);
-            assert!(files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_VERSIONING_CONFIG}")));
-            assert!(!files.contains_key(&format!("_diagnostic/{UNREADABLE}/{BUCKET_TARGETS_FILE}")));
-            let manifest: serde_json::Value =
-                serde_json::from_slice(&files[DIAGNOSTIC_EXPORT_MANIFEST]).expect("decode diagnostic manifest");
-            assert_eq!(
-                manifest,
-                serde_json::json!({
-                    "version": 1,
-                    "mode": "diagnostic",
-                    "complete": false,
-                    "errors": [{ "bucket": UNREADABLE, "config": BUCKET_TARGETS_FILE, "code": "configuration_unavailable" }],
-                })
-            );
-
-            let error = ImportBucketMetadata {}
-                .call(
-                    admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
-                    Params::new(),
-                )
-                .await
-                .expect_err("diagnostic exports are never backups");
-            assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
-            assert_eq!(
-                metadata_sys::get_config_from_disk(UNREADABLE)
-                    .await
-                    .expect("load rejected import state")
-                    .bucket_targets_config_json,
-                unreadable_targets
-            );
-
-            import_archive(zip_with_entries(UNREADABLE, &[(BUCKET_TARGETS_FILE, REPLACEMENT_TARGETS)])).await;
-            let recovered = metadata_sys::get_config_from_disk(UNREADABLE)
-                .await
-                .expect("read recovered targets from disk");
-            assert_eq!(recovered.bucket_incarnation_id, incarnation);
-            assert_eq!(recovered.bucket_targets_config_json, REPLACEMENT_TARGETS);
-            assert!(
-                metadata_sys::get_bucket_targets_config(UNREADABLE)
-                    .await
-                    .expect("existing import API must recover targets readers")
-                    .is_empty()
-            );
-        }
-
-        // The marker may be malformed, come last, or be removed while the
-        // reserved directory remains. None may allow an earlier config write.
-        for marker in [
-            DIAGNOSTIC_EXPORT_MANIFEST.to_string(),
-            DIAGNOSTIC_EXPORT_PREFIX.to_string(),
-            format!("{DIAGNOSTIC_EXPORT_PREFIX}/bucket/config"),
-        ] {
-            let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
-            writer
-                .start_file(format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}"), SimpleFileOptions::default())
-                .expect("start ordinary config before diagnostic marker");
-            writer
-                .write_all(b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
-                .expect("write ordinary config before diagnostic marker");
-            writer
-                .start_file(
-                    format!("diagnostic-never-created/{BUCKET_VERSIONING_CONFIG}"),
-                    SimpleFileOptions::default(),
-                )
-                .expect("start a nonexistent bucket config before diagnostic marker");
-            writer.write_all(VERSIONING_XML).expect("write nonexistent bucket config");
-            writer
-                .start_file(marker, SimpleFileOptions::default())
-                .expect("start diagnostic marker");
-            writer.write_all(b"not json").expect("write malformed diagnostic marker");
-            let error = ImportBucketMetadata {}
-                .call(
-                    admin_request(
-                        Method::PUT,
-                        Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"),
-                        writer.finish().expect("finish marked archive").into_inner(),
-                    ),
-                    Params::new(),
-                )
-                .await
-                .expect_err("diagnostic preflight must reject before any config write");
-            assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
-            assert_eq!(
-                metadata_sys::get_config_from_disk(HEALTHY)
-                    .await
-                    .expect("read healthy config after rejected import")
-                    .versioning_config_xml,
-                VERSIONING_XML
-            );
-            assert!(
-                env.ecstore
-                    .get_bucket_info("diagnostic-never-created", &BucketOptions::default())
-                    .await
-                    .is_err(),
-                "diagnostic preflight must reject before bucket creation"
-            );
-        }
-
-        let response = ExportBucketMetadata {}
-            .call(
-                admin_request(
-                    Method::GET,
-                    Uri::from_static("/rustfs/admin/v3/export-bucket-metadata?diagnostic=true"),
-                    Vec::new(),
-                ),
-                Params::new(),
-            )
-            .await
-            .expect("diagnostic export after recovery");
-        let bytes = response
-            .output
-            .1
-            .collect()
-            .await
-            .expect("read complete diagnostic archive")
-            .to_bytes();
-        let mut archive = ZipArchive::new(Cursor::new(&bytes)).expect("open complete diagnostic archive");
-        let manifest: serde_json::Value = serde_json::from_reader(
-            archive
-                .by_name(DIAGNOSTIC_EXPORT_MANIFEST)
-                .expect("complete diagnostic manifest"),
-        )
-        .expect("parse complete diagnostic manifest");
-        assert_eq!(manifest["complete"], true);
-        let error = ImportBucketMetadata {}
-            .call(
-                admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), bytes.to_vec()),
-                Params::new(),
-            )
-            .await
-            .expect_err("complete diagnostics must still reject import");
-        assert_eq!(*error.code(), s3s::S3ErrorCode::InvalidRequest);
-
-        let response = ExportBucketMetadata {}
-            .call(
-                admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
-                Params::new(),
-            )
-            .await
-            .expect("ordinary cluster export must work after API recovery");
-        assert!(!response.headers.contains_key("x-rustfs-bucket-metadata-export"));
-        assert_eq!(response.headers[CONTENT_DISPOSITION], "attachment; filename=bucket-meta.zip");
-        let bytes = response.output.1.collect().await.expect("read ordinary archive").to_bytes();
-        let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("open ordinary archive");
-        assert!(archive.by_name(DIAGNOSTIC_EXPORT_MANIFEST).is_err());
-        assert!(archive.by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")).is_ok());
-        assert!(archive.by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")).is_ok());
-    }
-
    #[tokio::test]
    #[serial_test::serial]
    async fn g_zip_001_002_003_use_real_admin_archive_and_persistence_paths() {
@@ -38,6 +38,16 @@ use crate::admin::runtime_sources::{
 };
 use crate::admin::storage_api::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG;
 use crate::admin::storage_api::bucket::metadata_sys;
+use crate::admin::storage_api::bucket::on_demand_migration::backfill::{
+    BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
+};
+use crate::admin::storage_api::bucket::on_demand_migration::source_client::{
+    SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
+};
+use crate::admin::storage_api::bucket::on_demand_migration::{
+    OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
+    source_backend_spec,
+};
 use crate::admin::storage_api::bucket::remote_s3_client::{
    PathStyle as RemotePathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
 };
@@ -48,16 +58,6 @@ use crate::admin::utils::{extract_query_params, read_compatible_admin_body};
 use crate::error::ApiError;
 use crate::license::license_check;
 use crate::module_switches::{ENV_ON_DEMAND_MIGRATION_ENABLED, on_demand_migration_enabled_from_env};
-use crate::on_demand_migration::backfill::{
-    BackfillCheckpoint, BackfillError, BackfillRequest, BackfillState, SkipExisting, global_backfill_runner,
-};
-use crate::on_demand_migration::source_client::{
-    SourceClient, SourceClientSpec, SourceError, SourceProbe, SourceProvider, SourceTimeouts,
-};
-use crate::on_demand_migration::{
-    OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
-    source_backend_spec,
-};
 use crate::server::ADMIN_PREFIX;
 use hyper::{Method, StatusCode};
 use matchit::Params;
@@ -90,8 +90,6 @@ const BACKFILL_OP_CANCEL: &str = "cancel";
 pub(crate) const ERR_CODE_MODULE_DISABLED: &str = "OnDemandMigrationDisabled";
 /// Error code returned when the source bucket did not answer the probe.
 pub(crate) const ERR_CODE_SOURCE_UNREACHABLE: &str = "OnDemandMigrationSourceUnreachable";
-/// Error code returned when the configured provider was excluded at build time.
-pub(crate) const ERR_CODE_BACKEND_NOT_COMPILED: &str = "OnDemandMigrationBackendNotCompiled";
 /// Error code returned by `GET` when the bucket has no configuration.
 pub(crate) const ERR_CODE_NO_SUCH_CONFIGURATION: &str = "NoSuchConfiguration";
 /// Error code (409) returned by `start` while a backfill job holds the lease.
@@ -580,7 +578,7 @@ async fn validate_config(bucket: &str, config: &OnDemandMigrationConfig) -> S3Re
 }

 fn source_provider(config: &OnDemandMigrationConfig) -> SourceProvider {
-    use crate::on_demand_migration::Provider;
+    use crate::admin::storage_api::bucket::on_demand_migration::Provider;
    match config.source.provider {
        Provider::S3 => SourceProvider::S3,
        Provider::Aws => SourceProvider::Aws,
@@ -632,17 +630,12 @@ pub(crate) fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClie
    }
 }

-/// Distinguishes excluded backends from invalid endpoint, CA or credentials.
-/// Anonymous S3 sources are not wired yet
+/// Builder failures are input errors: the endpoint policy, the CA PEM or the
+/// credentials the operator supplied. Anonymous sources are not wired yet
 /// (ODM-05 adds the credential-less path), so `MissingCredentials` is a 400
 /// naming the field instead of an opaque internal error.
 fn client_build_error(err: RemoteS3ClientError) -> S3Error {
    match err {
-        RemoteS3ClientError::BackendNotCompiled(provider) => custom_error(
-            ERR_CODE_BACKEND_NOT_COMPILED,
-            StatusCode::NOT_IMPLEMENTED,
-            format!("the {provider} backend is not included in this build; rebuild with the gcs feature"),
-        ),
        RemoteS3ClientError::MissingCredentials => admin_s3_error(
            S3ErrorCode::InvalidArgument,
            "source.credentials is required: anonymous sources are not supported yet",
@@ -787,7 +780,7 @@ impl Operation for GetBucketOnDemandMigrationHandler {
        let bucket = bucket_from_params(&params)?;
        let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;

-        let Some((config, updated_at)) = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
+        let Some((config, updated_at)) = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
            admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
        })?
        else {
@@ -846,7 +839,7 @@ impl Operation for GetBucketOnDemandMigrationStatusHandler {
        let bucket = bucket_from_params(&params)?;
        let cred = authorize_for_bucket(&req, AdminAction::GetBucketOnDemandMigrationAction, &bucket).await?;

-        let config = crate::on_demand_migration::config::get_config(&bucket).await.map_err(|err| {
+        let config = metadata_sys::get_on_demand_migration_config(&bucket).await.map_err(|err| {
            admin_s3_error(S3ErrorCode::InternalError, format!("failed to read on-demand migration config: {err}"))
        })?;
        let runtime = OnDemandMigrationSys::get().bucket_snapshot(&bucket);
@@ -1295,14 +1288,6 @@ mod tests {
        assert!(err.message().unwrap_or_default().contains("source.credentials"));
    }

-    #[test]
-    fn backend_not_compiled_is_distinct_from_invalid_credentials() {
-        let err = client_build_error(RemoteS3ClientError::BackendNotCompiled("gcs_native"));
-        assert_eq!(err.code(), &S3ErrorCode::Custom(ERR_CODE_BACKEND_NOT_COMPILED.into()));
-        assert_eq!(err.status_code(), Some(StatusCode::NOT_IMPLEMENTED));
-        assert!(err.message().unwrap_or_default().contains("gcs feature"));
-    }
-
    #[test]
    fn module_switch_defaults_on_and_reads_the_env() {
        temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, None::<&str>, || assert!(module_enabled()));
@@ -1763,19 +1748,6 @@ mod store_tests {
                assert_eq!(body["inflight_pulls"], Value::from(0));
                assert_eq!(body["queue_depth"], Value::from(0));

-                // A malformed replacement cannot overwrite the saved source.
-                let before = metadata_sys::get(BUCKET).await.expect("saved metadata");
-                for invalid in [b"not-json".to_vec(), br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec()] {
-                    let err = SetBucketOnDemandMigrationHandler {}
-                        .call(root_request(Method::PUT, config_uri(""), invalid), bucket_params(&router))
-                        .await
-                        .expect_err("malformed replacement must be rejected");
-                    assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
-                    let after = metadata_sys::get(BUCKET).await.expect("saved metadata remains readable");
-                    assert_eq!(after.on_demand_migration_config_json, before.on_demand_migration_config_json);
-                    assert_eq!(after.on_demand_migration_config_updated_at, before.on_demand_migration_config_updated_at);
-                }
-
                // The peer fan-out ran: the single unreachable peer is reported.
                let context = crate::admin::runtime_sources::current_app_context();
                let err = reload_peers(context.as_deref(), BUCKET)
@@ -20,8 +20,8 @@ use time::OffsetDateTime;

 mod ecstore_bucket {
    pub(crate) use crate::storage::storage_api::ecstore_bucket::{
-        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, quota, remote_s3_client,
-        replication, target, utils, versioning, versioning_sys,
+        bandwidth, bucket_target_sys, durability, lifecycle, metadata, metadata_sys, object_lock, on_demand_migration, quota,
+        remote_s3_client, replication, target, utils, versioning, versioning_sys,
    };
 }

@@ -284,6 +284,35 @@ pub(crate) mod durability {
    pub(crate) type BucketDurabilityConfig = super::ecstore_bucket::durability::BucketDurabilityConfig;
 }

+pub(crate) mod on_demand_migration {
+    pub(crate) type OdmBucketSnapshot = super::ecstore_bucket::on_demand_migration::OdmBucketSnapshot;
+    pub(crate) type OnDemandMigrationConfig = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfig;
+    pub(crate) type OnDemandMigrationConfigError = super::ecstore_bucket::on_demand_migration::OnDemandMigrationConfigError;
+    pub(crate) type OnDemandMigrationSys = super::ecstore_bucket::on_demand_migration::OnDemandMigrationSys;
+    pub(crate) type PathStyle = super::ecstore_bucket::on_demand_migration::PathStyle;
+    pub(crate) type Provider = super::ecstore_bucket::on_demand_migration::Provider;
+    pub(crate) type ValidationContext<'a> = super::ecstore_bucket::on_demand_migration::ValidationContext<'a>;
+    pub(crate) use super::ecstore_bucket::on_demand_migration::source_backend_spec;
+
+    pub(crate) mod backfill {
+        pub(crate) type BackfillCheckpoint = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillCheckpoint;
+        pub(crate) type BackfillError = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillError;
+        pub(crate) type BackfillRequest = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillRequest;
+        pub(crate) type BackfillState = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillState;
+        pub(crate) type SkipExisting = super::super::ecstore_bucket::on_demand_migration::backfill::SkipExisting;
+        pub(crate) use super::super::ecstore_bucket::on_demand_migration::backfill::global_backfill_runner;
+    }
+
+    pub(crate) mod source_client {
+        pub(crate) type SourceClient = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClient;
+        pub(crate) type SourceClientSpec = super::super::ecstore_bucket::on_demand_migration::source_client::SourceClientSpec;
+        pub(crate) type SourceError = super::super::ecstore_bucket::on_demand_migration::source_client::SourceError;
+        pub(crate) type SourceProbe = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProbe;
+        pub(crate) type SourceProvider = super::super::ecstore_bucket::on_demand_migration::source_client::SourceProvider;
+        pub(crate) type SourceTimeouts = super::super::ecstore_bucket::on_demand_migration::source_client::SourceTimeouts;
+    }
+}
+
 pub(crate) mod remote_s3_client {
    pub(crate) type PathStyle = super::ecstore_bucket::remote_s3_client::PathStyle;
    pub(crate) type RemoteCredentials = super::ecstore_bucket::remote_s3_client::RemoteCredentials;
@@ -426,6 +455,12 @@ pub(crate) mod metadata_sys {
        super::ecstore_bucket::metadata_sys::get_durability_config(bucket).await
    }

+    pub(crate) async fn get_on_demand_migration_config(
+        bucket: &str,
+    ) -> Result<Option<(super::on_demand_migration::OnDemandMigrationConfig, OffsetDateTime)>> {
+        super::ecstore_bucket::metadata_sys::get_on_demand_migration_config(bucket).await
+    }
+
    pub(crate) async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
        super::ecstore_bucket::metadata_sys::get_quota_config(bucket).await
    }
@@ -878,6 +913,7 @@ pub(crate) mod bucket {
    pub(crate) use super::lifecycle;
    pub(crate) use super::metadata;
    pub(crate) use super::metadata_sys;
+    pub(crate) use super::on_demand_migration;
    pub(crate) use super::quota;
    pub(crate) use super::remote_s3_client;
    pub(crate) use super::replication;
@@ -20,7 +20,7 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
 use super::storage_api::bucket_usecase::access::ReqInfo;
 use super::storage_api::bucket_usecase::access::{
    authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
-    prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
+    prepare_list_buckets_iam_authorization, req_info_ref,
 };
 #[cfg(test)]
 use super::storage_api::bucket_usecase::bucket::target::BucketTarget;
@@ -2724,14 +2724,7 @@ impl DefaultBucketUsecase {

    #[instrument(level = "trace", skip(self, req))]
    pub async fn execute_list_objects_v2(&self, req: S3Request<ListObjectsV2Input>) -> S3Result<S3Response<ListObjectsV2Output>> {
-        self.execute_list_objects_v2_inner(req, true).await
-    }
-
-    async fn execute_list_objects_v2_inner(
-        &self,
-        mut req: S3Request<ListObjectsV2Input>,
-        allow_list_through: bool,
-    ) -> S3Result<S3Response<ListObjectsV2Output>> {
+        // warn!("list_objects_v2 req {:?}", &req.input);
        let ListObjectsV2Input {
            bucket,
            continuation_token,
@@ -2742,7 +2735,7 @@ impl DefaultBucketUsecase {
            prefix,
            start_after,
            ..
-        } = req.input.clone();
+        } = req.input;

        let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;

@@ -2757,32 +2750,9 @@ impl DefaultBucketUsecase {
        // The on-demand migration envelope is decoded whether or not this
        // bucket still merges: a token handed out under `list_through` must keep
        // paginating after the policy is turned off (rustfs/backlog#2164).
-        if allow_list_through {
-            prepare_odm_read_generation(&store, &mut req, &bucket).await;
-        }
-        let (merged_token, source_state) = if allow_list_through {
-            (
-                list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?,
-                list_through::list_through_state(&store, &bucket, &req, &params).await?,
-            )
-        } else {
-            (None, None)
-        };
-        let (object_infos, degraded) = match (source_state, merged_token.as_ref()) {
-            (None, Some(token)) if params.max_keys == 0 => {
-                // No source was consulted, so retain every unconsumed side and
-                // the original wire format without spending its progress budget.
-                let is_truncated = !token.local_done || !token.source_done;
-                (
-                    StorageListObjectsV2Info {
-                        is_truncated,
-                        next_continuation_token: params.decoded_continuation_token.clone().filter(|_| is_truncated),
-                        ..Default::default()
-                    },
-                    false,
-                )
-            }
-            (Some(state), _) => {
+        let merged_token = list_through::decode_list_cursor(params.decoded_continuation_token.as_deref())?;
+        let (object_infos, degraded) = match list_through::list_through_state(&bucket, &req.headers) {
+            Some(state) => {
                let outcome = list_through::merged_list_objects_v2(
                    &store,
                    &state,
@@ -2795,12 +2765,12 @@ impl DefaultBucketUsecase {
                .await?;
                (outcome.info, outcome.degraded)
            }
-            (None, _) => {
+            None => {
                let cursor = list_through::local_cursor(params.decoded_continuation_token.as_deref(), merged_token.as_ref());
                match cursor {
                    list_through::LocalListCursor::Exhausted => (StorageListObjectsV2Info::default(), false),
                    list_through::LocalListCursor::Token(token) => {
-                        let mut infos = store
+                        let infos = store
                            .list_objects_v2(
                                &bucket,
                                &params.prefix,
@@ -2813,7 +2783,6 @@ impl DefaultBucketUsecase {
                            )
                            .await
                            .map_err(ApiError::from)?;
-                        list_through::preserve_framed_local_cursor(&mut infos, merged_token.as_ref());
                        (infos, false)
                    }
                }
@@ -2969,9 +2938,7 @@ impl DefaultBucketUsecase {
    #[instrument(level = "debug", skip(self, req))]
    pub async fn execute_list_objects(&self, req: S3Request<ListObjectsInput>) -> S3Result<S3Response<ListObjectsOutput>> {
        let request_marker = req.input.marker.clone();
-        // V1 markers are object keys, so they cannot carry the opaque merged
-        // pagination state used by V2 list-through.
-        let v2_resp = self.execute_list_objects_v2_inner(req.map_input(Into::into), false).await?;
+        let v2_resp = self.execute_list_objects_v2(req.map_input(Into::into)).await?;

        Ok(v2_resp.map_output(|v2| build_list_objects_output(v2, request_marker)))
    }
@@ -15,11 +15,11 @@
 //! GetObject / GetObjectAttributes read path: cold fill, resume, stream tuning.

 use super::*;
-use crate::on_demand_migration::WriteBackBody;
-use crate::on_demand_migration::{
+use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
    BucketOdmState, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, PullError, PullLeader, PullOutcome, PullReason, PullSlot,
    RangeGetPolicy, SourceBody, SourceClient, SourceError, SourceGet, SourceHead, commit_inline, idle_guarded_body,
 };
+use crate::app::storage_api::object_usecase::on_demand_migration::WriteBackBody;
 use rustfs_rio::{TeeOptions, TeePrimary, tee_reader_with_options};
 use tokio_stream::wrappers::ReceiverStream;

@@ -3843,11 +3843,11 @@ impl DefaultObjectUsecase {
        if !odm_get_may_consult_source(opts, part_number) {
            return None;
        }
-        let sys = OnDemandMigrationSys::get();
-        if !sys.is_module_enabled() {
-            return None;
-        }
-        let state = sys.state(bucket).filter(|state| state.matches_prefix(key))?;
+        let lookup = OnDemandMigrationSys::get().resolve(bucket, key)?;
+        let (state, client) = match odm_get_verdict(lookup) {
+            OdmGetVerdict::Fail(err) => return Some(OdmGetOutcome::Respond(Err(err))),
+            OdmGetVerdict::Consult { state, client } => (state, client),
+        };
        let policy = &state.config().policy;
        // The read path reports a latest delete marker as a plain 404, so the
        // marker is classified here, and only where one can exist.
@@ -3861,24 +3861,6 @@ impl DefaultObjectUsecase {
                None => return Some(OdmGetOutcome::RetryLocal),
            }
        }
-        let expected_incarnation = match odm_read_generation(req, bucket) {
-            Ok(Some(incarnation)) => incarnation,
-            Ok(None) => return None,
-            Err(err) => return Some(OdmGetOutcome::Respond(Err(err))),
-        };
-        match store.bucket_incarnation_id(bucket).await {
-            Ok(current) if current == expected_incarnation => {}
-            Ok(_) => return None,
-            Err(err) => return Some(OdmGetOutcome::Respond(Err(ApiError::from(err).into()))),
-        }
-        if !sys.is_module_enabled() {
-            return None;
-        }
-        let lookup = state.filter_incarnation(expected_incarnation)?.resolve_key(key)?;
-        let (state, client) = match odm_get_verdict(lookup) {
-            OdmGetVerdict::Fail(err) => return Some(OdmGetOutcome::Respond(Err(err))),
-            OdmGetVerdict::Consult { state, client } => (state, client),
-        };
        let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
        let reply = odm_get_from_source(&state, client.as_ref(), &req.headers, key, range, request_context).await;
        Some(match reply {
@@ -3913,7 +3895,7 @@ impl DefaultObjectUsecase {
        result
    }

-    async fn execute_get_object_inner(&self, mut req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
+    async fn execute_get_object_inner(&self, req: S3Request<GetObjectInput>) -> S3Result<S3Response<GetObjectOutput>> {
        let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).suppress_event();

        if let Some(context) = &self.context {
@@ -3999,8 +3981,6 @@ impl DefaultObjectUsecase {
                return Self::complete_get_object_error(helper, err);
            }
        };
-        let bucket = req.input.bucket.clone();
-        prepare_odm_read_generation(&store, &mut req, &bucket).await;
        if let Some(request_context_start) = request_context_start {
            rustfs_io_metrics::record_get_object_stage_duration(
                "s3_handler",
@@ -4818,11 +4798,11 @@ pub(super) async fn odm_get_from_source<S: OdmGetSource>(
 #[cfg(test)]
 mod on_demand_migration_tests {
    use super::*;
-    use crate::on_demand_migration::{
+    use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
        BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
        Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
    };
-    use crate::on_demand_migration::{
+    use crate::app::storage_api::object_usecase::on_demand_migration::{
        LocalObject, OdmWriteBack, WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest,
    };
    use async_trait::async_trait;
@@ -4939,7 +4919,7 @@ mod on_demand_migration_tests {
            Err(WriteBackError::Local("multipart is not part of the inline path".to_string()))
        }

-        async fn abort_multipart_upload(&self, _request: &WriteBackRequest, _upload_id: &str) -> Result<(), WriteBackError> {
+        async fn abort_multipart_upload(&self, _bucket: &str, _key: &str, _upload_id: &str) -> Result<(), WriteBackError> {
            Ok(())
        }
    }
@@ -15,7 +15,7 @@
 //! HeadObject path.

 use super::*;
-use crate::on_demand_migration::{
+use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
    BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OnDemandMigrationSys, SourceClient, SourceError, SourceHead,
 };

@@ -146,8 +146,6 @@ impl DefaultObjectUsecase {
    /// `None` means the runtime does not intervene and the caller keeps its
    /// original 404. The source answer is never written back or queued.
    async fn on_demand_migration_head(
-        req: &S3Request<HeadObjectInput>,
-        store: &ECStore,
        bucket: &str,
        key: &str,
        opts: &ObjectOptions,
@@ -156,33 +154,7 @@ impl DefaultObjectUsecase {
        if !odm_request_may_consult_source(opts) {
            return None;
        }
-        let sys = OnDemandMigrationSys::get();
-        if !sys.is_module_enabled() {
-            return None;
-        }
-        let state = sys.state(bucket).filter(|state| state.matches_prefix(key))?;
-        let policy = &state.config().policy;
-        if !odm_policy_admits_miss(policy, miss) {
-            return None;
-        }
-        if policy.head == HeadPolicy::LocalOnly {
-            state.stats().record_request(OdmOp::Head, OdmOutcome::Filtered);
-            return None;
-        }
-        let expected_incarnation = match odm_read_generation(req, bucket) {
-            Ok(Some(incarnation)) => incarnation,
-            Ok(None) => return None,
-            Err(err) => return Some(Err(err)),
-        };
-        match store.bucket_incarnation_id(bucket).await {
-            Ok(current) if current == expected_incarnation => {}
-            Ok(_) => return None,
-            Err(err) => return Some(Err(ApiError::from(err).into())),
-        }
-        if !sys.is_module_enabled() {
-            return None;
-        }
-        let lookup = state.filter_incarnation(expected_incarnation)?.resolve_key(key)?;
+        let lookup = OnDemandMigrationSys::get().resolve(bucket, key)?;
        match odm_head_verdict(lookup, miss) {
            OdmHeadVerdict::Ignore => None,
            OdmHeadVerdict::Fail(err) => Some(Err(err)),
@@ -297,7 +269,7 @@ impl DefaultObjectUsecase {
    }

    #[instrument(level = "debug", skip(self, req))]
-    pub async fn execute_head_object(&self, mut req: S3Request<HeadObjectInput>) -> S3Result<S3Response<HeadObjectOutput>> {
+    pub async fn execute_head_object(&self, req: S3Request<HeadObjectInput>) -> S3Result<S3Response<HeadObjectOutput>> {
        if let Some(context) = &self.context {
            let _ = context.object_store();
        }
@@ -342,8 +314,6 @@ impl DefaultObjectUsecase {
            .await
            .map_err(ApiError::from)?;

-        prepare_odm_read_generation(&store, &mut req, &bucket).await;
-
        // Modification Points: Explicitly handles get_object_info errors, distinguishing between object absence and other errors
        let lookup = store.get_object_info(&bucket, &key, &opts).await;
        // Single classification point for the on-demand migration gate
@@ -377,7 +347,7 @@ impl DefaultObjectUsecase {
                        return result;
                    }
                    if let Some(miss) = odm_miss
-                        && let Some(result) = Self::on_demand_migration_head(&req, &store, &bucket, &key, &opts, miss).await
+                        && let Some(result) = Self::on_demand_migration_head(&bucket, &key, &opts, miss).await
                    {
                        return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await;
                    }
@@ -392,7 +362,7 @@ impl DefaultObjectUsecase {
                // A latest delete marker is a local miss the source may still
                // answer when the bucket policy says so.
                if let Some(miss) = odm_miss
-                    && let Some(result) = Self::on_demand_migration_head(&req, &store, &bucket, &key, &opts, miss).await
+                    && let Some(result) = Self::on_demand_migration_head(&bucket, &key, &opts, miss).await
                {
                    return Self::finish_on_demand_migration_head(&req, &bucket, helper, result?).await;
                }
@@ -668,7 +638,7 @@ impl DefaultObjectUsecase {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::on_demand_migration::{
+    use crate::app::storage_api::object_usecase::bucket::on_demand_migration::{
        BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OdmStateError, OnDemandMigrationConfig, PathStyle, PolicyConfig,
        Provider, SourceConfig, SourceCredentials, SourceErrorPolicy, TlsConfig,
    };
@@ -769,9 +739,11 @@ mod tests {
            user_metadata: HashMap::from([("owner".to_string(), "alice".to_string())]),
            version_id: Some("v1".to_string()),
            storage_class: Some("STANDARD_IA".to_string()),
-            sse: Some(crate::on_demand_migration::source_client::SourceSse::Kms {
-                key_id: Some("key-1".to_string()),
-            }),
+            sse: Some(
+                crate::storage::storage_api::ecstore_bucket::on_demand_migration::source_client::SourceSse::Kms {
+                    key_id: Some("key-1".to_string()),
+                },
+            ),
            is_multipart_etag: true,
            etag_is_opaque: false,
        }
@@ -1083,12 +1055,7 @@ mod tests {
            head: HeadPolicy::LocalOnly,
            ..Default::default()
        });
-        sys.apply_for_incarnation(
-            &bucket,
-            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
-            Some(&cfg),
-        )
-        .await;
+        sys.apply(&bucket, Some(&cfg)).await;
        let state = sys.state(&bucket).expect("bucket runtime installed");

        // Local hit: served locally, the runtime is never entered.
@@ -1144,12 +1111,7 @@ mod tests {
        );

        cfg.policy.respect_local_delete_marker = false;
-        sys.apply_for_incarnation(
-            &bucket,
-            store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation"),
-            Some(&cfg),
-        )
-        .await;
+        sys.apply(&bucket, Some(&cfg)).await;
        let err = Box::pin(usecase.execute_head_object(head_input(&bucket, "present", None)))
            .await
            .expect_err("local_only still answers 404");
@@ -48,8 +48,6 @@ use http::HeaderName;
 /// other internal provenance is written verbatim.
 pub(crate) struct InternalPutContext {
    pub(crate) bucket: String,
-    /// Pins background work to its original bucket across deletion and recreation.
-    pub(crate) expected_bucket_incarnation_id: Option<Uuid>,
    pub(crate) key: String,
    /// Plaintext object length. The single-object path requires it, exactly
    /// like S3 PutObject rejects an unknown `Content-Length`.
@@ -241,7 +239,6 @@ impl DefaultObjectUsecase {
        let start_time = Instant::now();
        let InternalPutContext {
            bucket,
-            expected_bucket_incarnation_id,
            key,
            size,
            expected_md5_hex,
@@ -299,7 +296,6 @@ impl DefaultObjectUsecase {
                principal_id,
                emit_events,
                preserve_delete_marker,
-                expected_bucket_incarnation_id,
            },
        };
        let committed = self
@@ -371,8 +367,6 @@ impl DefaultObjectUsecase {
            .await
            .map_err(ApiError::from)?;

-        opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id;
-
        let dsc = must_replicate_object(
            &ctx.bucket,
            &ctx.key,
@@ -434,10 +428,7 @@ impl DefaultObjectUsecase {
        let bucket = ctx.bucket.as_str();
        let key = ctx.key.as_str();
        let store = self.object_store().ok_or_else(not_initialized)?;
-        let mut opts = ObjectOptions {
-            expected_bucket_incarnation_id: ctx.expected_bucket_incarnation_id,
-            ..Default::default()
-        };
+        let mut opts = ObjectOptions::default();
        let session = store
            .get_multipart_info(bucket, key, upload_id, &opts)
            .await
@@ -551,7 +542,6 @@ impl DefaultObjectUsecase {
        }
        let mut opts =
            get_complete_multipart_upload_opts_with_replication_authorization(&headers, false).map_err(ApiError::from)?;
-        opts.expected_bucket_incarnation_id = ctx.expected_bucket_incarnation_id;
        opts.preserve_etag = ctx.preserve_etag.clone();
        opts.preserve_delete_marker = ctx.preserve_delete_marker;
        let versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
@@ -709,24 +699,10 @@ impl DefaultObjectUsecase {
    }

    /// Discard an internal multipart upload and its staged parts.
-    pub(crate) async fn internal_abort_multipart_upload(
-        &self,
-        bucket: &str,
-        key: &str,
-        upload_id: &str,
-        expected_bucket_incarnation_id: Option<Uuid>,
-    ) -> Result<(), ApiError> {
+    pub(crate) async fn internal_abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), ApiError> {
        let store = self.object_store().ok_or_else(not_initialized)?;
        store
-            .abort_multipart_upload(
-                bucket,
-                key,
-                upload_id,
-                &ObjectOptions {
-                    expected_bucket_incarnation_id,
-                    ..Default::default()
-                },
-            )
+            .abort_multipart_upload(bucket, key, upload_id, &ObjectOptions::default())
            .await
            .map_err(ApiError::from)?;
        rustfs_scanner::record_dirty_usage_bucket(bucket);
@@ -780,7 +756,6 @@ mod tests {
    fn internal_context(bucket: &str, key: &str, body: &[u8]) -> InternalPutContext {
        InternalPutContext {
            bucket: bucket.to_string(),
-            expected_bucket_incarnation_id: None,
            key: key.to_string(),
            size: Some(body.len() as u64),
            expected_md5_hex: Some(md5_hex(body)),
@@ -1163,14 +1138,9 @@ mod tests {
        ))
        .await
        .expect("part of the aborted upload must stage");
-        Box::pin(usecase.internal_abort_multipart_upload(
-            &bucket,
-            &ctx.key,
-            &aborted_upload_id,
-            ctx.expected_bucket_incarnation_id,
-        ))
-        .await
-        .expect("internal abort must succeed");
+        Box::pin(usecase.internal_abort_multipart_upload(&bucket, &ctx.key, &aborted_upload_id))
+            .await
+            .expect("internal abort must succeed");
        let uploads = Box::pin(store.list_multipart_uploads(&bucket, &ctx.key, None, None, None, 100))
            .await
            .expect("list multipart uploads after abort");
@@ -21,8 +21,8 @@ use crate::storage_api::table::get_bucket_metadata;

 use super::storage_api::object_usecase::access::{
    PostObjectRequestMarker, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard, authorize_request,
-    has_bypass_governance_header, load_bucket_generation_from_store, odm_read_generation, prepare_odm_read_generation,
-    recursive_force_delete_is_authorized, replication_request_authorized, req_info_mut, req_info_ref,
+    has_bypass_governance_header, load_bucket_generation_from_store, recursive_force_delete_is_authorized,
+    replication_request_authorized, req_info_mut, req_info_ref,
 };
 #[cfg(test)]
 use super::storage_api::object_usecase::bucket::quota::BucketQuota;
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Zhengchao An	be3c25c36e	Merge branch 'test/distributed-e2e-hardening' into cursor/e2e-pool-tmpfs-fallback-f29d	2026-09-05 23:56:21 +08:00
overtrue	d8580ec970	test(e2e): prove operations overlap data movement	2026-09-05 22:50:00 +08:00
Zhengchao An	034efa8dac	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 22:21:31 +08:00
Cursor Agent	ff88b1d160	ci(e2e): use tmpfs for distributed pool isolation sm-standard-4 is an ARC pod without usable loop devices, so mount -o loop fails with ENOENT before any pool filesystem is attached. Sized tmpfs still gives each pool a distinct st_dev and independent 1G statfs capacity. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-05 12:58:22 +00:00
Zhengchao An	1277c11f04	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 11:57:33 +08:00
RustFS	bdb069c0fd	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 09:29:28 +08:00
RustFS	b65ab747d3	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 09:17:48 +08:00
loverustfs	8c48f184b5	test(e2e): add distributed 4x4 validation	2026-09-05 02:51:03 +08:00