test(e2e): require quota admission on over-limit 4x4 PUT

Do not treat SlowDown or AccessDenied as a hard-quota rejection. Match InvalidRequest/QuotaExceeded only with a Bucket quota exceeded message, matching the single-node quota e2e contract. Co-authored-by: RustFS <hello@rustfs.com>
test(e2e): tighten movement refusal and cluster start fail-fast
2026-09-05 19:55:37 +00:00 · 2026-09-04 16:22:07 +00:00 · 2026-09-04 16:16:41 +00:00 · 2026-09-04 16:04:25 +00:00 · 2026-09-04 15:59:21 +00:00 · 2026-09-04 15:48:48 +00:00
58 changed files with 8433 additions and 1012 deletions
@@ -0,0 +1,2 @@
+sha256-linux=4988bad7f5929152e0744f07393bc5d24aba2726e5daafa0eeca9a8c6a1f5683
+sha256-darwin=4988bad7f5929152e0744f07393bc5d24aba2726e5daafa0eeca9a8c6a1f5683
@@ -69,7 +69,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
 setup = 'ecstore-large-stack'

 [[profile.default.scripts]]
-filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
+filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'

 [[profile.default.scripts]]
@@ -183,6 +183,13 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'

+# 4-node 4-drive distributed Actions suite: each case starts four rustfs
+# processes and up to sixteen data directories. Serialize across nextest's
+# process boundary so several 4x4 clusters never overlap.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^distributed::/)'
+test-group = 'e2e-cluster-nightly'
+
 # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
 # does not cross nextest process boundaries, so keep every Vault-backed test in
 # one group.
@@ -210,7 +217,7 @@ filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_
 setup = 'ecstore-large-stack'

 [[profile.ci.scripts]]
-filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
+filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
 setup = 'ecstore-base-stack'

 [[profile.ci.scripts]]
@@ -526,6 +533,26 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'

+# ---------------------------------------------------------------------------
+# e2e-distributed profile — 4-node 4-disk Actions suite
+# ---------------------------------------------------------------------------
+# Nightly / dispatch lane owned by .github/workflows/e2e-distributed.yml.
+# Each case starts four rustfs processes (and for site replication, two
+# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
+# Serialized via e2e-cluster-nightly. Not a PR merge gate.
+[profile.e2e-distributed]
+default-filter = 'package(e2e_test) & test(/^distributed::/)'
+fail-fast = false
+# Decommission / rebalance cases poll for up to 180s with little stdout.
+slow-timeout = { period = "120s", terminate-after = 6 }
+
+[profile.e2e-distributed.junit]
+path = "junit.xml"
+
+[[profile.e2e-distributed.overrides]]
+filter = 'package(e2e_test)'
+test-group = 'e2e-cluster-nightly'
+
 # ---------------------------------------------------------------------------
 # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
 # ---------------------------------------------------------------------------
@@ -586,6 +613,10 @@ path = "junit.xml"
 #     cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
 #     excluded here because backlog#2213 promotes core heal rebuild coverage to
 #     this merge/main lane while retaining nightly coverage.
+#   * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
+#     replication, quota, observability, expand/decommission/rebalance, site
+#     replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
+#     .github/workflows/e2e-distributed.yml.
 #   * on_demand_migration::interop_test — the ODM-20 provider interoperability
 #     cases, which are meaningless without a source: they run in the dedicated
 #     [profile.e2e-odm-interop] lane below, where the workflow points them at a
@@ -607,6 +638,7 @@ default-filter = """
    package(e2e_test)
    & !test(/^protocols::/)
    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^distributed::/)
    & !test(/^replication_extension_test::/)
    & !test(/^replication_target_matrix_test::/)
    & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
@@ -4,6 +4,11 @@
  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  {
+    "workflow": ".github/workflows/e2e-distributed.yml",
+    "max_age_hours": 36,
+    "never_ran_grace_until": "2026-09-18T00:00:00Z"
+  },
  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
@@ -0,0 +1,140 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 4-node 4-disk distributed e2e lane.
+#
+# Each selected test starts a real localhost cluster via
+# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
+# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. This is not a required
+# merge check: it is the scheduled/dispatch counterpart to the hardware
+# functional chain that currently clones rustfs/auto-testing onto three VMs.
+# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
+
+name: e2e-distributed
+
+on:
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
+        required: false
+        default: ""
+  schedule:
+    # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
+    - cron: "53 5 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
+jobs:
+  distributed:
+    name: Distributed 4-node 4-disk e2e
+    runs-on: sm-standard-4
+    timeout-minutes: 180
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      NO_PROXY: 127.0.0.1,localhost
+      HTTP_PROXY: ""
+      HTTPS_PROXY: ""
+      # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
+      UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
+      UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
+      UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-e2e-distributed
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
+
+      - name: Download pinned previous release
+        env:
+          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
+        run: |
+          set -euo pipefail
+          mkdir -p "$SOURCE_DIR"
+          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
+          curl --fail --location --retry 3 --output "$archive" \
+            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
+          unzip -q "$archive" -d "$SOURCE_DIR"
+          chmod +x "$SOURCE_DIR/rustfs"
+          test -x "$SOURCE_DIR/rustfs"
+          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
+
+      - name: Build rustfs binary
+        run: |
+          cargo build -p rustfs --bins
+          : > target/debug/rustfs.features
+
+      - name: Verify distributed e2e membership
+        env:
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+        run: |
+          cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
+
+      - name: Run distributed 4-node e2e suite
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
+          NEXTEST_FILTER: ${{ github.event.inputs.filter }}
+        run: |
+          set -euo pipefail
+          if [ -n "${NEXTEST_FILTER}" ]; then
+            cargo nextest run --profile e2e-distributed -p e2e_test -E "${NEXTEST_FILTER}" --no-tests=fail
+          else
+            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+          fi
+
+      - name: Upload distributed e2e diagnostics
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-distributed-${{ github.run_number }}
+          path: |
+            target/nextest/e2e-distributed/junit.xml
+            ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+            ${{ runner.temp }}/rustfs-e2e-distributed-logs/
+          retention-days: 7
+          if-no-files-found: warn
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [distributed]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -22,6 +22,7 @@ on:
      - "Continuous Integration"
      - "coverage"
      - "e2e-nightly"
+      - "e2e-distributed"
      - "e2e-s3tests"
      - "Fuzz"
      - "mint"
@@ -1328,9 +1328,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-runtime-api"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9"
+checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223"
 dependencies = [
 "aws-smithy-async",
 "aws-smithy-runtime-api-macros",
@@ -1368,9 +1368,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-types"
-version = "1.6.2"
+version = "1.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
+checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9"
 dependencies = [
 "base64-simd",
 "bytes",
@@ -5103,9 +5103,9 @@ dependencies = [

 [[package]]
 name = "hickory-net"
-version = "0.26.1"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183"
+checksum = "084e7bd6a377435d568f652153e571b50970d7ccc1d1eeec0519f834632287e1"
 dependencies = [
 "async-trait",
 "cfg-if",
@@ -5127,9 +5127,9 @@ dependencies = [

 [[package]]
 name = "hickory-proto"
-version = "0.26.1"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643"
+checksum = "7e2da0694c15b44c6f68a6b05e0233617008c54080e31d6eb848d858a9c5b38d"
 dependencies = [
 "data-encoding",
 "idna",
@@ -5147,9 +5147,9 @@ dependencies = [

 [[package]]
 name = "hickory-resolver"
-version = "0.26.1"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c"
+checksum = "0e4f9f4603319422d482738f3f6fe5aac03157fdbfed1cd85a3ff45adb09072f"
 dependencies = [
 "cfg-if",
 "futures-util",
@@ -5226,9 +5226,9 @@ dependencies = [

 [[package]]
 name = "hotpath"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737"
+checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592"
 dependencies = [
 "arc-swap",
 "async-channel",
@@ -5240,7 +5240,6 @@ dependencies = [
 "futures-util",
 "hdrhistogram",
 "hotpath-macros",
- "hotpath-meta",
 "http 1.5.0",
 "libc",
 "object 0.36.7",
@@ -5260,30 +5259,15 @@ dependencies = [

 [[package]]
 name = "hotpath-macros"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1"
+checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
 dependencies = [
 "proc-macro2",
 "quote",
 "syn 2.0.119",
 ]

-[[package]]
-name = "hotpath-macros-meta"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b"
-
-[[package]]
-name = "hotpath-meta"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b"
-dependencies = [
- "hotpath-macros-meta",
-]
-
 [[package]]
 name = "htmlescape"
 version = "0.3.1"
@@ -5642,18 +5626,6 @@ dependencies = [
 "tempfile",
 ]

-[[package]]
-name = "internal-russh-num-bigint"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8"
-dependencies = [
- "num-integer",
- "num-traits",
- "rand 0.10.2",
- "rand_core 0.10.1",
-]
-
 [[package]]
 name = "io-uring"
 version = "0.7.14"
@@ -5904,17 +5876,20 @@ dependencies = [

 [[package]]
 name = "kafka-protocol"
-version = "0.17.0"
+version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2"
+checksum = "099d5c2f1b40cd830cbf18ca4d2a0805f2875b811ca18f0372b2433e68fe2dda"
 dependencies = [
 "anyhow",
 "bytes",
 "crc",
 "crc32c",
+ "flate2",
 "indexmap 2.14.1",
- "paste",
+ "lz4",
+ "snap",
 "uuid",
+ "zstd",
 ]

 [[package]]
@@ -6923,6 +6898,8 @@ checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
 dependencies = [
 "num-integer",
 "num-traits",
+ "rand 0.10.2",
+ "rand_core 0.10.1",
 ]

 [[package]]
@@ -7515,9 +7492,9 @@ dependencies = [

 [[package]]
 name = "pageant"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3adadc44070da6f464b0918655a12f5792c156e088d8c4082d13e27d94c3e791"
+checksum = "6d8eab09a361a4afe0b1668be978cd97e4f052e927a92b0b608cf902965d49ce"
 dependencies = [
 "base16ct 1.0.0",
 "byteorder",
@@ -7606,12 +7583,6 @@ dependencies = [
 "phc",
 ]

-[[package]]
-name = "paste"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
-
 [[package]]
 name = "path-absolutize"
 version = "4.0.1"
@@ -9250,9 +9221,9 @@ dependencies = [

 [[package]]
 name = "russh"
-version = "0.63.1"
+version = "0.63.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35bab1b87d915817d5d9cc352637cd40d5f0b298a48c6309af9156a4addc3031"
+checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd"
 dependencies = [
 "aes 0.9.3",
 "aws-lc-rs",
@@ -9281,13 +9252,12 @@ dependencies = [
 "hex-literal",
 "hmac 0.13.0",
 "inout 0.2.2",
- "internal-russh-num-bigint",
 "keccak",
 "log",
 "md5",
 "ml-kem",
 "module-lattice",
- "num-bigint 0.4.8",
+ "num-bigint 0.5.1",
 "p256 0.14.0",
 "p384 0.14.0",
 "p521",
@@ -9978,11 +9948,11 @@ dependencies = [

 [[package]]
 name = "rustfs-kafka"
-version = "1.2.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4eee0644a99743fb2f51db7fbae1a6ca2d85f064daf7595784eaa52834a68c96"
+checksum = "759f8ddf709b497006e0f422a89890f11ac86c84c2c39df282891477c22f1a13"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
 "bytes",
 "fnv",
 "hmac 0.13.0",
@@ -9992,22 +9962,22 @@ dependencies = [
 "pbkdf2 0.13.0",
 "rand 0.10.2",
 "rustls",
- "rustls-native-certs",
 "sha2 0.11.0",
 "socket2",
 "thiserror 2.0.20",
 "tracing",
 "twox-hash",
+ "uuid",
 "webpki-roots 1.0.9",
 ]

 [[package]]
 name = "rustfs-kafka-async"
-version = "1.2.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cd1997c3116cb94ede80d9a0b828f46dd27386cc93825fce141a92eb3aa9630"
+checksum = "5aa574a13d91ef529e3a08f059a5c3ee8d4c3b16cf62f62c72595a6dc4f79f8f"
 dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
 "bytes",
 "hmac 0.13.0",
 "kafka-protocol",
@@ -10016,11 +9986,11 @@ dependencies = [
 "rand 0.10.2",
 "rustfs-kafka",
 "rustls",
- "rustls-native-certs",
 "sha2 0.11.0",
 "tokio",
 "tokio-rustls",
 "tracing",
+ "uuid",
 "webpki-roots 1.0.9",
 ]

@@ -10194,18 +10164,18 @@ dependencies = [

 [[package]]
 name = "rustfs-mimalloc"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85d1a75bd188260754c4fa8ace1bb1f6a9956ee36c1d330f98fed5d25e39fa15"
+checksum = "46a7b69356718defa4060de3059e609c9da6619c35924fefef55d902ccd94958"
 dependencies = [
 "rustfs-mimalloc-sys",
 ]

 [[package]]
 name = "rustfs-mimalloc-sys"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d3adbf24cbe37f040f856c04f8d4baf75626eacd4e8b17eb6b3d79de7fde692"
+checksum = "1dd3ec9b7e7b9fed453acd8b4e32713ddc0e01e1aee31a3deec5b8025c0880c3"
 dependencies = [
 "cc",
 ]
@@ -12301,7 +12271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
 "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
 "once_cell",
 "rustix",
 "windows-sys 0.61.2",
@@ -12485,9 +12455,9 @@ dependencies = [

 [[package]]
 name = "tinyvec"
-version = "1.12.0"
+version = "1.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
+checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b"
 dependencies = [
 "tinyvec_macros",
 ]
@@ -13652,13 +13622,14 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"

 [[package]]
 name = "wnaf"
-version = "0.14.0"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab12e7090f27e2ffd9322651492942d50c2926094af30601e1964337db39daf1"
+checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa"
 dependencies = [
 "ff 0.14.0",
 "group 0.14.0",
 "hybrid-array",
+ "primefield",
 ]

 [[package]]
@@ -165,7 +165,7 @@ http-body = "1.1.0"
 http-body-util = "0.1.5"
 minlz = "1.2.3"
 reqwest = "0.13.4"
-rustfs-kafka-async = { version = "1.2.0" }
+rustfs-kafka-async = { version = "1.3.1" }
 socket2 = { version = "0.6.5" }
 tokio = { version = "1.53.1" }
 tokio-rustls = { default-features = false, version = "0.26.4" }
@@ -244,8 +244,8 @@ aws-sdk-kms = { default-features = false, version = "1.117.0" }
 aws-sdk-s3 = { default-features = false, version = "1.144.0" }
 aws-sdk-sts = { default-features = false, version = "1.113.0" }
 aws-smithy-http-client = { default-features = false, version = "1.4.0" }
-aws-smithy-runtime-api = { version = "1.15.0" }
-aws-smithy-types = { version = "1.6.2" }
+aws-smithy-runtime-api = { version = "1.16.0" }
+aws-smithy-types = { version = "1.6.3" }
 base64-simd = "0.8.0"
 brotli = "9.0.0"
 clap = { version = "4.6.6" }
@@ -359,15 +359,15 @@ libunftp = { version = "0.23.0" }
 unftp-core = "0.1.0"
 suppaftp = { version = "11.0.0" }
 rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
-russh = { version = "0.63.1" }
+russh = { version = "0.63.2" }
 russh-sftp = "2.4.0"

 # WebDAV
 dav-server = "0.11.0"

 # Performance Analysis and Memory Profiling
-rustfs-mimalloc = { version = "0.5.2" }
-hotpath = { version = "0.24.0", default-features = false }
+rustfs-mimalloc = { version = "0.5.3" }
+hotpath = { version = "0.25.0", default-features = false }
 # Snapshot testing for output format regression detection
 insta = { version = "1.48" }

@@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |

@@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker.
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
 | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch; not a merge gate) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
+# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
+# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
+cargo nextest run --profile e2e-distributed -p e2e_test
 # Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
@@ -1483,8 +1483,9 @@ impl RustFSTestClusterEnvironment {
            self.spawn_node(node_idx, binary_path, &volumes_arg)?;
        }

-        for (i, node) in self.nodes.iter().enumerate() {
-            self.wait_for_node_ready(&node.address, i).await?;
+        for i in 0..self.nodes.len() {
+            let address = self.nodes[i].address.clone();
+            self.wait_for_node_ready(&address, i).await?;
        }

        for node_idx in 0..self.nodes.len() {
@@ -1510,7 +1511,8 @@ impl RustFSTestClusterEnvironment {
        let volumes_arg = self.build_volumes_arg();
        self.spawn_node(node_idx, binary_path, &volumes_arg)?;

-        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
+        let address = self.nodes[node_idx].address.clone();
+        self.wait_for_node_ready(&address, node_idx).await?;
        self.wait_for_node_service_ready(node_idx).await?;
        Ok(())
    }
@@ -1559,8 +1561,18 @@ impl RustFSTestClusterEnvironment {
    ///
    /// Attempts to establish a TCP connection to the node's address, retries up to 60 times
    /// with a 1-second interval between attempts. Fails if the port is unreachable after all retries.
-    async fn wait_for_node_ready(&self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    fn node_process_exited(&mut self, idx: usize) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+        let Some(process) = self.nodes.get_mut(idx).and_then(|node| node.process.as_mut()) else {
+            return Ok(true);
+        };
+        Ok(process.try_wait()?.is_some())
+    }
+
+    async fn wait_for_node_ready(&mut self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        for attempt in 0..60 {
+            if self.node_process_exited(idx)? {
+                return Err(format!("cluster node {idx} process exited before TCP ready").into());
+            }
            if TcpStream::connect(address).await.is_ok() {
                info!("Node {} ({}) TCP ready after {} attempts", idx, address, attempt + 1);
                return Ok(());
@@ -1574,10 +1586,13 @@ impl RustFSTestClusterEnvironment {
    ///
    /// Verifies service availability by calling the S3 `list_buckets` API against the requested node,
    /// retries up to 120 times with a 1-second interval between attempts.
-    async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    async fn wait_for_node_service_ready(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
        let client = self.create_s3_client(node_idx)?;

        for attempt in 0..120 {
+            if self.node_process_exited(node_idx)? {
+                return Err(format!("cluster node {node_idx} process exited before S3 ready").into());
+            }
            match client.list_buckets().send().await {
                Ok(_) => {
                    info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);
@@ -1700,6 +1715,64 @@ impl RustFSTestClusterEnvironment {
        Ok(())
    }

+    /// Append a new single-node erasure pool to a stopped multi-pool cluster.
+    ///
+    /// Used to simulate pool expansion on localhost: every pool already owns
+    /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
+    /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
+    /// allocated a fresh port and empty drive directories; callers must
+    /// [`Self::start`] afterwards so every process picks up the extended
+    /// volumes argument. Existing data directories are left untouched.
+    pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+        if self.nodes.iter().any(|node| node.process.is_some()) {
+            return Err("stop the cluster before appending a pool".into());
+        }
+        if self.topology.drives_per_node < 2 {
+            return Err(
+                "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
+                    .into(),
+            );
+        }
+
+        let mut pools = self.topology.normalized_pools();
+        for (pool_idx, nodes) in pools.iter().enumerate() {
+            if nodes.len() != 1 {
+                return Err(format!(
+                    "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
+                    nodes.len()
+                )
+                .into());
+            }
+        }
+
+        let new_idx = self.nodes.len();
+        let port = RustFSTestEnvironment::find_available_port().await?;
+        let address = format!("127.0.0.1:{port}");
+        let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
+            .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
+            .collect();
+        for dir in &data_dirs {
+            fs::create_dir_all(dir).await?;
+        }
+
+        self.nodes.push(ClusterNode {
+            url: format!("http://{address}"),
+            address,
+            data_dir: data_dirs[0].clone(),
+            data_dirs,
+            pool_idx: pools.len(),
+            process: None,
+        });
+        pools.push(vec![new_idx]);
+        self.topology.node_count = self.nodes.len();
+        self.topology.pools = pools;
+        self.node_extra_env.push(Vec::new());
+        self.node_capture_log_paths.push(None);
+        self.volume_proxy_addresses.push(None);
+
+        Ok(new_idx)
+    }
+
    /// Gracefully stop one cluster node and wait for its process to exit.
    ///
    /// This is intentionally separate from [`Self::stop_node`]: the latter is
@@ -0,0 +1,108 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, bring_drive_online, put_object, retrying_get_equals,
+    take_drive_offline, unique_bucket, wait_for_ready,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn kill_and_restart_node_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("killnode");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x11u8; 128 * 1024];
+    put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
+
+    dist.cluster.stop_node(3)?;
+    retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn full_cluster_restart_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("pwr");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x44u8; 64 * 1024];
+    put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn offline_drive_then_replace_keeps_object_readable() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("baddrive");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x22u8; 96 * 1024];
+    put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
+
+    take_drive_offline(&dist.cluster, 0, 0)?;
+    retrying_get_equals(&dist.client(2)?, &bucket, "durable.bin", &body, Duration::from_secs(20)).await?;
+    bring_drive_online(&dist.cluster, 0, 0)?;
+    retrying_get_equals(&dist.client(3)?, &bucket, "durable.bin", &body, Duration::from_secs(20)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("getkill");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x7Au8; 96 * 1024];
+    put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
+
+    let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
+    let start = Arc::new(Barrier::new(13));
+    let mut handles = Vec::new();
+    for idx in 0..12 {
+        let client = live[idx % live.len()].clone();
+        let bucket = bucket.clone();
+        let body = body.clone();
+        let start = start.clone();
+        handles.push(tokio::spawn(async move {
+            start.wait().await;
+            retrying_get_equals(&client, &bucket, "steady.bin", &body, Duration::from_secs(20)).await
+        }));
+    }
+    start.wait().await;
+    dist.cluster.stop_node(3)?;
+    for handle in handles {
+        handle.await??;
+    }
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
+    Ok(())
+}
@@ -0,0 +1,57 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use crate::common::init_logging;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn four_node_high_concurrency_puts_are_readable_from_every_node() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("conc");
+    dist.create_bucket(&bucket).await?;
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(32));
+
+    let mut handles = Vec::new();
+    for idx in 0..32 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("c/{idx:02}.bin");
+            let body = payload_for(&key, 16 * 1024);
+            put_object(client, &bucket, &key, body.clone()).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    let mut inventory = Vec::new();
+    for handle in handles {
+        inventory.push(handle.await??);
+    }
+
+    for (node_idx, client) in clients.iter().enumerate() {
+        for (key, body) in &inventory {
+            assert_object_bytes(client, &bucket, key, body)
+                .await
+                .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
+        }
+    }
+    Ok(())
+}
@@ -0,0 +1,67 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, payload_for, put_inventory_retrying,
+    retrying_get_equals, retrying_put, unique_bucket, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("concdecom");
+    dist.create_bucket(&bucket).await?;
+    let baseline_client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&baseline_client, &bucket, 10, 24 * 1024, Duration::from_secs(30)).await?;
+
+    let decommission_started = decommission_started_or_refused(&dist.cluster, 0).await?;
+
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(16));
+    let mut handles = Vec::new();
+    for idx in 0..16 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("live/{idx:02}.bin");
+            let body = payload_for(&key, 8 * 1024);
+            retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    let mut live_objects = Vec::new();
+    for handle in handles {
+        live_objects.push(handle.await??);
+    }
+
+    if decommission_started {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+
+    let checker = dist.client(3)?;
+    assert_inventory(&checker, &bucket, &inventory).await?;
+    for (key, body) in live_objects {
+        retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,44 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, put_inventory_retrying, sha256_hex,
+    unique_bucket, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn decommission_attempt_does_not_alter_object_sha256() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("integrity");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 20, 64 * 1024, Duration::from_secs(30)).await?;
+    let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
+
+    if decommission_started_or_refused(&dist.cluster, 0).await? {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+
+    let after_client = dist.client(2)?;
+    assert_inventory(&after_client, &bucket, &inventory).await?;
+    for (key, expected_hash) in before {
+        let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
+        let body = got.body.collect().await?.into_bytes();
+        assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
+    }
+    Ok(())
+}
@@ -0,0 +1,71 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, list_pools_json, put_inventory,
+    rebalance_started_or_refused, unique_bucket, wait_for_decommission_complete, wait_for_rebalance_idle,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_restart_preserves_objects_then_rebalance_attempt() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("expand");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory(&client, &bucket, 12, 32 * 1024).await?;
+    assert_inventory(&client, &bucket, &inventory).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+
+    let after_restart = dist.client(0)?;
+    assert_inventory(&after_restart, &bucket, &inventory).await?;
+    let peer = dist.client(3)?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+
+    if rebalance_started_or_refused(&dist.cluster).await? {
+        wait_for_rebalance_idle(&dist.cluster, Duration::from_secs(90)).await?;
+    }
+    assert_inventory(&peer, &bucket, &inventory).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_decommission_attempt_does_not_lose_objects() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("decom");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(1)?;
+    let inventory = put_inventory(&client, &bucket, 16, 48 * 1024).await?;
+
+    let pools_before = list_pools_json(&dist.cluster).await?;
+    let pool_count = pools_before
+        .as_array()
+        .map(Vec::len)
+        .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
+        .unwrap_or(1);
+    assert!(pool_count >= 1, "expected at least one pool before decommission: {pools_before}");
+
+    if decommission_started_or_refused(&dist.cluster, 0).await? {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+
+    let after = dist.client(3)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,149 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("extra");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+
+    let key = "multipart.bin";
+    let part1 = vec![0x41u8; 5 * 1024 * 1024];
+    let part2 = vec![0x42u8; 5 * 1024 * 1024];
+    let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
+
+    let uploaded1 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(part1.clone()))
+        .send()
+        .await?;
+    let uploaded2 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(2)
+        .body(ByteStream::from(part2.clone()))
+        .send()
+        .await?;
+
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded1.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded2.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let mut expected = part1;
+    expected.extend_from_slice(&part2);
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
+    }
+
+    put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
+    put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
+    let mut seen = Vec::new();
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let listed = dist
+            .client(node_idx)?
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("list/")
+            .send()
+            .await?;
+        let keys: Vec<String> = listed
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        seen.push(keys);
+    }
+    for keys in &seen[1..] {
+        assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
+    }
+
+    let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
+    assert_eq!(got, b"a");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("listed");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node_idx)?;
+        let name = bucket.clone();
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = client.clone();
+                let name = name.clone();
+                async move {
+                    let listed = client.list_buckets().send().await?;
+                    Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
+                }
+            },
+            &format!("node {node_idx} lists {bucket}"),
+        )
+        .await?;
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = dist.client(node_idx).expect("client");
+                let name = bucket.clone();
+                async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
+            },
+            &format!("node {node_idx} reads seed.bin"),
+        )
+        .await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,933 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared 4-node distributed e2e helpers.
+//!
+//! Two localhost-expressible layouts cover the suite:
+//!
+//! * **4×4 single pool** (`four_by_four`) — four processes, four drives each,
+//!   one `DistErasure` pool (16 explicit volume endpoints). This is the
+//!   default S3 / lock / versioning / chaos topology.
+//! * **4×4 four pool** — `append_single_node_pool` exists for harness unit
+//!   tests. Live expand-then-restart currently hits `pool metadata recovery
+//!   required` on localhost DistErasure. That is a production bootstrap-proof
+//!   limitation this test lane does not change. Movement tests use 4×4 single
+//!   pool and classify decommission/rebalance product refusals (and opaque
+//!   500 InternalError) as a refused move while still asserting object bytes.
+//!
+//! Genuine multi-node *striped* pools still need multi-host CI (backlog
+//! #1313 / #1314). Site replication uses two 4-node 1-drive clusters so the
+//! process count stays at eight rather than sixteen.
+
+use crate::common::{
+    ClusterTopology, FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, admin_request, build_test_s3_config,
+    local_http_client, replication_fast_env, signed_request,
+};
+use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+use http::{Method, StatusCode};
+use sha2::{Digest, Sha256};
+use std::collections::BTreeMap;
+use std::path::Path;
+use std::time::Duration;
+use tokio::time::{Instant, sleep};
+use uuid::Uuid;
+
+pub(crate) type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
+
+pub(crate) const NODE_COUNT: usize = 4;
+pub(crate) const DRIVES_PER_NODE: usize = 4;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum DistLayout {
+    /// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
+    FourByFour,
+    /// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
+    FourNodeFourDisk,
+}
+
+pub(crate) struct DistCluster {
+    pub cluster: RustFSTestClusterEnvironment,
+}
+
+impl DistCluster {
+    pub async fn start(layout: DistLayout) -> TestResult<Self> {
+        Self::start_with_env(layout, &[]).await
+    }
+
+    pub async fn start_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
+        let mut dist = Self::new_stopped_with_env(layout, extra_env).await?;
+        dist.cluster.start().await?;
+        Ok(dist)
+    }
+
+    /// Allocate ports and data dirs without spawning processes.
+    ///
+    /// Upgrade tests configure capture logs, then start a pinned previous
+    /// binary against the same directories.
+    pub async fn new_stopped(layout: DistLayout) -> TestResult<Self> {
+        Self::new_stopped_with_env(layout, &[]).await
+    }
+
+    pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
+        let topology = match layout {
+            DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
+            DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
+        };
+        let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
+        cluster.set_env("NO_PROXY", "127.0.0.1,localhost");
+        cluster.set_env("HTTP_PROXY", "");
+        cluster.set_env("HTTPS_PROXY", "");
+        for &(key, value) in extra_env {
+            cluster.set_env(key, value);
+        }
+        Ok(Self { cluster })
+    }
+
+    /// Start every node with a specific `rustfs` binary, keeping the allocated
+    /// data directories. Used to seed an old on-disk format before upgrading.
+    pub async fn start_from_binary(&mut self, binary: &Path) -> TestResult {
+        self.cluster.start_with_binary(binary).await?;
+        wait_for_ready(&self.cluster).await?;
+        Ok(())
+    }
+
+    /// Stop every node and bring the same data directories up on the workspace
+    /// binary (direct upgrade).
+    pub async fn restart_with_current_binary(&mut self) -> TestResult {
+        self.cluster.stop();
+        self.cluster.start().await?;
+        wait_for_ready(&self.cluster).await?;
+        Ok(())
+    }
+
+    /// Replace one running node with the workspace binary (rolling upgrade).
+    pub async fn replace_node_with_current_binary(&mut self, node_idx: usize) -> TestResult {
+        self.cluster.stop_node(node_idx)?;
+        self.cluster.start_node(node_idx).await?;
+        wait_for_ready(&self.cluster).await?;
+        Ok(())
+    }
+
+    pub fn client_with_credentials(&self, node_idx: usize, access_key: &str, secret_key: &str) -> TestResult<Client> {
+        if node_idx >= self.cluster.nodes.len() {
+            return Err("node_idx is invalid".into());
+        }
+        Ok(Client::from_conf(build_test_s3_config(
+            &self.cluster.nodes[node_idx].url,
+            access_key,
+            secret_key,
+            None,
+            "cluster-iam-test",
+        )))
+    }
+
+    pub async fn start_replication_pair() -> TestResult<(Self, Self)> {
+        let mut extra: Vec<(&str, &str)> = replication_fast_env();
+        extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+        extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV);
+        let source = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
+        let target = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
+        Ok((source, target))
+    }
+
+    pub fn client(&self, node_idx: usize) -> TestResult<Client> {
+        self.cluster.create_s3_client(node_idx)
+    }
+
+    pub fn clients(&self) -> TestResult<Vec<Client>> {
+        self.cluster.create_all_clients()
+    }
+
+    pub async fn create_bucket(&self, bucket: &str) -> TestResult {
+        self.cluster.create_test_bucket(bucket).await
+    }
+}
+
+pub(crate) fn unique_bucket(prefix: &str) -> String {
+    let id = Uuid::new_v4().simple().to_string();
+    format!("{prefix}-{}", &id[..12])
+}
+
+pub(crate) fn sha256_hex(bytes: &[u8]) -> String {
+    let digest = Sha256::digest(bytes);
+    digest.iter().map(|byte| format!("{byte:02x}")).collect()
+}
+
+pub(crate) fn payload_for(key: &str, size: usize) -> Vec<u8> {
+    let seed = key.as_bytes();
+    (0..size)
+        .map(|idx| seed.get(idx % seed.len()).copied().unwrap_or(0) ^ (idx as u8))
+        .collect()
+}
+
+pub(crate) async fn put_object(client: &Client, bucket: &str, key: &str, body: Vec<u8>) -> TestResult {
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(body))
+        .send()
+        .await?;
+    Ok(())
+}
+
+pub(crate) async fn get_object_bytes(client: &Client, bucket: &str, key: &str) -> TestResult<Vec<u8>> {
+    let output = client.get_object().bucket(bucket).key(key).send().await?;
+    Ok(output.body.collect().await?.into_bytes().to_vec())
+}
+
+pub(crate) async fn assert_object_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let got = get_object_bytes(client, bucket, key).await?;
+    if got.as_slice() != expected {
+        return Err(format!(
+            "object {bucket}/{key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            got.len(),
+            sha256_hex(&got)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+pub(crate) async fn put_inventory(
+    client: &Client,
+    bucket: &str,
+    count: usize,
+    size: usize,
+) -> TestResult<BTreeMap<String, Vec<u8>>> {
+    let mut inventory = BTreeMap::new();
+    for idx in 0..count {
+        let key = format!("obj-{idx:04}");
+        let body = payload_for(&key, size);
+        put_object(client, bucket, &key, body.clone()).await?;
+        inventory.insert(key, body);
+    }
+    Ok(inventory)
+}
+
+/// Localhost DistErasure can 500 a PUT while heal_bucket hits a pool-meta
+/// write fence. Retry only those transient codes.
+pub(crate) async fn put_inventory_retrying(
+    client: &Client,
+    bucket: &str,
+    count: usize,
+    size: usize,
+    timeout: Duration,
+) -> TestResult<BTreeMap<String, Vec<u8>>> {
+    let mut inventory = BTreeMap::new();
+    for idx in 0..count {
+        let key = format!("obj-{idx:04}");
+        let body = payload_for(&key, size);
+        retrying_put(client, bucket, &key, body.clone(), timeout).await?;
+        inventory.insert(key, body);
+    }
+    Ok(inventory)
+}
+
+pub(crate) async fn assert_inventory(client: &Client, bucket: &str, inventory: &BTreeMap<String, Vec<u8>>) -> TestResult {
+    for (key, expected) in inventory {
+        assert_object_bytes(client, bucket, key, expected).await?;
+    }
+    Ok(())
+}
+
+pub(crate) async fn enable_versioning(client: &Client, bucket: &str) -> TestResult {
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+    Ok(())
+}
+
+pub(crate) async fn wait_until<F, Fut>(timeout: Duration, mut probe: F, label: &str) -> TestResult
+where
+    F: FnMut() -> Fut,
+    Fut: std::future::Future<Output = TestResult<bool>>,
+{
+    let deadline = Instant::now() + timeout;
+    let mut delay = Duration::from_millis(50);
+    loop {
+        let last_error = match probe().await {
+            Ok(true) => return Ok(()),
+            Ok(false) => format!("{label} still false"),
+            Err(error) => error.to_string(),
+        };
+        if Instant::now() >= deadline {
+            return Err(format!("{label} did not become true within {timeout:?}: {last_error}").into());
+        }
+        sleep(delay).await;
+        delay = (delay * 2).min(Duration::from_secs(1));
+    }
+}
+
+pub(crate) async fn cluster_admin(
+    cluster: &RustFSTestClusterEnvironment,
+    method: Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> TestResult<(StatusCode, String)> {
+    admin_request(
+        &cluster.nodes[0].url,
+        method,
+        path_and_query,
+        body,
+        &cluster.access_key,
+        &cluster.secret_key,
+    )
+    .await
+}
+
+pub(crate) async fn cluster_admin_ok(
+    cluster: &RustFSTestClusterEnvironment,
+    method: Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> TestResult<String> {
+    let (status, response) = cluster_admin(cluster, method.clone(), path_and_query, body).await?;
+    if !status.is_success() {
+        return Err(format!("{method} {path_and_query} failed: {status} {response}").into());
+    }
+    Ok(response)
+}
+
+pub(crate) async fn wait_for_ready(cluster: &RustFSTestClusterEnvironment) -> TestResult {
+    let client = local_http_client();
+    for node in &cluster.nodes {
+        let url = format!("{}/health/ready", node.url);
+        wait_until(
+            Duration::from_secs(30),
+            || {
+                let client = client.clone();
+                let url = url.clone();
+                async move {
+                    match client.get(&url).send().await {
+                        Ok(response) if response.status().is_success() => Ok(true),
+                        _ => Ok(false),
+                    }
+                }
+            },
+            &format!("node {} ready", node.address),
+        )
+        .await?;
+    }
+    Ok(())
+}
+
+pub(crate) fn take_drive_offline(
+    cluster: &RustFSTestClusterEnvironment,
+    node_idx: usize,
+    drive_idx: usize,
+) -> TestResult<String> {
+    let dir = cluster
+        .nodes
+        .get(node_idx)
+        .and_then(|node| node.data_dirs.get(drive_idx))
+        .ok_or("invalid node/drive index")?;
+    let offline = format!("{dir}.offline");
+    if Path::new(&offline).exists() {
+        return Err(format!("drive already offline: {offline}").into());
+    }
+    std::fs::rename(dir, &offline)?;
+    Ok(offline)
+}
+
+pub(crate) fn bring_drive_online(cluster: &RustFSTestClusterEnvironment, node_idx: usize, drive_idx: usize) -> TestResult {
+    let dir = cluster
+        .nodes
+        .get(node_idx)
+        .and_then(|node| node.data_dirs.get(drive_idx))
+        .ok_or("invalid node/drive index")?;
+    let offline = format!("{dir}.offline");
+    if Path::new(dir).exists() {
+        std::fs::remove_dir_all(dir)?;
+    }
+    std::fs::rename(&offline, dir)?;
+    Ok(())
+}
+
+pub(crate) async fn set_remote_target(
+    source: &RustFSTestClusterEnvironment,
+    source_bucket: &str,
+    target: &RustFSTestClusterEnvironment,
+    target_bucket: &str,
+) -> TestResult<String> {
+    let body = serde_json::json!({
+        "endpoint": target.nodes[0].address,
+        "credentials": {
+            "accessKey": target.access_key,
+            "secretKey": target.secret_key
+        },
+        "targetbucket": target_bucket,
+        "secure": false,
+        "type": "replication"
+    });
+    let url = format!(
+        "{}/rustfs/admin/v3/set-remote-target?bucket={}",
+        source.nodes[0].url,
+        urlencoding::encode(source_bucket)
+    );
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &source.access_key,
+        &source.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("set remote target failed: {status} {body}").into());
+    }
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+pub(crate) async fn put_bucket_replication(source: &RustFSTestClusterEnvironment, bucket: &str, target_arn: &str) -> TestResult {
+    let body = format!(
+        r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Role></Role>
+  <Rule>
+    <ID>rule-1</ID>
+    <Priority>1</Priority>
+    <Status>Enabled</Status>
+    <DeleteMarkerReplication>
+      <Status>Enabled</Status>
+    </DeleteMarkerReplication>
+    <ExistingObjectReplication>
+      <Status>Enabled</Status>
+    </ExistingObjectReplication>
+    <Destination>
+      <Bucket>{target_arn}</Bucket>
+    </Destination>
+  </Rule>
+</ReplicationConfiguration>"#
+    );
+    let url = format!("{}/{bucket}?replication", source.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &source.access_key,
+        &source.secret_key,
+        Some(body.into_bytes()),
+        Some("application/xml"),
+    )
+    .await?;
+    if !response.status().is_success() {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("put bucket replication failed: {status} {body}").into());
+    }
+    Ok(())
+}
+
+pub(crate) async fn wait_for_replicated_bytes(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    expected: &[u8],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            match get_object_bytes(client, bucket, key).await {
+                Ok(got) if got.as_slice() == expected => Ok(true),
+                Ok(_) => Ok(false),
+                Err(error) => {
+                    let message = error.to_string();
+                    if message.contains("NoSuchKey") || message.contains("NotFound") {
+                        Ok(false)
+                    } else {
+                        Err(error)
+                    }
+                }
+            }
+        },
+        &format!("replicated object {bucket}/{key}"),
+    )
+    .await
+}
+
+pub(crate) async fn set_bucket_quota(cluster: &RustFSTestClusterEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, _) =
+                cluster_admin(cluster, Method::GET, &format!("/rustfs/admin/v3/quota-stats/{bucket}"), None).await?;
+            Ok(status.is_success() || status == StatusCode::NOT_FOUND)
+        },
+        "quota stats ready",
+    )
+    .await?;
+    let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, response) =
+                cluster_admin(cluster, Method::PUT, &format!("/rustfs/admin/v3/quota/{bucket}"), Some(body.clone())).await?;
+            if status.is_success() {
+                return Ok(true);
+            }
+            if status == StatusCode::SERVICE_UNAVAILABLE {
+                return Ok(false);
+            }
+            Err(format!("failed to set quota for {bucket}: {status} {response}").into())
+        },
+        "set hard quota",
+    )
+    .await
+}
+
+/// Localhost DistErasure can boot and serve S3 while refusing pool.bin
+/// mutations (`pool metadata writes remain blocked` / missing fleet
+/// capability proof). Single-pool 4×4 also rejects decommission/rebalance
+/// with a product error. Tests must not pretend a move ran.
+pub(crate) fn is_pool_meta_write_fence(body: &str) -> bool {
+    body.contains("pool metadata writes remain blocked")
+        || body.contains("pool metadata recovery required")
+        || body.contains("pool activation requires a live fleet capability proof")
+        || body.contains("pool activation fleet capability proof expired")
+        || body.contains("live fleet capability proof")
+}
+
+/// Product refusals that movement tests observe. Opaque 500 InternalError stays
+/// in [`classify_data_movement_http`] because admin often wraps the fence as
+/// InternalError XML without the inner string. 502/503 and auth failures are
+/// not refusals.
+pub(crate) fn is_known_data_movement_refusal(body: &str) -> bool {
+    is_pool_meta_write_fence(body)
+        || body.contains("NotImplemented")
+        || body.contains("single pool deployments do not support")
+        || body.contains("at least one active pool must remain")
+}
+
+#[derive(Debug)]
+pub(crate) enum DataMovementStart {
+    Started,
+    Refused(String),
+}
+
+pub(crate) fn classify_data_movement_http(status: StatusCode, body: &str) -> Result<DataMovementStart, String> {
+    if status.is_success() {
+        return Ok(DataMovementStart::Started);
+    }
+    if is_known_data_movement_refusal(body) || status.as_u16() == 501 || status == StatusCode::INTERNAL_SERVER_ERROR {
+        return Ok(DataMovementStart::Refused(format!("{status} {body}")));
+    }
+    Err(format!("{status} {body}"))
+}
+
+pub(crate) async fn try_start_decommission(
+    cluster: &RustFSTestClusterEnvironment,
+    pool_id: usize,
+) -> TestResult<DataMovementStart> {
+    let path = format!("/rustfs/admin/v3/pools/decommission?pool={pool_id}&by-id=true");
+    let (status, response) = cluster_admin(cluster, Method::POST, &path, None).await?;
+    classify_data_movement_http(status, &response).map_err(|detail| format!("POST {path} failed: {detail}").into())
+}
+
+/// Returns whether decommission actually started. A product refusal or opaque
+/// 500 InternalError is not a test failure: callers still assert object bytes.
+pub(crate) async fn decommission_started_or_refused(cluster: &RustFSTestClusterEnvironment, pool_id: usize) -> TestResult<bool> {
+    match try_start_decommission(cluster, pool_id).await? {
+        DataMovementStart::Started => Ok(true),
+        DataMovementStart::Refused(detail) => {
+            eprintln!("decommission POST refused; objects still asserted: {detail}");
+            Ok(false)
+        }
+    }
+}
+
+pub(crate) async fn decommission_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult<serde_json::Value> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/decommission/status", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+fn pool_entry(status: &serde_json::Value, pool_id: usize) -> Option<&serde_json::Value> {
+    if let Some(pools) = status.get("pools").and_then(serde_json::Value::as_array) {
+        return pools
+            .iter()
+            .find(|pool| pool.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64));
+    }
+    if status.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64) {
+        Some(status)
+    } else {
+        None
+    }
+}
+
+fn decommission_pool_failed(pool: &serde_json::Value) -> bool {
+    let info = pool.get("decommissionInfo");
+    let flagged = |key: &str| info.and_then(|value| value.get(key)).and_then(serde_json::Value::as_bool) == Some(true);
+    flagged("failed")
+        || flagged("canceled")
+        || pool
+            .get("status")
+            .and_then(serde_json::Value::as_str)
+            .is_some_and(|status| status.eq_ignore_ascii_case("failed") || status.eq_ignore_ascii_case("canceled"))
+}
+
+pub(crate) fn decommission_complete(status: &serde_json::Value, pool_id: usize) -> bool {
+    let Some(pool) = pool_entry(status, pool_id) else {
+        return false;
+    };
+    if decommission_pool_failed(pool) {
+        return false;
+    }
+    let info_complete = pool
+        .get("decommissionInfo")
+        .and_then(|value| value.get("complete"))
+        .and_then(serde_json::Value::as_bool)
+        == Some(true);
+    let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or("");
+    let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or("");
+    info_complete || status_text.eq_ignore_ascii_case("complete") || pool_status.eq_ignore_ascii_case("decommissioned")
+}
+
+pub(crate) fn decommission_failed(status: &serde_json::Value, pool_id: usize) -> bool {
+    pool_entry(status, pool_id).is_some_and(decommission_pool_failed)
+}
+
+/// `Ok(true)` complete, `Ok(false)` still running, `Err` terminal failure.
+pub(crate) fn decommission_progress(status: &serde_json::Value, pool_id: usize) -> Result<bool, String> {
+    if decommission_failed(status, pool_id) {
+        return Err(format!("decommission failed for pool {pool_id}: {status}"));
+    }
+    Ok(decommission_complete(status, pool_id))
+}
+
+pub(crate) async fn wait_for_decommission_complete(
+    cluster: &RustFSTestClusterEnvironment,
+    pool_id: usize,
+    timeout: Duration,
+) -> TestResult {
+    let deadline = Instant::now() + timeout;
+    let mut delay = Duration::from_millis(50);
+    let mut last_error;
+    loop {
+        last_error = match decommission_status_json(cluster).await {
+            Ok(status) => match decommission_progress(&status, pool_id) {
+                Ok(true) => return Ok(()),
+                Ok(false) => format!("decommission complete still false: {status}"),
+                Err(failed) => return Err(failed.into()),
+            },
+            Err(error) => error.to_string(),
+        };
+        if Instant::now() >= deadline {
+            return Err(format!("decommission complete did not become true within {timeout:?}: {last_error}").into());
+        }
+        sleep(delay).await;
+        delay = (delay * 2).min(Duration::from_secs(1));
+    }
+}
+
+pub(crate) async fn try_start_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<DataMovementStart> {
+    let path = "/rustfs/admin/v3/rebalance/start";
+    let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
+    classify_data_movement_http(status, &response).map_err(|detail| format!("POST {path} failed: {detail}").into())
+}
+
+pub(crate) async fn rebalance_started_or_refused(cluster: &RustFSTestClusterEnvironment) -> TestResult<bool> {
+    match try_start_rebalance(cluster).await? {
+        DataMovementStart::Started => Ok(true),
+        DataMovementStart::Refused(detail) => {
+            eprintln!("rebalance POST refused; objects still asserted: {detail}");
+            Ok(false)
+        }
+    }
+}
+
+pub(crate) async fn rebalance_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult<serde_json::Value> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/rebalance/status", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+pub(crate) fn rebalance_active(status: &serde_json::Value) -> bool {
+    status
+        .get("pools")
+        .and_then(serde_json::Value::as_array)
+        .is_some_and(|pools| {
+            pools.iter().any(|pool| {
+                let stopping = pool.get("stopping").and_then(serde_json::Value::as_bool) == Some(true);
+                let value = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or("");
+                stopping
+                    || value.eq_ignore_ascii_case("started")
+                    || value.eq_ignore_ascii_case("active")
+                    || value.eq_ignore_ascii_case("running")
+                    || value.eq_ignore_ascii_case("stopping")
+            })
+        })
+}
+
+pub(crate) async fn wait_for_rebalance_idle(cluster: &RustFSTestClusterEnvironment, timeout: Duration) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            match rebalance_status_json(cluster).await {
+                Ok(status) => Ok(!rebalance_active(&status)),
+                Err(error) => {
+                    let message = error.to_string();
+                    if message.contains("NoSuchResource") || message.contains("404") || message.contains("not started") {
+                        Ok(true)
+                    } else {
+                        Err(error)
+                    }
+                }
+            }
+        },
+        "rebalance idle",
+    )
+    .await
+}
+
+pub(crate) async fn list_pools_json(cluster: &RustFSTestClusterEnvironment) -> TestResult<serde_json::Value> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/pools/list", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+pub(crate) async fn retrying_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, timeout: Duration) -> TestResult {
+    wait_until(
+        timeout,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                match put_object(&client, &bucket, &key, body).await {
+                    Ok(()) => Ok(true),
+                    Err(error) => {
+                        let message = error.to_string();
+                        if message.contains("SlowDown")
+                            || message.contains("ServiceUnavailable")
+                            || message.contains("InternalError")
+                            || message.contains("503")
+                            || message.contains("500")
+                        {
+                            Ok(false)
+                        } else {
+                            Err(error)
+                        }
+                    }
+                }
+            }
+        },
+        &format!("put {bucket}/{key} during data movement"),
+    )
+    .await
+}
+
+pub(crate) async fn retrying_get_equals(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    expected: &[u8],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            match get_object_bytes(client, bucket, key).await {
+                Ok(got) if got.as_slice() == expected => Ok(true),
+                Ok(_) => Ok(false),
+                Err(error) => {
+                    let message = error.to_string();
+                    if message.contains("NoSuchKey")
+                        || message.contains("SlowDown")
+                        || message.contains("ServiceUnavailable")
+                        || message.contains("InternalError")
+                        || message.contains("503")
+                        || message.contains("500")
+                    {
+                        Ok(false)
+                    } else {
+                        Err(error)
+                    }
+                }
+            }
+        },
+        &format!("get {bucket}/{key} during data movement"),
+    )
+    .await
+}
+
+#[tokio::test]
+async fn append_single_node_pool_extends_ellipses_volumes() {
+    let mut env =
+        RustFSTestClusterEnvironment::with_topology(ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0], vec![1]]))
+            .await
+            .expect("two-pool seed topology");
+    assert_eq!(env.rustfs_volumes_arg().split(' ').count(), 2);
+
+    let added = env.append_single_node_pool().await.expect("append third pool");
+    assert_eq!(added, 2);
+    assert_eq!(env.nodes.len(), 3);
+    assert_eq!(env.nodes[2].pool_idx, 2);
+    assert_eq!(env.nodes[2].data_dirs.len(), DRIVES_PER_NODE);
+    let volumes = env.rustfs_volumes_arg();
+    assert_eq!(volumes.split(' ').count(), 3, "expected three pool arguments, got: {volumes}");
+    assert!(volumes.contains("/drive{0...3}"), "expanded layout must keep drive ellipses: {volumes}");
+}
+
+#[tokio::test]
+async fn append_single_node_pool_rejects_striped_single_pool() {
+    let mut env = RustFSTestClusterEnvironment::new(4).await.expect("four-node single pool");
+    let err = env
+        .append_single_node_pool()
+        .await
+        .expect_err("a striped single pool cannot gain a localhost pool");
+    let message = err.to_string();
+    assert!(
+        message.contains("drives_per_node") || message.contains("one node per pool"),
+        "unexpected error: {message}"
+    );
+}
+
+#[cfg(unix)]
+#[tokio::test]
+async fn cluster_start_fails_fast_when_node_process_exits() {
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk)
+        .await
+        .expect("stopped 4-node cluster");
+    let script = format!("{}/immediate-exit.sh", dist.cluster.temp_dir);
+    std::fs::write(&script, "#!/bin/sh\nexit 1\n").expect("write exit stub");
+    let mut perms = std::fs::metadata(&script).expect("stat exit stub").permissions();
+    std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
+    std::fs::set_permissions(&script, perms).expect("chmod exit stub");
+
+    let started = Instant::now();
+    let err = dist
+        .start_from_binary(Path::new(&script))
+        .await
+        .expect_err("a node that exits immediately must fail start");
+    let elapsed = started.elapsed();
+    let message = err.to_string();
+    assert!(
+        message.contains("exited before TCP ready") || message.contains("exited before S3 ready"),
+        "unexpected start error: {message}"
+    );
+    assert!(
+        elapsed < Duration::from_secs(30),
+        "cluster start must fail fast when a node exits, took {elapsed:?}"
+    );
+}
+
+#[test]
+fn decommission_complete_reads_pool_status_and_info_flag() {
+    let status = serde_json::json!({
+        "pools": [
+            {
+                "id": 0,
+                "status": "complete",
+                "poolStatus": "decommissioned",
+                "decommissionInfo": { "complete": true, "failed": false, "canceled": false }
+            },
+            { "id": 1, "status": "none", "poolStatus": "active" }
+        ]
+    });
+    assert!(decommission_complete(&status, 0));
+    assert!(!decommission_complete(&status, 1));
+    assert!(!decommission_failed(&status, 0));
+    assert!(decommission_progress(&status, 0).expect("complete pool"));
+    assert!(!decommission_progress(&status, 1).expect("other pool is not complete"));
+}
+
+#[test]
+fn decommission_progress_fails_closed_on_failed_flag() {
+    let failed = serde_json::json!({
+        "pools": [{
+            "id": 0,
+            "status": "failed",
+            "decommissionInfo": { "complete": false, "failed": true, "canceled": false }
+        }]
+    });
+    let err = decommission_progress(&failed, 0).expect_err("failed decommission must not look complete");
+    assert!(err.contains("decommission failed for pool 0"), "{err}");
+    assert!(!decommission_progress(&failed, 1).expect("missing pool is still running"));
+}
+
+#[test]
+fn rebalance_active_treats_started_as_in_progress() {
+    let started = serde_json::json!({ "pools": [{ "id": 0, "status": "Started", "stopping": false }] });
+    let done = serde_json::json!({ "pools": [{ "id": 0, "status": "Completed", "stopping": false }] });
+    assert!(rebalance_active(&started));
+    assert!(!rebalance_active(&done));
+}
+
+#[test]
+fn pool_meta_write_fence_matches_known_product_gates() {
+    assert!(is_pool_meta_write_fence(
+        "heal_bucket: pool metadata writes remain blocked after a recovery-required replica state"
+    ));
+    assert!(is_pool_meta_write_fence(
+        "rebalance meta save failed: pool activation requires a live fleet capability proof"
+    ));
+    assert!(is_pool_meta_write_fence("pool metadata recovery required: no durable bootstrap identity"));
+    assert!(!is_pool_meta_write_fence("NotImplemented: single pool cannot decommission"));
+    assert!(!is_pool_meta_write_fence("AccessDenied"));
+}
+
+#[test]
+fn classify_data_movement_http_observes_product_refusals_not_auth_failures() {
+    assert!(matches!(classify_data_movement_http(StatusCode::OK, ""), Ok(DataMovementStart::Started)));
+    assert!(matches!(
+        classify_data_movement_http(
+            StatusCode::BAD_REQUEST,
+            "failed to start decommission: single pool deployments do not support decommission"
+        ),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(
+            StatusCode::BAD_REQUEST,
+            "failed to start decommission: at least one active pool must remain after decommission start"
+        ),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(StatusCode::NOT_IMPLEMENTED, "NotImplemented"),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "pool metadata writes remain blocked after a recovery-required replica state"
+        ),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(StatusCode::INTERNAL_SERVER_ERROR, "InternalError"),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    let denied = classify_data_movement_http(StatusCode::FORBIDDEN, "AccessDenied").expect_err("auth failure is not a refusal");
+    assert!(denied.contains("AccessDenied"), "{denied}");
+    let unavailable = classify_data_movement_http(StatusCode::SERVICE_UNAVAILABLE, "ServiceUnavailable")
+        .expect_err("503 is not a product refusal");
+    assert!(unavailable.contains("ServiceUnavailable"), "{unavailable}");
+    let bad_gateway =
+        classify_data_movement_http(StatusCode::BAD_GATEWAY, "Bad Gateway").expect_err("502 is not a product refusal");
+    assert!(bad_gateway.contains("502") || bad_gateway.contains("Bad Gateway"), "{bad_gateway}");
+}
@@ -0,0 +1,35 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node 4-drive distributed e2e coverage.
+//!
+//! Selected by `[profile.e2e-distributed]` and run from
+//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
+//! each case starts four real `rustfs` processes.
+
+mod chaos_test;
+mod concurrency_stability_test;
+mod concurrent_data_movement_test;
+mod data_integrity_movement_test;
+mod expand_decommission_rebalance_test;
+mod extra_test;
+mod harness;
+mod object_lock_test;
+mod observability_test;
+mod replication_quota_test;
+mod s3_basic_test;
+mod s3_during_data_movement_test;
+mod site_replication_test;
+mod upgrade_test;
+mod versioning_test;
@@ -0,0 +1,111 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
+use crate::common::init_logging;
+use crate::object_lock::common::{delete_object_with_bypass, put_object_with_legal_hold, put_object_with_retention};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::operation::delete_object::DeleteObjectError;
+use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
+use chrono::{Duration as ChronoDuration, Utc};
+
+fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
+    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+    if code == Some("AccessDenied") {
+        Ok(())
+    } else {
+        Err(format!("{context}: expected AccessDenied, got {error:?}").into())
+    }
+}
+
+async fn expect_versioned_delete_denied(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    bypass: bool,
+    context: &str,
+) -> TestResult {
+    match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
+        Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
+        Err(error) => delete_denied(error.as_ref(), context),
+    }
+}
+
+#[tokio::test]
+async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let client = dist.client(0)?;
+    let peer = dist.client(2)?;
+    let bucket = unique_bucket("objlock");
+
+    client
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    let retain_until = Utc::now() + ChronoDuration::days(1);
+
+    let compliance_key = "compliance.bin";
+    let compliance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        compliance_key,
+        b"locked-compliance",
+        ObjectLockRetentionMode::Compliance,
+        retain_until,
+    )
+    .await?;
+
+    // Unversioned DELETE is allowed: it only creates a delete marker. WORM
+    // applies to a specific version id.
+    let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
+    assert_eq!(
+        marker.delete_marker(),
+        Some(true),
+        "unversioned DELETE on a locked object must create a delete marker"
+    );
+
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
+        .await?;
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
+
+    let governance_key = "governance.bin";
+    let governance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        governance_key,
+        b"locked-governance",
+        ObjectLockRetentionMode::Governance,
+        retain_until,
+    )
+    .await?;
+
+    expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
+        .await?;
+    delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
+
+    let hold_key = "legal-hold.bin";
+    let hold_version =
+        put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
+
+    Ok(())
+}
@@ -0,0 +1,80 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, put_object, unique_bucket, wait_for_ready,
+};
+use crate::common::{init_logging, local_http_client};
+use http::Method;
+
+#[tokio::test]
+async fn four_node_four_drive_health_admin_info_and_audit_list() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    wait_for_ready(&dist.cluster).await?;
+
+    let http = local_http_client();
+    for node in &dist.cluster.nodes {
+        let ready = http.get(format!("{}/health/ready", node.url)).send().await?;
+        assert!(ready.status().is_success(), "node {} not ready: {}", node.address, ready.status());
+        let live = http.get(format!("{}/health/live", node.url)).send().await;
+        if let Ok(response) = live {
+            assert!(
+                response.status().is_success() || response.status().as_u16() == 404,
+                "unexpected live probe on {}: {}",
+                node.address,
+                response.status()
+            );
+        }
+    }
+
+    let info = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    assert!(!info.is_empty(), "admin info was empty");
+    let storage = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    assert!(
+        storage.contains("disks") || storage.contains("backend") || storage.contains("info"),
+        "storageinfo missing expected fields: {storage}"
+    );
+
+    let audit = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
+    let trimmed = audit.trim();
+    if !trimmed.is_empty() && trimmed != "null" && !trimmed.starts_with('[') && !trimmed.starts_with('{') {
+        return Err(format!("audit target list was not machine-readable: {audit}").into());
+    }
+
+    // Optional surfaces: 404/400/501 are acceptable (route missing or stubbed);
+    // unexpected 5xx is not. A 2xx body must be non-empty.
+    for path in [
+        "/rustfs/admin/v3/log/search",
+        "/rustfs/admin/v4/runtime/capabilities",
+        "/minio/v2/metrics/cluster",
+    ] {
+        let (status, body) = cluster_admin(&dist.cluster, Method::GET, path, None).await?;
+        assert!(
+            status.is_success() || status.is_client_error() || status.as_u16() == 501,
+            "observability path {path} returned {status}: {body}"
+        );
+        if status.is_success() {
+            assert!(!body.trim().is_empty(), "empty body from {path}");
+        }
+    }
+
+    let bucket = unique_bucket("obs");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "probe.log", b"observability".to_vec()).await?;
+
+    let trace = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    assert!(!trace.is_empty());
+    Ok(())
+}
@@ -0,0 +1,144 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
+    set_remote_target, unique_bucket, wait_for_replicated_bytes, wait_until,
+};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use http::Method;
+use std::time::Duration;
+
+/// `Ok(true)` quota admission rejected the PUT, `Ok(false)` retry, `Err` not quota.
+fn quota_over_limit_put_outcome(code: Option<&str>, message: Option<&str>) -> Result<bool, String> {
+    let quota_message = message.is_some_and(|text| text.starts_with("Bucket quota exceeded"));
+    match code {
+        Some("InvalidRequest" | "QuotaExceeded") if quota_message => Ok(true),
+        Some("SlowDown" | "ServiceUnavailable") => Ok(false),
+        Some("AccessDenied") => Err("AccessDenied is not a quota admission rejection".to_string()),
+        Some("InvalidRequest" | "QuotaExceeded") => {
+            Err(format!("InvalidRequest/QuotaExceeded without quota admission message: {message:?}"))
+        }
+        other => Err(format!("unexpected over-quota error code {other:?} message {message:?}")),
+    }
+}
+
+#[tokio::test]
+async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
+    init_logging();
+    let (source, target) = DistCluster::start_replication_pair().await?;
+    let source_bucket = unique_bucket("replsrc");
+    let target_bucket = unique_bucket("repldst");
+    source.create_bucket(&source_bucket).await?;
+    target.create_bucket(&target_bucket).await?;
+
+    let source_client = source.client(0)?;
+    let target_client = target.client(0)?;
+    enable_versioning(&source_client, &source_bucket).await?;
+    enable_versioning(&target_client, &target_bucket).await?;
+
+    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    let key = "replicated.bin";
+    let body = b"distributed-bucket-replication".to_vec();
+    put_object(&source_client, &source_bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
+
+    let peer_read = target.client(3)?;
+    wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
+    let bucket = unique_bucket("quota");
+    dist.create_bucket(&bucket).await?;
+    set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
+
+    let client = dist.client(1)?;
+    retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, body) = super::harness::cluster_admin(
+                &dist.cluster,
+                Method::GET,
+                &format!("/rustfs/admin/v3/quota-stats/{bucket}"),
+                None,
+            )
+            .await?;
+            if !status.is_success() {
+                return Ok(false);
+            }
+            let stats: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
+            Ok(stats.get("current_usage").and_then(serde_json::Value::as_u64).unwrap_or(0) >= 1024)
+        },
+        "quota stats observe small object",
+    )
+    .await?;
+
+    let mut oversized_attempt = 0u32;
+    wait_until(
+        Duration::from_secs(30),
+        || {
+            oversized_attempt += 1;
+            let key = format!("too-big-{oversized_attempt}.bin");
+            let client = client.clone();
+            let bucket = bucket.clone();
+            async move {
+                match client
+                    .put_object()
+                    .bucket(&bucket)
+                    .key(key)
+                    .body(vec![0u8; 16 * 1024].into())
+                    .send()
+                    .await
+                {
+                    Ok(_) => Ok(false),
+                    Err(error) => {
+                        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+                        let message = error.as_service_error().and_then(ProvideErrorMetadata::message);
+                        match quota_over_limit_put_outcome(code, message) {
+                            Ok(done) => Ok(done),
+                            Err(detail) => Err(format!("{detail}: {error:?}").into()),
+                        }
+                    }
+                }
+            }
+        },
+        "hard quota rejects oversized PUT",
+    )
+    .await?;
+    Ok(())
+}
+
+#[test]
+fn quota_over_limit_put_outcome_requires_quota_admission() {
+    assert_eq!(
+        quota_over_limit_put_outcome(Some("InvalidRequest"), Some("Bucket quota exceeded for bucket x")),
+        Ok(true)
+    );
+    assert_eq!(
+        quota_over_limit_put_outcome(Some("QuotaExceeded"), Some("Bucket quota exceeded")),
+        Ok(true)
+    );
+    assert_eq!(quota_over_limit_put_outcome(Some("SlowDown"), Some("slow down")), Ok(false));
+    assert_eq!(quota_over_limit_put_outcome(Some("ServiceUnavailable"), Some("unavailable")), Ok(false));
+    assert!(quota_over_limit_put_outcome(Some("AccessDenied"), Some("Access Denied")).is_err());
+    assert!(quota_over_limit_put_outcome(Some("InvalidRequest"), Some("invalid argument")).is_err());
+}
@@ -0,0 +1,111 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
+use crate::common::{init_logging, local_http_client};
+use aws_sdk_s3::presigning::PresigningConfig;
+use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3basic");
+    dist.create_bucket(&bucket).await?;
+
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let key = "dir/object.bin";
+    let body = vec![0xA5u8; 256 * 1024];
+    put_object(&writer, &bucket, key, body.clone()).await?;
+
+    let head = reader.head_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(head.content_length(), Some(body.len() as i64));
+    assert_object_bytes(&reader, &bucket, key, &body).await?;
+
+    let ranged = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .range("bytes=0-15")
+        .send()
+        .await?;
+    let ranged_body = ranged.body.collect().await?.into_bytes();
+    assert_eq!(ranged_body.as_ref(), &body[..16]);
+
+    let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
+    let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+    assert_eq!(keys, vec![key]);
+
+    let copy_key = "dir/object-copy.bin";
+    reader
+        .copy_object()
+        .bucket(&bucket)
+        .key(copy_key)
+        .copy_source(format!("{bucket}/{key}"))
+        .metadata_directive(MetadataDirective::Copy)
+        .send()
+        .await?;
+    assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
+
+    let moved_key = "dir/object-moved.bin";
+    writer
+        .copy_object()
+        .bucket(&bucket)
+        .key(moved_key)
+        .copy_source(format!("{bucket}/{copy_key}"))
+        .send()
+        .await?;
+    writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
+    match writer.head_object().bucket(&bucket).key(copy_key).send().await {
+        Ok(_) => return Err("copied source still present after rename delete".into()),
+        Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
+        Err(error) => return Err(error.into()),
+    }
+    assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
+
+    let presigned = writer
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
+        .await?;
+    let response = local_http_client().get(presigned.uri().to_string()).send().await?;
+    assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
+    let presigned_body = response.bytes().await?;
+    assert_eq!(presigned_body.as_ref(), body.as_slice());
+
+    let empty_key = "empty";
+    put_object(&writer, &bucket, empty_key, Vec::new()).await?;
+    let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
+    assert!(empty.is_empty());
+
+    writer
+        .delete_objects()
+        .bucket(&bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key(key).build()?)
+                .objects(ObjectIdentifier::builder().key(moved_key).build()?)
+                .objects(ObjectIdentifier::builder().key(empty_key).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+
+    let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
+    Ok(())
+}
@@ -0,0 +1,82 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, put_inventory_retrying,
+    rebalance_started_or_refused, retrying_get_equals, retrying_put, unique_bucket, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3move");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 8, 16 * 1024, Duration::from_secs(30)).await?;
+
+    let decommission_started = decommission_started_or_refused(&dist.cluster, 0).await?;
+    let live = dist.client(2)?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-decommission.bin")),
+        "list during decommission missed the newly written key"
+    );
+
+    if decommission_started {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+    assert_inventory(&live, &bucket, &inventory).await?;
+
+    let _ = rebalance_started_or_refused(&dist.cluster).await?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing",
+        Duration::from_secs(30),
+    )
+    .await?;
+    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,87 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
+};
+use crate::common::{init_logging, signed_request};
+use http::{Method, StatusCode};
+use rustfs_madmin::PeerSite;
+use std::time::Duration;
+
+async fn site_replication_add(cluster: &crate::common::RustFSTestClusterEnvironment, sites: &[PeerSite]) -> TestResult<String> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &cluster.access_key,
+        &cluster.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+    Ok(response.text().await?)
+}
+
+#[tokio::test]
+async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+    let bucket = unique_bucket("siterepl");
+    site_a.create_bucket(&bucket).await?;
+    site_b.create_bucket(&bucket).await?;
+
+    let client_a = site_a.client(0)?;
+    let client_b = site_b.client(0)?;
+    enable_versioning(&client_a, &bucket).await?;
+    enable_versioning(&client_b, &bucket).await?;
+
+    let sites = vec![
+        PeerSite {
+            name: "site-a".to_string(),
+            endpoint: site_a.cluster.nodes[0].url.clone(),
+            access_key: site_a.cluster.access_key.clone(),
+            secret_key: site_a.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+        PeerSite {
+            name: "site-b".to_string(),
+            endpoint: site_b.cluster.nodes[0].url.clone(),
+            access_key: site_b.cluster.access_key.clone(),
+            secret_key: site_b.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+    ];
+    site_replication_add(&site_a.cluster, &sites).await?;
+
+    let info = cluster_admin_ok(&site_a.cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
+    assert!(
+        info.contains("site-a") || info.contains("enabled") || info.contains("true"),
+        "site replication info did not show a configured peer: {info}"
+    );
+
+    let key = "site-object.bin";
+    let body = b"four-node-site-replication".to_vec();
+    put_object(&client_a, &bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
+
+    let peer_b = site_b.client(3)?;
+    wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
+    Ok(())
+}
@@ -0,0 +1,363 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node upgrade coverage for historical objects and IAM AK/SK.
+//!
+//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
+//! mixed-version listing). This module pins the distributed contract the
+//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
+//! written on the previous release still read back, and IAM user credentials
+//! created before the upgrade still authenticate.
+//!
+//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
+//! release. The `e2e-distributed` workflow downloads that binary; a local run
+//! without it fails closed rather than skipping.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
+    unique_bucket, wait_until,
+};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use std::ffi::OsString;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use uuid::Uuid;
+
+const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const IAM_SECRET: &str = "UpgradeTestSecretKey1";
+const WRONG_SECRET: &str = "WrongSecretKey000000";
+const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct UpgradeSeed {
+    history_bucket: String,
+    history_key: &'static str,
+    history_body: Vec<u8>,
+    versioned_bucket: String,
+    versioned_key: &'static str,
+    version1: String,
+    version1_body: Vec<u8>,
+    version2: String,
+    version2_body: Vec<u8>,
+    iam_bucket: String,
+    iam_key: &'static str,
+    iam_body: Vec<u8>,
+    iam_user: String,
+    iam_secret: &'static str,
+}
+
+fn resolve_source_binary(value: Option<OsString>) -> TestResult<PathBuf> {
+    let path = value.map(PathBuf::from).ok_or_else(|| {
+        format!(
+            "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
+        )
+    })?;
+    if !path.is_file() {
+        return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
+    }
+    Ok(path)
+}
+
+fn source_binary() -> TestResult<PathBuf> {
+    resolve_source_binary(std::env::var_os(SOURCE_BINARY_ENV))
+}
+
+fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
+        cluster
+            .cluster
+            .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+fn iam_rw_policy(bucket: &str) -> String {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}"),
+                format!("arn:aws:s3:::{bucket}/*")
+            ]
+        }]
+    })
+    .to_string()
+}
+
+async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
+    let url = &dist.cluster.nodes[0].url;
+    let access = &dist.cluster.access_key;
+    let admin_secret = &dist.cluster.secret_key;
+    admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
+    admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
+    admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
+    Ok(())
+}
+
+async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                put_object(&client, &bucket, &key, body).await?;
+                Ok(true)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let expected = expected.to_vec();
+            async move {
+                let got = get_object_bytes(&client, &bucket, &key).await?;
+                Ok(got == expected)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
+    let history_bucket = unique_bucket("upg-hist");
+    let versioned_bucket = unique_bucket("upg-ver");
+    let iam_bucket = unique_bucket("upg-iam");
+    dist.create_bucket(&history_bucket).await?;
+    dist.create_bucket(&versioned_bucket).await?;
+    dist.create_bucket(&iam_bucket).await?;
+
+    let root = dist.client(0)?;
+    enable_versioning(&root, &versioned_bucket).await?;
+
+    let history_key = "plain-history.bin";
+    let history_body = b"written by the previous 4-node release".to_vec();
+    put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
+
+    let versioned_key = "versioned-history.txt";
+    let version1_body = b"version-one-before-upgrade".to_vec();
+    let version1 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("first versioned PUT omitted version ID")?
+        .to_string();
+    let version2_body = b"version-two-before-upgrade".to_vec();
+    let version2 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("second versioned PUT omitted version ID")?
+        .to_string();
+
+    let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
+
+    let iam_key = "iam-history.bin";
+    let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
+    let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
+    wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
+
+    Ok(UpgradeSeed {
+        history_bucket,
+        history_key,
+        history_body,
+        versioned_bucket,
+        versioned_key,
+        version1,
+        version1_body,
+        version2,
+        version2_body,
+        iam_bucket,
+        iam_key,
+        iam_body,
+        iam_user,
+        iam_secret: IAM_SECRET,
+    })
+}
+
+async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
+    let root_a = dist.client(0)?;
+    let root_b = dist.client(3)?;
+    wait_for_bytes(
+        &root_b,
+        &seed.history_bucket,
+        seed.history_key,
+        &seed.history_body,
+        &format!("{context}: root GET historical object"),
+    )
+    .await?;
+    assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
+
+    let v1 = root_b
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version1)
+        .send()
+        .await?;
+    let v1_body = v1.body.collect().await?.into_bytes();
+    if v1_body.as_ref() != seed.version1_body.as_slice() {
+        return Err(format!("{context}: version 1 bytes changed after upgrade").into());
+    }
+    let v2 = root_a
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version2)
+        .send()
+        .await?;
+    let v2_body = v2.body.collect().await?.into_bytes();
+    if v2_body.as_ref() != seed.version2_body.as_slice() {
+        return Err(format!("{context}: version 2 bytes changed after upgrade").into());
+    }
+
+    let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    if !users.contains(&seed.iam_user) {
+        return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
+    }
+
+    let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
+    let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
+    wait_for_bytes(
+        &iam_on_upgraded,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 0"),
+    )
+    .await?;
+    wait_for_bytes(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 3"),
+    )
+    .await?;
+
+    let post_key = format!("after-upgrade-{context}.txt");
+    let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
+    wait_for_put(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        &post_key,
+        post_body.clone(),
+        &format!("{context}: IAM PUT after upgrade"),
+    )
+    .await?;
+    assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
+
+    let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
+    match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
+        Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
+        Err(error) => {
+            let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+            if code == Some("SignatureDoesNotMatch")
+                || code == Some("InvalidAccessKeyId")
+                || code == Some("AccessDenied")
+                || code == Some("InvalidArgument")
+            {
+            } else if error.raw_response().is_some_and(|response| response.status().as_u16() == 403) {
+            } else {
+                return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
+            }
+        }
+    }
+
+    let post_root_key = format!("root-after-{context}.bin");
+    let post_root_body = format!("{context}: root write after upgrade").into_bytes();
+    put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
+    assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "direct-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+    dist.restart_with_current_binary().await?;
+    assert_history_and_iam(&dist, &seed, "direct").await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+
+    dist.replace_node_with_current_binary(0).await?;
+    assert_history_and_iam(&dist, &seed, "one-current-node").await?;
+
+    for node_idx in [1, 2] {
+        dist.replace_node_with_current_binary(node_idx).await?;
+    }
+    assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
+
+    dist.replace_node_with_current_binary(3).await?;
+    assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
+    Ok(())
+}
+
+#[test]
+fn missing_upgrade_source_binary_fails_closed() {
+    let err = resolve_source_binary(None).expect_err("absent env must fail closed");
+    assert!(err.to_string().contains(SOURCE_BINARY_ENV), "{err}");
+}
+
+#[test]
+fn missing_upgrade_source_binary_file_fails_closed() {
+    let err = resolve_source_binary(Some("/no/such/rustfs-upgrade-source".into())).expect_err("missing file must fail closed");
+    assert!(err.to_string().contains("does not exist"), "{err}");
+}
@@ -0,0 +1,88 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+
+#[tokio::test]
+async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("version");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "versioned.txt";
+    put_object(&writer, &bucket, key, b"v1".to_vec()).await?;
+    put_object(&writer, &bucket, key, b"v2".to_vec()).await?;
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let version_ids: Vec<String> = versions
+        .versions()
+        .iter()
+        .filter_map(|version| version.version_id().map(str::to_string))
+        .collect();
+    assert!(version_ids.len() >= 2, "expected at least two versions, got {version_ids:?}");
+
+    let latest = get_object_bytes(&reader, &bucket, key).await?;
+    assert_eq!(latest, b"v2");
+
+    let older_id = versions
+        .versions()
+        .iter()
+        .find(|version| version.is_latest() != Some(true))
+        .and_then(|version| version.version_id())
+        .ok_or("missing non-latest version id")?;
+    let older = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(older_id)
+        .send()
+        .await?;
+    let older_body = older.body.collect().await?.into_bytes();
+    assert_eq!(older_body.as_ref(), b"v1");
+
+    writer.delete_object().bucket(&bucket).key(key).send().await?;
+    let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    assert!(
+        !after_delete.delete_markers().is_empty(),
+        "delete marker missing after unversioned-style delete: {after_delete:?}"
+    );
+
+    let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
+    match latest_after_delete {
+        Ok(_) => return Err("current version should be a delete marker".into()),
+        Err(error)
+            if error
+                .as_service_error()
+                .and_then(ProvideErrorMetadata::code)
+                .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
+        Err(error) => return Err(error.into()),
+    }
+
+    let restored = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(older_id)
+        .send()
+        .await?;
+    let restored_body = restored.body.collect().await?.into_bytes();
+    assert_eq!(restored_body.as_ref(), b"v1");
+    Ok(())
+}
@@ -378,6 +378,11 @@ mod bucket_stats_regression_test;
 #[cfg(test)]
 mod distributed_startup_regression_test;

+// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
+// quota, observability, expand/decommission/rebalance, site replication, chaos).
+#[cfg(test)]
+mod distributed;
+
 // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
 #[cfg(test)]
 mod tier_transition_regression_test;
@@ -252,6 +252,13 @@ pub mod bucket {
        };
    }

+    pub mod sealed_credentials {
+        pub use crate::bucket::sealed_credentials::{
+            CredentialSealer, SEALED_CREDENTIAL_VERSION, SealScope, SealedCredential, SealedCredentialError,
+            SealedCredentialStore, credential_sealer, install_credential_sealer, seal_secret, unseal_secret,
+        };
+    }
+
    pub mod replication {
        pub use crate::bucket::replication::replication_pool::{
            DurableMrfBacklogSummary, DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBacklogObservabilitySummary,
@@ -87,16 +87,12 @@ use rustfs_filemeta::{
 use rustfs_scanner_metrics::metrics::{
    IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
 };
-use rustfs_utils::{
-    get_env_i64, get_env_usize,
-    path::encode_dir_object,
-    string::{parse_bool, strings_has_prefix_fold},
-};
+use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
 use s3s::dto::{
    BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
    Timestamp,
 };
-use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
+use s3s::header::X_AMZ_RESTORE;
 use sha2::{Digest, Sha256};
 use std::any::Any;
 use std::collections::{BTreeMap, HashMap, HashSet};
@@ -165,7 +161,6 @@ pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
    reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
 )]
 pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
-pub const AMZ_ENCRYPTION_AES: &str = "AES256";
 #[allow(
    dead_code,
    reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
@@ -4802,24 +4797,24 @@ fn attach_tier_operation_lease(mut reader: GetObjectReader, lease: TierOperation
    reader
 }

-pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, std::io::Error> {
+/// Resolve the RestoreObject request options.
+///
+/// Returns the typed [`StorageError`]: flattening these into an opaque
+/// `io::Error` string erased the identity the S3 layer needs to answer
+/// InvalidArgument instead of a generic 500 (backlog#2205).
+pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) -> Result<ObjectOptions, Error> {
    let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
    let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
    let vid = version_id.trim();
    if !vid.is_empty() && vid != NULL_VERSION_ID {
        if let Err(_err) = Uuid::parse_str(vid) {
-            return Err(std::io::Error::other(
-                StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()).to_string(),
-            ));
+            return Err(StorageError::InvalidVersionID(bucket.to_string(), object.to_string(), vid.to_string()));
        }
        if !versioned && !version_suspended {
-            return Err(std::io::Error::other(
-                StorageError::InvalidArgument(
-                    bucket.to_string(),
-                    object.to_string(),
-                    format!("version-id specified {} but versioning is not enabled on {}", vid, bucket),
-                )
-                .to_string(),
+            return Err(StorageError::InvalidArgument(
+                bucket.to_string(),
+                object.to_string(),
+                format!("version-id specified {vid} but versioning is not enabled on {bucket}"),
            ));
        }
    }
@@ -4872,43 +4867,18 @@ pub async fn put_restore_opts(
    }
    meta.insert(X_AMZ_STORAGE_CLASS.as_str().to_lowercase(), sc);*/

-    if let Some(type_) = &rreq.type_
-        && type_.as_str() == RestoreRequestType::SELECT
+    // A SELECT restore must never reach the restore writer: the caller writes
+    // the retrieved bytes back to the source bucket/object, so building
+    // SELECT output options here produced a source overwrite carrying only
+    // the OutputLocation metadata instead of a write to `OutputLocation.S3`
+    // (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
+    // is the fail-closed backstop for any other caller.
+    if rreq
+        .type_
+        .as_ref()
+        .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
    {
-        let Some(s3) = select_restore_s3_location(rreq)? else {
-            return Err(std::io::Error::other("OutputLocation.S3 required for SELECT requests"));
-        };
-        if let Some(user_metadata) = s3.user_metadata.as_ref() {
-            for metadata in user_metadata {
-                let name = metadata
-                    .name
-                    .as_deref()
-                    .ok_or_else(|| std::io::Error::other("SELECT restore metadata name is required"))?;
-                let value = metadata.value.clone().unwrap_or_default();
-                if strings_has_prefix_fold(name, "x-amz-meta") {
-                    meta.insert(name.to_string(), value);
-                } else {
-                    meta.insert(format!("x-amz-meta-{name}"), value);
-                }
-            }
-        }
-        if let Some(tags) = &s3.tagging {
-            meta.insert(
-                AMZ_OBJECT_TAGGING.to_string(),
-                serde_urlencoded::to_string(tags.tag_set.clone()).unwrap_or_else(|_| "".to_string()),
-            );
-        }
-        if let Some(encryption) = &s3.encryption
-            && encryption.encryption_type.as_str() != ""
-        {
-            meta.insert(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str().to_string(), AMZ_ENCRYPTION_AES.to_string());
-        }
-        return Ok(ObjectOptions {
-            versioned: BucketVersioningSys::prefix_enabled(bucket, object).await,
-            version_suspended: BucketVersioningSys::prefix_suspended(bucket, object).await,
-            user_defined: meta,
-            ..Default::default()
-        });
+        return Err(std::io::Error::other("SELECT restore requests are not supported"));
    }
    for (k, v) in oi.user_defined.iter() {
        meta.insert(k.to_string(), v.clone());
@@ -31,6 +31,7 @@ pub mod policy_sys;
 pub mod quota;
 pub mod remote_s3_client;
 pub mod replication;
+pub mod sealed_credentials;
 pub mod tagging;
 pub mod target;
 pub mod utils;
@@ -14,9 +14,9 @@

 //! Outbound client for an on-demand migration source bucket.
 //!
-//! `SourceClient` wraps an `aws_sdk_s3::Client` built through the shared
-//! remote builder and exposes the read-only surface the migration path
-//! needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
+//! `SourceClient` maps local keys onto a read-only `SourceBackend`. The
+//! S3 backend uses the shared remote builder and exposes the surface the
+//! migration path needs (HEAD, ranged streaming GET, ListObjectsV2, GetObjectTagging, a
 //! probe for admin validation). Every request carries the
 //! `source-proxy-request` anti-loop marker in both the `x-rustfs-` and
 //! `x-minio-` prefixes so a RustFS/MinIO source answers locally instead of
@@ -511,7 +511,7 @@ pub struct SourceObject {
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct SourcePage {
    pub objects: Vec<SourceObject>,
-    /// Rolled-up prefixes, in the local namespace; always empty when the
+    /// Rolled-up prefixes, in the same namespace as `objects`; always empty when the
    /// request carried no delimiter.
    pub common_prefixes: Vec<String>,
    pub is_truncated: bool,
@@ -519,7 +519,8 @@ pub struct SourcePage {
 }

 /// One `ListObjectsV2` page request against the source. Keys are given in the
-/// local namespace; `SourceClient` maps them through `source_prefix`.
+/// local namespace at `SourceClient`, and in the source namespace at
+/// `SourceBackend`; `SourceClient` maps them through `source_prefix`.
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct SourceListRequest<'a> {
    pub prefix: Option<&'a str>,
@@ -575,8 +576,30 @@ impl Intercept for SourceProxyMarkerInterceptor {
    }
 }

-pub struct SourceClient {
+/// Read-only provider operations in the source bucket namespace.
+///
+/// Implementations must preserve streaming, honor the requested range and
+/// pagination cursor, and classify failures without including credentials.
+/// `SourceClient` owns prefix mapping so every provider shares the same local
+/// namespace. Continuation tokens are opaque and must never be prefix-mapped.
+#[async_trait::async_trait]
+pub trait SourceBackend: Send + Sync {
+    async fn head(&self, key: &str) -> Result<SourceHead, SourceError>;
+    async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError>;
+    async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError>;
+    async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError>;
+    /// Verify bucket access; `SourceClient` separately probes a filtered listing.
+    async fn probe(&self) -> Result<(), SourceError>;
+}
+
+/// S3-compatible implementation, including request signing and anti-loop headers.
+pub struct S3SourceBackend {
    client: S3Client,
+    bucket: String,
+}
+
+pub struct SourceClient {
+    backend: Box<dyn SourceBackend>,
    endpoint: String,
    bucket: String,
    source_prefix: Option<String>,
@@ -609,7 +632,10 @@ impl SourceClient {
    fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
        let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
        Self {
-            client,
+            backend: Box::new(S3SourceBackend {
+                client,
+                bucket: spec.bucket.clone(),
+            }),
            endpoint,
            bucket: spec.bucket.clone(),
            source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
@@ -652,35 +678,15 @@ impl SourceClient {
    }

    pub async fn head_object(&self, key: &str) -> Result<SourceHead, SourceError> {
-        let output = self
-            .client
-            .head_object()
-            .bucket(&self.bucket)
-            .key(self.source_key(key))
-            .send()
-            .await
-            .map_err(classify_sdk_error)?;
-        source_head_from_head_output(output)
+        self.backend.head(&self.source_key(key)).await
    }

-    /// Streams the object; `range` is passed through as an HTTP `Range`
-    /// header and omitted entirely when `None`.
+    /// Streams the object, preserving an optional HTTP byte range.
    pub async fn get_object(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
-        let range = range.map(range_header_value).transpose()?;
-        let output = self
-            .client
-            .get_object()
-            .bucket(&self.bucket)
-            .key(self.source_key(key))
-            .set_range(range)
-            .send()
-            .await
-            .map_err(classify_sdk_error)?;
-        source_get_from_output(output)
+        self.backend.get(&self.source_key(key), range).await
    }

-    /// Lists one page under the local `prefix`. Keys are returned in the
-    /// local namespace; entries outside `source_prefix` are skipped.
+    /// Lists one page under the local prefix.
    pub async fn list_objects_v2(
        &self,
        prefix: Option<&str>,
@@ -696,9 +702,81 @@ impl SourceClient {
        .await
    }

-    /// [`Self::list_objects_v2`] with the delimiter and start-after the
-    /// list-through merge needs (rustfs/backlog#2164).
+    /// Maps keys and common prefixes while leaving opaque cursors untouched.
    pub async fn list_page(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
+        let prefix = self.source_key(request.prefix.unwrap_or_default());
+        let start_after = request.start_after.map(|key| self.source_key(key));
+        let mut page = self
+            .backend
+            .list(&SourceListRequest {
+                prefix: Some(&prefix),
+                start_after: start_after.as_deref(),
+                ..*request
+            })
+            .await?;
+        page.objects = page
+            .objects
+            .into_iter()
+            .filter_map(|object| self.local_object(object))
+            .collect();
+        page.common_prefixes = page
+            .common_prefixes
+            .into_iter()
+            .filter_map(|prefix| self.local_key(&prefix).map(str::to_string))
+            .collect();
+        Ok(page)
+    }
+
+    fn local_object(&self, mut object: SourceObject) -> Option<SourceObject> {
+        object.key = self.local_key(&object.key)?.to_string();
+        Some(object)
+    }
+
+    pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
+        self.backend.tagging(&self.source_key(key)).await
+    }
+
+    pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
+        self.backend.probe().await?;
+        let page = self.list_objects_v2(None, None, 1).await?;
+        Ok(SourceProbe {
+            sample_object: page.objects.into_iter().next(),
+            has_more_objects: page.is_truncated,
+        })
+    }
+}
+
+#[async_trait::async_trait]
+impl SourceBackend for S3SourceBackend {
+    async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
+        let output = self
+            .client
+            .head_object()
+            .bucket(&self.bucket)
+            .key(key)
+            .send()
+            .await
+            .map_err(classify_sdk_error)?;
+        source_head_from_head_output(output)
+    }
+
+    /// Streams the object; `range` is passed through as an HTTP `Range`
+    /// header and omitted entirely when `None`.
+    async fn get(&self, key: &str, range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
+        let range = range.map(range_header_value).transpose()?;
+        let output = self
+            .client
+            .get_object()
+            .bucket(&self.bucket)
+            .key(key)
+            .set_range(range)
+            .send()
+            .await
+            .map_err(classify_sdk_error)?;
+        source_get_from_output(output)
+    }
+
+    async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
        // `start_after` is silently ignored by S3 once a continuation token is
        // present; refuse the ambiguous pair rather than list from the wrong
        // position.
@@ -711,9 +789,9 @@ impl SourceClient {
            .client
            .list_objects_v2()
            .bucket(&self.bucket)
-            .prefix(self.source_key(request.prefix.unwrap_or_default()))
+            .prefix(request.prefix.unwrap_or_default())
            .set_delimiter(request.delimiter.map(str::to_string))
-            .set_start_after(request.start_after.map(|after| self.source_key(after)))
+            .set_start_after(request.start_after.map(str::to_string))
            .set_continuation_token(request.continuation_token.map(str::to_string))
            .max_keys(request.max_keys)
            .send()
@@ -731,13 +809,13 @@ impl SourceClient {
            .contents
            .unwrap_or_default()
            .into_iter()
-            .filter_map(|object| self.source_object(object))
+            .filter_map(s3_source_object)
            .collect();
        let common_prefixes = output
            .common_prefixes
            .unwrap_or_default()
            .into_iter()
-            .filter_map(|prefix| Some(self.local_key(prefix.prefix.as_deref()?)?.to_string()))
+            .filter_map(|prefix| prefix.prefix)
            .collect();

        Ok(SourcePage {
@@ -748,48 +826,43 @@ impl SourceClient {
        })
    }

-    fn source_object(&self, object: SdkObject) -> Option<SourceObject> {
-        let key = self.local_key(object.key.as_deref()?)?.to_string();
-        let etag = normalize_etag(object.e_tag);
-        let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
-        Some(SourceObject {
-            key,
-            etag,
-            size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
-            last_modified: system_time(object.last_modified),
-            storage_class: object.storage_class.map(|class| class.as_str().to_string()),
-            is_multipart_etag,
-        })
-    }
-
-    pub async fn get_object_tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
+    async fn tagging(&self, key: &str) -> Result<HashMap<String, String>, SourceError> {
        let output = self
            .client
            .get_object_tagging()
            .bucket(&self.bucket)
-            .key(self.source_key(key))
+            .key(key)
            .send()
            .await
            .map_err(classify_sdk_error)?;
        Ok(output.tag_set.into_iter().map(|tag| (tag.key, tag.value)).collect())
    }

-    /// Admin validation: HeadBucket plus a one-key listing under the prefix.
-    pub async fn probe(&self) -> Result<SourceProbe, SourceError> {
+    async fn probe(&self) -> Result<(), SourceError> {
        self.client
            .head_bucket()
            .bucket(&self.bucket)
            .send()
            .await
            .map_err(classify_sdk_error)?;
-        let page = self.list_objects_v2(None, None, 1).await?;
-        Ok(SourceProbe {
-            sample_object: page.objects.into_iter().next(),
-            has_more_objects: page.is_truncated,
-        })
+        Ok(())
    }
 }

+fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
+    let key = object.key?;
+    let etag = normalize_etag(object.e_tag);
+    let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
+    Some(SourceObject {
+        key,
+        etag,
+        size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
+        last_modified: system_time(object.last_modified),
+        storage_class: object.storage_class.map(|class| class.as_str().to_string()),
+        is_multipart_etag,
+    })
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -1194,6 +1267,68 @@ mod tests {
        assert!(requests[1].uri.contains("continuation-token=token-1"), "{}", requests[1].uri);
    }

+    #[tokio::test]
+    async fn list_page_maps_delimiter_prefixes_and_start_after_but_not_cursors() {
+        let body = r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+<IsTruncated>true</IsTruncated><NextContinuationToken>data/opaque</NextContinuationToken>
+<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
+<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
+</ListBucketResult>"#;
+        let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
+        let first = client
+            .list_page(&SourceListRequest {
+                prefix: Some("photos/"),
+                delimiter: Some("/"),
+                start_after: Some("photos/a"),
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("delimiter listing should succeed");
+        assert_eq!(first.common_prefixes, vec!["photos/"]);
+        assert_eq!(first.next_continuation_token.as_deref(), Some("data/opaque"));
+        let second = client
+            .list_page(&SourceListRequest {
+                continuation_token: first.next_continuation_token.as_deref(),
+                max_keys: 2,
+                ..Default::default()
+            })
+            .await
+            .expect("opaque continuation should succeed");
+        assert_eq!(second.common_prefixes, first.common_prefixes);
+        let requests = recorded(&requests);
+        let query = |request: &RecordedRequest| {
+            Url::parse(&request.uri)
+                .expect("request URI")
+                .query_pairs()
+                .into_owned()
+                .collect::<HashMap<_, _>>()
+        };
+        let first_query = query(&requests[0]);
+        assert_eq!(first_query.get("prefix").map(String::as_str), Some("data/photos/"));
+        assert_eq!(first_query.get("start-after").map(String::as_str), Some("data/photos/a"));
+        assert_eq!(first_query.get("delimiter").map(String::as_str), Some("/"));
+        let second_query = query(&requests[1]);
+        assert_eq!(second_query.get("continuation-token").map(String::as_str), Some("data/opaque"));
+        assert!(!second_query.contains_key("start-after"));
+    }
+
+    #[tokio::test]
+    async fn list_page_rejects_ambiguous_cursor_before_sending() {
+        let (client, requests) = scripted_client(&spec(Some("data/")), vec![]).await;
+        let err = client
+            .list_page(&SourceListRequest {
+                start_after: Some("a"),
+                continuation_token: Some("opaque"),
+                max_keys: 1,
+                ..Default::default()
+            })
+            .await
+            .expect_err("ambiguous list position must fail");
+        assert!(matches!(err, SourceError::Other(_)));
+        assert!(recorded(&requests).is_empty(), "invalid request must never reach the source");
+    }
+
    #[tokio::test]
    async fn list_objects_v2_rejects_truncated_page_without_token() {
        let (client, _) = scripted_client(&spec(None), vec![ok(Vec::new(), LIST_TRUNCATED_WITHOUT_TOKEN)]).await;
@@ -1357,11 +1492,14 @@ mod tests {

    fn prefix_client(prefix: Option<String>) -> SourceClient {
        SourceClient {
-            client: S3Client::from_conf(
-                aws_sdk_s3::Config::builder()
-                    .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-                    .build(),
-            ),
+            backend: Box::new(S3SourceBackend {
+                client: S3Client::from_conf(
+                    aws_sdk_s3::Config::builder()
+                        .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
+                        .build(),
+                ),
+                bucket: "bucket".to_string(),
+            }),
            endpoint: "https://source.example.com".to_string(),
            bucket: "bucket".to_string(),
            source_prefix: prefix.filter(|prefix| !prefix.is_empty()),
@@ -0,0 +1,351 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Sealed remote credentials shared by the three stores that persist one
+//! (rustfs/backlog#2168): replication targets (`bucket-targets.json`), remote
+//! tiers (`tier-config.bin`) and on-demand migration sources
+//! (`on-demand-migration.json`).
+//!
+//! The design record is `docs/architecture/remote-credential-sealing-adr.md`.
+//! What this module owns: the versioned envelope, the encryption context that
+//! binds a ciphertext to the record owning it, the sealer registration point,
+//! and the fail-closed error type. What it deliberately does not own: any KMS
+//! call (ECStore does not depend on `rustfs-kms`; the binary installs a
+//! sealer, exactly like `ON_DEMAND_MIGRATION_CONFIG_HOOK` and the event
+//! dispatch hook in `crates/ecstore/src/services/event_notification.rs`), and
+//! any decision about which stored field a consumer writes.
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::fmt;
+use std::sync::{Arc, OnceLock};
+
+/// Envelope format this build writes. A reader accepts only versions it
+/// knows; an unknown version is a typed error, never a fallback.
+pub const SEALED_CREDENTIAL_VERSION: u8 = 1;
+
+/// Which store a sealed value belongs to. Part of the encryption context, so
+/// a ciphertext cannot be replayed into a different store.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum SealedCredentialStore {
+    /// `bucket-targets.json` (replication and other bucket targets).
+    BucketTargets,
+    /// `tier-config.bin` (remote tiers).
+    TierConfig,
+    /// `on-demand-migration.json` (migration sources).
+    OnDemandMigration,
+}
+
+impl SealedCredentialStore {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            SealedCredentialStore::BucketTargets => "bucket-targets",
+            SealedCredentialStore::TierConfig => "tier-config",
+            SealedCredentialStore::OnDemandMigration => "on-demand-migration",
+        }
+    }
+}
+
+/// Identity of the record a secret belongs to: the store, its owner (bucket
+/// name, tier name, or target ARN) and the field name. Rendered into the KMS
+/// encryption context so a ciphertext moved between buckets, tiers or fields
+/// fails to decrypt instead of silently authorizing a different remote.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SealScope {
+    pub store: SealedCredentialStore,
+    pub owner: String,
+    pub field: &'static str,
+}
+
+impl SealScope {
+    pub fn new(store: SealedCredentialStore, owner: impl Into<String>, field: &'static str) -> Self {
+        Self {
+            store,
+            owner: owner.into(),
+            field,
+        }
+    }
+
+    /// The encryption context handed to the sealer. Keys are stable: they are
+    /// part of the on-disk contract, because a ciphertext only decrypts under
+    /// the same context.
+    pub fn encryption_context(&self) -> HashMap<String, String> {
+        HashMap::from([
+            ("rustfs:store".to_string(), self.store.as_str().to_string()),
+            ("rustfs:owner".to_string(), self.owner.clone()),
+            ("rustfs:field".to_string(), self.field.to_string()),
+        ])
+    }
+}
+
+/// A sealed secret as persisted. `Debug` prints no ciphertext: a sealed value
+/// is not a secret, but it is noise in a log line and an operator reading one
+/// should see the key it is wrapped under, not the bytes.
+#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SealedCredential {
+    /// Envelope version; see [`SEALED_CREDENTIAL_VERSION`].
+    pub v: u8,
+    /// KMS master key id the data key is wrapped under.
+    pub key_id: String,
+    /// Master key version, when the backend reports one. Carried so the KMS
+    /// re-wrap job (`docs/architecture/kms-bulk-rekey-contract.md`) can tell
+    /// stale envelopes apart; nothing here rotates on its own.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub key_version: Option<String>,
+    /// Algorithm label reported by the sealer, for forensics and migration.
+    pub alg: String,
+    /// Ciphertext blob as produced by the sealer, base64 (standard, padded)
+    /// in the JSON stores and raw inside the tier msgpack payload.
+    pub ct: String,
+}
+
+impl fmt::Debug for SealedCredential {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("SealedCredential")
+            .field("v", &self.v)
+            .field("key_id", &self.key_id)
+            .field("key_version", &self.key_version)
+            .field("alg", &self.alg)
+            .field("ct", &format_args!("<{} bytes sealed>", self.ct.len()))
+            .finish()
+    }
+}
+
+impl SealedCredential {
+    /// Rejects an envelope this build cannot read. Called before every
+    /// unseal so an unknown version fails here rather than inside a backend.
+    pub fn check_version(&self) -> Result<(), SealedCredentialError> {
+        if self.v == SEALED_CREDENTIAL_VERSION {
+            Ok(())
+        } else {
+            Err(SealedCredentialError::UnsupportedVersion(self.v))
+        }
+    }
+}
+
+/// Why a seal or unseal did not produce a usable value. Every variant is
+/// terminal for the record that carried it: a caller reports the remote as
+/// unusable, and never substitutes a default or empty credential.
+#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
+pub enum SealedCredentialError {
+    /// No sealer is installed: KMS is not configured, or the process has not
+    /// finished startup. Reading a sealed record is impossible here.
+    #[error("no credential sealer is installed")]
+    NoSealer,
+    /// The stored envelope is from a newer (or otherwise unknown) format.
+    #[error("unsupported sealed credential envelope version {0}")]
+    UnsupportedVersion(u8),
+    /// The stored bytes are not a well-formed envelope.
+    #[error("malformed sealed credential: {0}")]
+    Malformed(String),
+    /// The sealer refused: wrong encryption context, missing key, revoked
+    /// access, or a failed authentication tag.
+    #[error("sealed credential could not be unsealed: {0}")]
+    Kms(String),
+}
+
+/// The KMS-backed half, installed by the binary at startup.
+#[async_trait]
+pub trait CredentialSealer: Send + Sync + 'static {
+    /// Wraps `plaintext` under the scope's encryption context.
+    async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError>;
+
+    /// Unwraps a stored envelope. Must fail when the envelope was sealed
+    /// under a different scope.
+    async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError>;
+}
+
+static CREDENTIAL_SEALER: OnceLock<Arc<dyn CredentialSealer>> = OnceLock::new();
+
+/// Installs the process-wide sealer. Returns `false` when one is already
+/// installed, matching the other ECStore hooks.
+pub fn install_credential_sealer(sealer: Arc<dyn CredentialSealer>) -> bool {
+    CREDENTIAL_SEALER.set(sealer).is_ok()
+}
+
+/// The installed sealer, or `None` when KMS is not wired. Callers that only
+/// need to know whether sealing is possible use this; callers that must have
+/// it use [`seal_secret`] / [`unseal_secret`] and get the typed error.
+pub fn credential_sealer() -> Option<Arc<dyn CredentialSealer>> {
+    CREDENTIAL_SEALER.get().cloned()
+}
+
+/// Seals one secret field. Fails closed: without a sealer the caller must
+/// reject the write rather than persist the secret in clear text after the
+/// operator asked for sealing.
+pub async fn seal_secret(plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
+    let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
+    sealer.seal(plaintext, scope).await
+}
+
+/// Unseals one secret field, rejecting an unknown envelope version first.
+pub async fn unseal_secret(sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
+    sealed.check_version()?;
+    let sealer = credential_sealer().ok_or(SealedCredentialError::NoSealer)?;
+    sealer.unseal(sealed, scope).await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use parking_lot::Mutex;
+
+    /// Stands in for the KMS-backed sealer: records the context it was called
+    /// with, and refuses a ciphertext presented under a different one.
+    #[derive(Default)]
+    struct FakeSealer {
+        sealed_contexts: Mutex<Vec<HashMap<String, String>>>,
+    }
+
+    #[async_trait]
+    impl CredentialSealer for FakeSealer {
+        async fn seal(&self, plaintext: &str, scope: &SealScope) -> Result<SealedCredential, SealedCredentialError> {
+            let context = scope.encryption_context();
+            self.sealed_contexts.lock().push(context.clone());
+            let mut bound = serde_json::to_string(&context).expect("context serializes");
+            bound.push('|');
+            bound.push_str(plaintext);
+            Ok(SealedCredential {
+                v: SEALED_CREDENTIAL_VERSION,
+                key_id: "key-1".to_string(),
+                key_version: Some("3".to_string()),
+                alg: "AES-256-GCM".to_string(),
+                ct: base64_simd::STANDARD.encode_to_string(bound.as_bytes()),
+            })
+        }
+
+        async fn unseal(&self, sealed: &SealedCredential, scope: &SealScope) -> Result<String, SealedCredentialError> {
+            let raw = base64_simd::STANDARD
+                .decode_to_vec(sealed.ct.as_bytes())
+                .map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
+            let bound = String::from_utf8(raw).map_err(|err| SealedCredentialError::Malformed(err.to_string()))?;
+            let expected = serde_json::to_string(&scope.encryption_context()).expect("context serializes");
+            bound
+                .strip_prefix(&expected)
+                .and_then(|rest| rest.strip_prefix('|'))
+                .map(str::to_string)
+                .ok_or_else(|| SealedCredentialError::Kms("encryption context mismatch".to_string()))
+        }
+    }
+
+    fn scope(owner: &str) -> SealScope {
+        SealScope::new(SealedCredentialStore::OnDemandMigration, owner, "secret_key")
+    }
+
+    #[tokio::test]
+    async fn seal_round_trips_and_binds_the_scope() {
+        let sealer = Arc::new(FakeSealer::default());
+        let sealed = sealer.seal("super-secret", &scope("photos")).await.expect("seal");
+        assert_eq!(sealed.v, SEALED_CREDENTIAL_VERSION);
+        assert_eq!(sealed.key_version.as_deref(), Some("3"));
+        assert_eq!(sealer.unseal(&sealed, &scope("photos")).await.expect("unseal"), "super-secret");
+
+        // The same ciphertext under another bucket must not unseal.
+        let err = sealer
+            .unseal(&sealed, &scope("other-bucket"))
+            .await
+            .expect_err("a ciphertext must not move between owners");
+        assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
+
+        // Nor under another field of the same record.
+        let other_field = SealScope::new(SealedCredentialStore::OnDemandMigration, "photos", "session_token");
+        let err = sealer
+            .unseal(&sealed, &other_field)
+            .await
+            .expect_err("a ciphertext must not move between fields");
+        assert!(matches!(err, SealedCredentialError::Kms(_)), "{err}");
+
+        let contexts = sealer.sealed_contexts.lock();
+        assert_eq!(contexts.len(), 1);
+        assert_eq!(contexts[0]["rustfs:store"], "on-demand-migration");
+        assert_eq!(contexts[0]["rustfs:owner"], "photos");
+        assert_eq!(contexts[0]["rustfs:field"], "secret_key");
+    }
+
+    #[tokio::test]
+    async fn an_unknown_envelope_version_is_rejected_before_the_sealer_is_asked() {
+        let sealed = SealedCredential {
+            v: SEALED_CREDENTIAL_VERSION + 1,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        assert_eq!(
+            sealed.check_version().expect_err("a newer envelope must not be read"),
+            SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
+        );
+        // The global helper reports the version, not "no sealer", even in a
+        // process where none is installed.
+        assert_eq!(
+            unseal_secret(&sealed, &scope("photos")).await.expect_err("version first"),
+            SealedCredentialError::UnsupportedVersion(SEALED_CREDENTIAL_VERSION + 1)
+        );
+    }
+
+    #[tokio::test]
+    async fn without_a_sealer_both_directions_fail_closed() {
+        // This test binary installs no sealer, so the global helpers must
+        // report NoSealer rather than fall back to clear text.
+        assert!(credential_sealer().is_none(), "no sealer is installed in unit tests");
+        assert_eq!(
+            seal_secret("super-secret", &scope("photos")).await.expect_err("seal"),
+            SealedCredentialError::NoSealer
+        );
+        let sealed = SealedCredential {
+            v: SEALED_CREDENTIAL_VERSION,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        assert_eq!(
+            unseal_secret(&sealed, &scope("photos")).await.expect_err("unseal"),
+            SealedCredentialError::NoSealer
+        );
+    }
+
+    #[test]
+    fn debug_and_serde_keep_the_on_disk_shape_stable() {
+        let sealed = SealedCredential {
+            v: 1,
+            key_id: "key-1".to_string(),
+            key_version: None,
+            alg: "AES-256-GCM".to_string(),
+            ct: "Zm9v".to_string(),
+        };
+        // key_version is omitted when absent, so an envelope from a backend
+        // without version history stays compact.
+        assert_eq!(
+            serde_json::to_string(&sealed).expect("serialize"),
+            r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#
+        );
+        let parsed: SealedCredential = serde_json::from_str(r#"{"v":1,"key_id":"key-1","alg":"AES-256-GCM","ct":"Zm9v"}"#)
+            .expect("an envelope without key_version parses");
+        assert_eq!(parsed, sealed);
+
+        let rendered = format!("{sealed:?}");
+        assert!(rendered.contains("key-1"), "{rendered}");
+        assert!(!rendered.contains("Zm9v"), "Debug must not print the ciphertext: {rendered}");
+    }
+
+    #[test]
+    fn a_malformed_envelope_is_a_typed_error() {
+        let err = serde_json::from_str::<SealedCredential>(r#"{"v":1,"key_id":"key-1"}"#)
+            .map_err(|err| SealedCredentialError::Malformed(err.to_string()))
+            .expect_err("a truncated envelope must not parse");
+        assert!(matches!(err, SealedCredentialError::Malformed(_)), "{err}");
+    }
+}
@@ -18,25 +18,203 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de};

-#[derive(Serialize, Deserialize, Default, Debug, Clone)]
+#[derive(Serialize, Deserialize, Default, Clone)]
+#[serde(default)]
+pub struct TierServicePrincipalAuth {
+    #[serde(rename = "TenantID", alias = "tenantID", alias = "tenant_id")]
+    pub tenant_id: String,
+    #[serde(rename = "ClientID", alias = "clientID", alias = "client_id")]
+    pub client_id: String,
+    #[serde(rename = "ClientSecret", alias = "clientSecret", alias = "client_secret")]
+    pub client_secret: String,
+}
+
+impl TierServicePrincipalAuth {
+    pub(crate) fn is_empty(&self) -> bool {
+        self.tenant_id.is_empty() && self.client_id.is_empty() && self.client_secret.is_empty()
+    }
+}
+
+impl std::fmt::Debug for TierServicePrincipalAuth {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TierServicePrincipalAuth")
+            .field("tenant_id", &self.tenant_id)
+            .field("client_id", &self.client_id)
+            .field("client_secret", &"REDACTED")
+            .finish()
+    }
+}
+
+#[derive(Serialize, Deserialize, Default, Clone)]
 #[serde(default)]
 pub struct TierCreds {
-    #[serde(rename = "accessKey")]
+    #[serde(rename = "access", alias = "accessKey")]
    pub access_key: String,
-    #[serde(rename = "secretKey")]
+    #[serde(rename = "secret", alias = "secretKey")]
    pub secret_key: String,

-    #[serde(rename = "awsRole")]
+    #[serde(rename = "awsrole", alias = "awsRole")]
    pub aws_role: bool,
-    #[serde(rename = "awsRoleWebIdentityTokenFile")]
+    #[serde(rename = "awsroleWebIdentity", alias = "awsRoleWebIdentityTokenFile")]
    pub aws_role_web_identity_token_file: String,
-    #[serde(rename = "awsRoleArn")]
+    #[serde(rename = "awsroleARN", alias = "awsRoleArn", alias = "awsRoleARN")]
    pub aws_role_arn: String,

-    //azsp: ServicePrincipalAuth,
+    #[serde(rename = "azSP", alias = "azsp", skip_serializing_if = "TierServicePrincipalAuth::is_empty")]
+    pub azure_service_principal: TierServicePrincipalAuth,

-    //#[serde(rename = "credsJson")]
+    #[serde(
+        rename = "creds",
+        alias = "credsJson",
+        alias = "credsJSON",
+        alias = "creds_json",
+        default,
+        skip_serializing_if = "Vec::is_empty",
+        with = "base64_bytes"
+    )]
    pub creds_json: Vec<u8>,
 }
+
+impl std::fmt::Debug for TierCreds {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TierCreds")
+            .field("access_key", &self.access_key)
+            .field("secret_key", &"REDACTED")
+            .field("aws_role", &self.aws_role)
+            .field(
+                "aws_role_web_identity_token_file",
+                &(!self.aws_role_web_identity_token_file.is_empty()).then_some("REDACTED"),
+            )
+            .field("aws_role_arn", &self.aws_role_arn)
+            .field("azure_service_principal", &self.azure_service_principal)
+            .field("creds_json", &(!self.creds_json.is_empty()).then_some("REDACTED"))
+            .finish()
+    }
+}
+
+mod base64_bytes {
+    use super::*;
+
+    pub(super) fn serialize<S>(value: &[u8], serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&base64_simd::STANDARD.encode_to_string(value))
+    }
+
+    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        #[derive(Deserialize)]
+        #[serde(untagged)]
+        enum EncodedBytes {
+            Base64(String),
+            Legacy(Vec<u8>),
+        }
+
+        match EncodedBytes::deserialize(deserializer)? {
+            EncodedBytes::Base64(value) => base64_simd::STANDARD
+                .decode_to_vec(value.as_bytes())
+                .or_else(|_| base64_simd::STANDARD_NO_PAD.decode_to_vec(value.as_bytes()))
+                .or_else(|_| base64_simd::URL_SAFE.decode_to_vec(value.as_bytes()))
+                .or_else(|_| base64_simd::URL_SAFE_NO_PAD.decode_to_vec(value.as_bytes()))
+                .map_err(de::Error::custom),
+            EncodedBytes::Legacy(value) => Ok(value),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tier_creds_accepts_madmin_wire_names_and_base64_gcs_json() {
+        let service_account = r#"{"type":"service_account","project_id":"tier-🚀x"}"#.as_bytes();
+        let encoded = "eyJ0eXBlIjoic2VydmljZV9hY2NvdW50IiwicHJvamVjdF9pZCI6InRpZXIt8J+agHgifQ==";
+        let creds: TierCreds = serde_json::from_value(serde_json::json!({
+            "access": "access",
+            "secret": "secret",
+            "awsrole": false,
+            "creds": encoded,
+        }))
+        .expect("madmin tier credentials should decode");
+
+        assert_eq!(creds.access_key, "access");
+        assert_eq!(creds.secret_key, "secret");
+        assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
+
+        let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
+        assert_eq!(wire["access"], "access");
+        assert_eq!(wire["secret"], "secret");
+        assert_eq!(wire["creds"], encoded);
+        assert!(wire.get("accessKey").is_none());
+        assert!(wire.get("secretKey").is_none());
+
+        let legacy: TierCreds = serde_json::from_value(serde_json::json!({
+            "accessKey": "legacy-access",
+            "secretKey": "legacy-secret",
+            "credsJson": service_account,
+        }))
+        .expect("the former RustFS field names and byte-array encoding should remain readable");
+        assert_eq!(legacy.access_key, "legacy-access");
+        assert_eq!(legacy.secret_key, "legacy-secret");
+        assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
+    }
+
+    #[test]
+    fn tier_creds_accepts_all_supported_base64_alphabets_and_padding_modes() {
+        let service_account = r#"{"type":"service_account","project_id":"tier-🚀"}"#.as_bytes();
+        for encoder in [
+            base64_simd::STANDARD,
+            base64_simd::STANDARD_NO_PAD,
+            base64_simd::URL_SAFE,
+            base64_simd::URL_SAFE_NO_PAD,
+        ] {
+            let encoded = encoder.encode_to_string(service_account);
+            let creds: TierCreds = serde_json::from_value(serde_json::json!({ "creds": encoded }))
+                .expect("all supported madmin base64 forms should decode");
+            assert_eq!(creds.creds_json, service_account);
+        }
+    }
+
+    #[test]
+    fn tier_creds_debug_redacts_secret_payloads() {
+        let creds = TierCreds {
+            access_key: "access".to_string(),
+            secret_key: "tier-secret-value".to_string(),
+            aws_role_web_identity_token_file: "/var/run/private-token".to_string(),
+            creds_json: br#"{"private_key":"gcs-private-key-value"}"#.to_vec(),
+            ..Default::default()
+        };
+
+        let rendered = format!("{creds:?}");
+        assert!(!rendered.contains("tier-secret-value"));
+        assert!(!rendered.contains("/var/run/private-token"));
+        assert!(!rendered.contains("gcs-private-key-value"));
+    }
+
+    #[test]
+    fn tier_creds_accepts_canonical_madmin_azure_service_principal_wire_shape() {
+        let creds: TierCreds = serde_json::from_value(serde_json::json!({
+            "azSP": {
+                "TenantID": "tenant",
+                "ClientID": "client",
+                "ClientSecret": "service-principal-secret"
+            }
+        }))
+        .expect("canonical madmin azure service principal credentials should decode");
+
+        assert_eq!(creds.azure_service_principal.tenant_id, "tenant");
+        assert_eq!(creds.azure_service_principal.client_id, "client");
+        assert_eq!(creds.azure_service_principal.client_secret, "service-principal-secret");
+        let wire = serde_json::to_value(&creds).expect("canonical madmin credentials should encode");
+        assert_eq!(wire["azSP"]["TenantID"], "tenant");
+        assert_eq!(wire["azSP"]["ClientID"], "client");
+        assert_eq!(wire["azSP"]["ClientSecret"], "service-principal-secret");
+        assert!(!format!("{creds:?}").contains("service-principal-secret"));
+    }
+}
@@ -42,7 +42,7 @@ const WASABI_ALTERNATIVE_ENDPOINTS: &[(&str, &str)] = &[
 pub enum TierType {
    #[default]
    Unsupported,
-    #[serde(rename = "s3")]
+    #[serde(rename = "s3", alias = "S3")]
    S3,
    #[serde(rename = "wasabi")]
    Wasabi,
@@ -58,7 +58,7 @@ pub enum TierType {
    Huaweicloud,
    #[serde(rename = "azure")]
    Azure,
-    #[serde(rename = "gcs")]
+    #[serde(rename = "gcs", alias = "GCS")]
    GCS,
    #[serde(rename = "r2")]
    R2,
@@ -138,16 +138,18 @@ impl TierType {
    }
 }

-#[derive(Default, Debug, Serialize, Deserialize)]
+pub(crate) const TIER_CREDENTIAL_REDACTED: &str = "REDACTED";
+
+#[derive(Default, Serialize, Deserialize)]
 #[serde(default)]
 pub struct TierConfig {
    #[serde(skip)]
    pub version: String,
-    #[serde(rename = "type")]
+    #[serde(rename = "type", alias = "Type")]
    pub tier_type: TierType,
-    #[serde(skip)]
+    #[serde(rename = "Name", alias = "name", skip_serializing)]
    pub name: String,
-    #[serde(rename = "s3", skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "s3", alias = "S3", skip_serializing_if = "Option::is_none")]
    pub s3: Option<TierS3>,
    #[serde(rename = "wasabi", skip_serializing_if = "Option::is_none")]
    pub wasabi: Option<TierWasabi>,
@@ -159,7 +161,7 @@ pub struct TierConfig {
    pub huaweicloud: Option<TierHuaweicloud>,
    #[serde(rename = "azure", skip_serializing_if = "Option::is_none")]
    pub azure: Option<TierAzure>,
-    #[serde(rename = "gcs", skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "gcs", alias = "GCS", skip_serializing_if = "Option::is_none")]
    pub gcs: Option<TierGCS>,
    #[serde(rename = "r2", skip_serializing_if = "Option::is_none")]
    pub r2: Option<TierR2>,
@@ -170,109 +172,91 @@ pub struct TierConfig {
 }

 impl Clone for TierConfig {
-    fn clone(&self) -> TierConfig {
-        let mut s3 = None;
-        let mut wasabi = None;
-        let mut r = None;
-        let mut compatible_backend = None;
-        let mut aliyun = None;
-        let mut tencent = None;
-        let mut huaweicloud = None;
-        let mut azure = None;
-        let mut gcs = None;
-        let mut r2 = None;
-        match self.tier_type {
-            TierType::S3 => {
-                if let Some(s3_) = self.s3.as_ref() {
-                    let mut s3_clone = s3_.clone();
-                    s3_clone.secret_key = "REDACTED".to_string();
-                    s3 = Some(s3_clone);
-                }
-            }
-            TierType::Wasabi => {
-                if let Some(wasabi_) = self.wasabi.as_ref() {
-                    let mut wasabi_clone = wasabi_.clone();
-                    wasabi_clone.secret_key = "REDACTED".to_string();
-                    wasabi = Some(wasabi_clone);
-                }
-            }
-            TierType::RustFS => {
-                if let Some(r_) = self.rustfs.as_ref() {
-                    let mut r_clone = r_.clone();
-                    r_clone.secret_key = "REDACTED".to_string();
-                    r = Some(r_clone);
-                }
-            }
-            TierType::MinIO => {
-                if let Some(compatible_backend_) = self.minio.as_ref() {
-                    let mut compatible_backend_clone = compatible_backend_.clone();
-                    compatible_backend_clone.secret_key = "REDACTED".to_string();
-                    compatible_backend = Some(compatible_backend_clone);
-                }
-            }
-            TierType::Aliyun => {
-                if let Some(aliyun_) = self.aliyun.as_ref() {
-                    let mut aliyun_clone = aliyun_.clone();
-                    aliyun_clone.secret_key = "REDACTED".to_string();
-                    aliyun = Some(aliyun_clone);
-                }
-            }
-            TierType::Tencent => {
-                if let Some(tencent_) = self.tencent.as_ref() {
-                    let mut tencent_clone = tencent_.clone();
-                    tencent_clone.secret_key = "REDACTED".to_string();
-                    tencent = Some(tencent_clone);
-                }
-            }
-            TierType::Huaweicloud => {
-                if let Some(huaweicloud_) = self.huaweicloud.as_ref() {
-                    let mut huaweicloud_clone = huaweicloud_.clone();
-                    huaweicloud_clone.secret_key = "REDACTED".to_string();
-                    huaweicloud = Some(huaweicloud_clone);
-                }
-            }
-            TierType::Azure => {
-                if let Some(azure_) = self.azure.as_ref() {
-                    let mut azure_clone = azure_.clone();
-                    azure_clone.secret_key = "REDACTED".to_string();
-                    azure = Some(azure_clone);
-                }
-            }
-            TierType::GCS => {
-                if let Some(gcs_) = self.gcs.as_ref() {
-                    let mut gcs_clone = gcs_.clone();
-                    gcs_clone.creds = "REDACTED".to_string();
-                    gcs = Some(gcs_clone);
-                }
-            }
-            TierType::R2 => {
-                if let Some(r2_) = self.r2.as_ref() {
-                    let mut r2_clone = r2_.clone();
-                    r2_clone.secret_key = "REDACTED".to_string();
-                    r2 = Some(r2_clone);
-                }
-            }
-            _ => (),
-        }
-        TierConfig {
-            version: self.version.clone(),
-            tier_type: self.tier_type.clone(),
-            name: self.name.clone(),
-            s3,
-            wasabi,
-            rustfs: r,
-            minio: compatible_backend,
-            aliyun,
-            tencent,
-            huaweicloud,
-            azure,
-            gcs,
-            r2,
-        }
+    fn clone(&self) -> Self {
+        self.redacted()
    }
 }

 impl TierConfig {
+    pub(crate) fn redacted(&self) -> Self {
+        let mut redacted = Self {
+            version: self.version.clone(),
+            tier_type: self.tier_type.clone(),
+            name: self.name.clone(),
+            ..Default::default()
+        };
+        match self.tier_type {
+            TierType::S3 => {
+                redacted.s3 = self.s3.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    if !backend.aws_role_web_identity_token_file.is_empty() {
+                        backend.aws_role_web_identity_token_file = TIER_CREDENTIAL_REDACTED.to_string();
+                    }
+                    backend
+                });
+            }
+            TierType::Wasabi => {
+                redacted.wasabi = self.wasabi.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::RustFS => {
+                redacted.rustfs = self.rustfs.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::MinIO => {
+                redacted.minio = self.minio.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Aliyun => {
+                redacted.aliyun = self.aliyun.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Tencent => {
+                redacted.tencent = self.tencent.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Huaweicloud => {
+                redacted.huaweicloud = self.huaweicloud.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Azure => {
+                redacted.azure = self.azure.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    if !backend.sp_auth.client_secret.is_empty() {
+                        backend.sp_auth.client_secret = TIER_CREDENTIAL_REDACTED.to_string();
+                    }
+                    backend
+                });
+            }
+            TierType::GCS => {
+                redacted.gcs = self.gcs.clone().map(|mut backend| {
+                    backend.creds = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::R2 => {
+                redacted.r2 = self.r2.clone().map(|mut backend| {
+                    backend.secret_key = TIER_CREDENTIAL_REDACTED.to_string();
+                    backend
+                });
+            }
+            TierType::Unsupported => {}
+        }
+        redacted
+    }
+
    pub(crate) fn clone_with_credentials(&self) -> Self {
        Self {
            version: self.version.clone(),
@@ -372,31 +356,61 @@ impl TierConfig {
    }
 }

+impl std::fmt::Debug for TierConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let redacted = self.redacted();
+        f.debug_struct("TierConfig")
+            .field("version", &redacted.version)
+            .field("tier_type", &redacted.tier_type)
+            .field("name", &redacted.name)
+            .field("s3", &redacted.s3)
+            .field("wasabi", &redacted.wasabi)
+            .field("aliyun", &redacted.aliyun)
+            .field("tencent", &redacted.tencent)
+            .field("huaweicloud", &redacted.huaweicloud)
+            .field("azure", &redacted.azure)
+            .field("gcs", &redacted.gcs)
+            .field("r2", &redacted.r2)
+            .field("rustfs", &redacted.rustfs)
+            .field("minio", &redacted.minio)
+            .finish()
+    }
+}
+
 //type S3Options = impl Fn(TierS3) -> Pin<Box<Result<()>>> + Send + Sync + 'static;

 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct TierS3 {
+    #[serde(alias = "Name")]
    pub name: String,
+    #[serde(alias = "Endpoint")]
    pub endpoint: String,
-    #[serde(rename = "accessKey")]
+    #[serde(rename = "accessKey", alias = "AccessKey")]
    pub access_key: String,
-    #[serde(rename = "secretKey")]
+    #[serde(rename = "secretKey", alias = "SecretKey")]
    pub secret_key: String,
+    #[serde(alias = "Bucket")]
    pub bucket: String,
+    #[serde(alias = "Prefix")]
    pub prefix: String,
+    #[serde(alias = "Region")]
    pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
    pub storage_class: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRole", alias = "awsRole", skip_serializing)]
    pub aws_role: bool,
-    #[serde(skip)]
+    #[serde(
+        rename = "AWSRoleWebIdentityTokenFile",
+        alias = "awsRoleWebIdentityTokenFile",
+        skip_serializing
+    )]
    pub aws_role_web_identity_token_file: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleARN", alias = "awsRoleARN", alias = "awsRoleArn", skip_serializing)]
    pub aws_role_arn: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleSessionName", alias = "awsRoleSessionName", skip_serializing)]
    pub aws_role_session_name: String,
-    #[serde(skip)]
+    #[serde(rename = "AWSRoleDurationSeconds", alias = "awsRoleDurationSeconds", skip_serializing)]
    pub aws_role_duration_seconds: i32,
 }

@@ -623,8 +637,11 @@ pub struct TierHuaweicloud {
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct ServicePrincipalAuth {
+    #[serde(alias = "TenantID")]
    pub tenant_id: String,
+    #[serde(alias = "ClientID")]
    pub client_id: String,
+    #[serde(alias = "ClientSecret")]
    pub client_secret: String,
 }

@@ -640,9 +657,9 @@ pub struct TierAzure {
    pub bucket: String,
    pub prefix: String,
    pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
    pub storage_class: String,
-    #[serde(rename = "spAuth")]
+    #[serde(rename = "spAuth", alias = "SPAuth")]
    pub sp_auth: ServicePrincipalAuth,
 }

@@ -696,14 +713,19 @@ fn AzureStorageClass(sc string) func(az *TierAzure) error {
 #[derive(Serialize, Deserialize, Default, Debug, Clone)]
 #[serde(default)]
 pub struct TierGCS {
+    #[serde(alias = "Name")]
    pub name: String,
+    #[serde(alias = "Endpoint")]
    pub endpoint: String,
-    #[serde(rename = "creds")]
+    #[serde(rename = "creds", alias = "Creds")]
    pub creds: String,
+    #[serde(alias = "Bucket")]
    pub bucket: String,
+    #[serde(alias = "Prefix")]
    pub prefix: String,
+    #[serde(alias = "Region")]
    pub region: String,
-    #[serde(rename = "storageClass")]
+    #[serde(rename = "storageClass", alias = "StorageClass")]
    pub storage_class: String,
 }

@@ -725,6 +747,43 @@ pub struct TierR2 {
 mod tests {
    use super::*;

+    #[test]
+    fn s3_gcs_type_uppercase_aliases_preserve_lowercase_output() {
+        let s3: TierType = serde_json::from_str(r#""S3""#).expect("uppercase S3 wire value should decode");
+        let gcs: TierType = serde_json::from_str(r#""GCS""#).expect("uppercase GCS wire value should decode");
+        assert!(matches!(s3, TierType::S3));
+        assert!(matches!(gcs, TierType::GCS));
+        assert_eq!(serde_json::to_string(&s3).expect("S3 type should encode"), r#""s3""#);
+        assert_eq!(serde_json::to_string(&gcs).expect("GCS type should encode"), r#""gcs""#);
+    }
+
+    #[test]
+    fn azure_service_principal_accepts_canonical_madmin_field_names() {
+        for field in ["TenantID", "ClientID", "ClientSecret"] {
+            let mut sp_auth = serde_json::Map::new();
+            sp_auth.insert(field.to_string(), serde_json::Value::String("present".to_string()));
+            let config: TierConfig = serde_json::from_value(serde_json::json!({
+                "type": "azure",
+                "Name": "COLD-AZURE",
+                "azure": {
+                    "name": "COLD-AZURE",
+                    "endpoint": "https://azure.example.invalid",
+                    "accessKey": "account",
+                    "secretKey": "key",
+                    "bucket": "archive",
+                    "SPAuth": sp_auth
+                }
+            }))
+            .expect("mixed RustFS/madmin Azure payload should decode");
+            let sp_auth = &config.azure.expect("Azure payload should exist").sp_auth;
+
+            assert!(
+                !sp_auth.tenant_id.is_empty() || !sp_auth.client_id.is_empty() || !sp_auth.client_secret.is_empty(),
+                "canonical {field} must not be silently discarded"
+            );
+        }
+    }
+
    fn wasabi_config() -> TierWasabi {
        TierWasabi {
            name: "COLD-WASABI".to_string(),
@@ -838,9 +897,14 @@ mod tests {
        let config = TierConfig {
            tier_type: TierType::Wasabi,
            wasabi: Some(wasabi_config()),
+            rustfs: Some(TierRustFS {
+                access_key: "inactive-access".to_string(),
+                secret_key: "inactive-secret".to_string(),
+                ..Default::default()
+            }),
            ..Default::default()
        };
-        let redacted = config.clone();
+        let redacted = config.redacted();
        assert_eq!(
            redacted
                .wasabi
@@ -849,21 +913,46 @@ mod tests {
                .secret_key,
            "REDACTED"
        );
+        assert!(redacted.rustfs.is_none(), "the external view should retain only the active provider");
+        let cloned = config.clone();
+        assert_eq!(cloned.wasabi.expect("redacted Wasabi clone should remain").secret_key, "REDACTED");
+        assert!(cloned.rustfs.is_none(), "ordinary Clone must retain its redacted API semantics");
+        let preserved = config.clone_with_credentials();
        assert_eq!(
-            config
-                .clone_with_credentials()
+            preserved
                .wasabi
                .as_ref()
-                .expect("credential-bearing Wasabi payload should remain")
+                .expect("credential-bearing Wasabi snapshot should remain")
                .secret_key,
            "secret"
        );
+        assert_eq!(
+            preserved
+                .rustfs
+                .expect("credential-bearing snapshots should preserve inactive provider data")
+                .secret_key,
+            "inactive-secret"
+        );

        let mut debug_config = wasabi_config();
        debug_config.secret_key = "wasabi-debug-secret-value".to_string();
        let debug = format!("{debug_config:?}");
        assert!(debug.contains("REDACTED"));
        assert!(!debug.contains("wasabi-debug-secret-value"));
+
+        let debug = format!(
+            "{:?}",
+            TierConfig {
+                tier_type: TierType::RustFS,
+                rustfs: Some(TierRustFS {
+                    secret_key: "rustfs-debug-secret-value".to_string(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }
+        );
+        assert!(debug.contains("REDACTED"));
+        assert!(!debug.contains("rustfs-debug-secret-value"));
    }

    #[test]
@@ -894,7 +983,7 @@ mod tests {
        assert_eq!(encoded, expected);
        let decoded: TierConfig = serde_json::from_value(encoded).expect("Wasabi Admin JSON should decode");
        assert!(matches!(decoded.tier_type, TierType::Wasabi));
-        let redacted = config.clone();
+        let redacted = config.redacted();
        assert_eq!(
            config
                .wasabi
@@ -916,4 +1005,87 @@ mod tests {
            "REDACTED"
        );
    }
+
+    #[test]
+    fn api_serialization_and_debug_redact_s3_gcs_and_azure_credentials() {
+        let cases = [
+            (
+                "s3",
+                TierConfig {
+                    tier_type: TierType::S3,
+                    s3: Some(TierS3 {
+                        secret_key: "s3-secret-bytes".to_string(),
+                        aws_role_web_identity_token_file: "/var/run/s3-private-token".to_string(),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["s3-secret-bytes", "/var/run/s3-private-token"],
+            ),
+            (
+                "gcs",
+                TierConfig {
+                    tier_type: TierType::GCS,
+                    gcs: Some(TierGCS {
+                        creds: r#"{"type":"service_account","private_key":"gcs-private-key-bytes"}"#.to_string(),
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["gcs-private-key-bytes"],
+            ),
+            (
+                "azure",
+                TierConfig {
+                    tier_type: TierType::Azure,
+                    azure: Some(TierAzure {
+                        secret_key: "azure-account-secret-bytes".to_string(),
+                        sp_auth: ServicePrincipalAuth {
+                            client_secret: "azure-client-secret-bytes".to_string(),
+                            ..Default::default()
+                        },
+                        ..Default::default()
+                    }),
+                    ..Default::default()
+                },
+                vec!["azure-account-secret-bytes", "azure-client-secret-bytes"],
+            ),
+        ];
+
+        for (provider, config, secrets) in cases {
+            let api = serde_json::to_string(&config.redacted()).expect("redacted API config should serialize");
+            let debug = format!("{config:?}");
+            assert!(api.contains(TIER_CREDENTIAL_REDACTED), "{provider} API output should be visibly redacted");
+            assert!(
+                debug.contains(TIER_CREDENTIAL_REDACTED),
+                "{provider} Debug output should be visibly redacted"
+            );
+            for secret in secrets {
+                assert!(!api.contains(secret), "{provider} API output exposed credential bytes");
+                assert!(!debug.contains(secret), "{provider} Debug output exposed credential bytes");
+            }
+        }
+    }
+
+    #[test]
+    fn azure_static_account_redaction_preserves_an_empty_service_principal_secret() {
+        let config = TierConfig {
+            tier_type: TierType::Azure,
+            azure: Some(TierAzure {
+                secret_key: "azure-account-secret-bytes".to_string(),
+                sp_auth: ServicePrincipalAuth::default(),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let api = serde_json::to_value(config.redacted()).expect("redacted Azure API config should serialize");
+        let debug = format!("{config:?}");
+
+        assert_eq!(api["azure"]["secretKey"], TIER_CREDENTIAL_REDACTED);
+        assert_eq!(api["azure"]["spAuth"]["client_secret"], "");
+        assert!(debug.contains("client_secret: \"\""));
+        assert!(!debug.contains("client_secret: \"REDACTED\""));
+        assert!(!debug.contains("azure-account-secret-bytes"));
+    }
 }
@@ -20,7 +20,7 @@

 use crate::error::is_err_bucket_not_found;
 use crate::services::tier::{
-    tier::{ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
+    tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
    tier_config::{TierConfig, TierType},
    tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
    warm_backend_aliyun::WarmBackendAliyun,
@@ -55,18 +55,21 @@ use s3s::header::{
 };
 use std::collections::HashMap;
 use std::sync::Arc;
+use std::time::Duration;
 use time::OffsetDateTime;
 use time::format_description::well_known::{Rfc2822, Rfc3339};
+use tokio::io::AsyncReadExt;
 use tracing::{info, warn};

 pub type WarmBackendImpl = Box<dyn WarmBackend + Send + Sync + 'static>;

-const PROBE_OBJECT: &str = "probeobject";
-
 /// Largest object the S3-compatible warm backends accept for a multipart put.
 pub(crate) const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
 /// Part-count ceiling S3-compatible services impose on a multipart upload.
 pub(crate) const MAX_PARTS_COUNT: i64 = 10000;
+pub(crate) const WARM_BACKEND_PROBE_TIMEOUT: Duration = Duration::from_secs(30);
+const WARM_BACKEND_PROBE_RECONCILE_INTERVAL: Duration = Duration::from_secs(1);
+const WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT: Duration = Duration::from_secs(1);

 #[derive(Default)]
 pub struct WarmBackendGetOpts {
@@ -260,6 +263,23 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> {
    pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>,
 }

+/// Return the authority format accepted by `TransitionClient::new` while
+/// retaining an explicitly configured port. `url::Url::host_str()` omits the
+/// brackets needed when an IPv6 literal is combined with a port.
+pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Error> {
+    let host = url
+        .host_str()
+        .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
+    let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });
+    if host.starts_with('[') && host.ends_with(']') {
+        Ok(format!("{host}:{port}"))
+    } else if host.contains(':') {
+        Ok(format!("[{host}]:{port}"))
+    } else {
+        Ok(format!("{host}:{port}"))
+    }
+}
+
 /// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
 ///
 /// Credential, bucket, and endpoint validation run in this order because the
@@ -298,17 +318,11 @@ pub(crate) async fn new_s3_compatible_warm_backend(
        bucket_lookup: params.bucket_lookup,
        ..Default::default()
    };
-    let scheme = u.scheme();
-    let default_port = if scheme == "https" { 443 } else { 80 };
-    let host = u
-        .host_str()
-        .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
-    // Runs after the host-presence check above (not immediately after Url::parse) so a
-    // host-less endpoint still reports this constructor's own "missing host" text instead of
-    // validate_endpoint's differently-worded rejection for the same input.
+    let endpoint = endpoint_authority(&u)?;
+    // Run the SSRF guard after the host-presence check so a host-less endpoint
+    // keeps this constructor's stable error text.
    (params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
-    let client =
-        TransitionClient::new(&format!("{}:{}", host, u.port().unwrap_or(default_port)), opts, params.provider_tag).await?;
+    let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;

    let client = Arc::new(client);
    let core = TransitionCore(Arc::clone(&client));
@@ -451,25 +465,187 @@ impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
    }
 }

-pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
-    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
-    w.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
-    let remote_version_id = w
-        .put(PROBE_OBJECT, ReaderImpl::Body(Bytes::from("RustFS".as_bytes().to_vec())), 5)
-        .await
-        .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+async fn remove_discovered_probe_candidate(
+    w: &WarmBackendImpl,
+    probe_object: &str,
+    candidate: TransitionCandidateProbe,
+) -> Result<bool, std::io::Error> {
+    match candidate {
+        TransitionCandidateProbe::Missing => Ok(false),
+        TransitionCandidateProbe::VersionedPresent(remote_version_id) => {
+            w.remove_exact(probe_object, &remote_version_id).await?;
+            Ok(true)
+        }
+        TransitionCandidateProbe::UnversionedPresent => {
+            w.remove(probe_object, "").await?;
+            Ok(true)
+        }
+        TransitionCandidateProbe::Ambiguous => {
+            Err(std::io::Error::other("remote tier probe PUT produced multiple possible versions"))
+        }
+        TransitionCandidateProbe::Unsupported => Err(std::io::Error::new(
+            std::io::ErrorKind::Unsupported,
+            "remote tier cannot discover the outcome of a probe PUT",
+        )),
+    }
+}

-    if w.validate_remote_version_id(&remote_version_id).is_err() {
-        w.remove_exact(PROBE_OBJECT, &remote_version_id)
+async fn compensate_uncertain_probe_put(
+    w: &WarmBackendImpl,
+    probe_object: &str,
+    settle_deadline: tokio::time::Instant,
+) -> Result<(), std::io::Error> {
+    let final_deadline = settle_deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT;
+    let mut removed_any = false;
+    while tokio::time::Instant::now() < settle_deadline {
+        let candidate = match tokio::time::timeout_at(settle_deadline, w.probe_transition_candidate(probe_object)).await {
+            Ok(candidate) => candidate?,
+            Err(_) => break,
+        };
+        if matches!(candidate, TransitionCandidateProbe::Missing) && removed_any {
+            break;
+        }
+        removed_any |= tokio::time::timeout_at(settle_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
            .await
-            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
-        return Err(ERR_TIER_INVALID_CONFIG.clone());
+            .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reconciling a remote tier probe PUT"))??;
+
+        let now = tokio::time::Instant::now();
+        if now >= settle_deadline {
+            break;
+        }
+        tokio::time::sleep_until(std::cmp::min(settle_deadline, now + WARM_BACKEND_PROBE_RECONCILE_INTERVAL)).await;
    }

-    let read_result = w.get(PROBE_OBJECT, &remote_version_id, WarmBackendGetOpts::default()).await;
-    let remove_result = w.remove(PROBE_OBJECT, &remote_version_id).await;
-    //xhttp.DrainBody(r);
-    if read_result.is_err() || remove_result.is_err() {
+    let candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
+        .await
+        .map_err(|_| {
+            std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming the final remote tier probe state")
+        })??;
+    if !tokio::time::timeout_at(final_deadline, remove_discovered_probe_candidate(w, probe_object, candidate))
+        .await
+        .map_err(|_| {
+            std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out removing the final remote tier probe candidate")
+        })??
+    {
+        return Ok(());
+    }
+
+    let final_candidate = tokio::time::timeout_at(final_deadline, w.probe_transition_candidate(probe_object))
+        .await
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out confirming remote tier probe cleanup"))??;
+    match final_candidate {
+        TransitionCandidateProbe::Missing => Ok(()),
+        _ => Err(std::io::Error::other("remote tier probe cleanup could not be confirmed")),
+    }
+}
+
+fn probe_cleanup_incomplete_error() -> AdminError {
+    let mut err = ERR_TIER_PERM_ERR.clone();
+    err.message = "Remote tier probe outcome is uncertain; cleanup is incomplete".to_string();
+    err
+}
+
+async fn check_warm_backend_with_deadlines(
+    w: Option<&WarmBackendImpl>,
+    deadline: tokio::time::Instant,
+    cleanup_deadline: tokio::time::Instant,
+) -> Result<(), AdminError> {
+    let w = w.ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
+    let probe_object = format!("rustfs-tier-probe-{}", uuid::Uuid::new_v4());
+    let timeout_error = || {
+        let mut err = ERR_TIER_BACKEND_IN_USE.clone();
+        err.message = "Timed out validating the remote tier mutation".to_string();
+        err
+    };
+    tokio::time::timeout_at(deadline, w.validate())
+        .await
+        .map_err(|_| timeout_error())?
+        .map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
+    let put_result =
+        tokio::time::timeout_at(deadline, w.put(&probe_object, ReaderImpl::Body(Bytes::from_static(b"RustFS")), 6)).await;
+    let remote_version_id = match put_result {
+        Ok(Ok(remote_version_id)) => remote_version_id,
+        Ok(Err(_)) => {
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => ERR_TIER_PERM_ERR.clone(),
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+        Err(_) => {
+            let err = timeout_error();
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => err,
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+    };
+
+    // S3-family backends do not replay a failed request before returning `Ok`,
+    // while GCS discovers every matching generation. The authoritative probe
+    // below therefore closes the acknowledged-PUT path; only an error or
+    // timeout needs the longer visibility reconciliation above.
+    let authoritative_candidate = match tokio::time::timeout_at(deadline, w.probe_transition_candidate(&probe_object)).await {
+        Ok(Ok(candidate)) => candidate,
+        Ok(Err(_)) | Err(_) => {
+            return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+                Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
+                Err(_) => probe_cleanup_incomplete_error(),
+            });
+        }
+    };
+    let response_version_is_valid = w.validate_remote_version_id(&remote_version_id).is_ok();
+    let response_matches_candidate = match &authoritative_candidate {
+        TransitionCandidateProbe::UnversionedPresent => remote_version_id.is_empty(),
+        TransitionCandidateProbe::VersionedPresent(candidate_version) => candidate_version == &remote_version_id,
+        TransitionCandidateProbe::Missing | TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => false,
+    };
+    if !response_version_is_valid || !response_matches_candidate {
+        return Err(match compensate_uncertain_probe_put(w, &probe_object, cleanup_deadline).await {
+            Ok(()) => ERR_TIER_INVALID_CONFIG.clone(),
+            Err(_) => probe_cleanup_incomplete_error(),
+        });
+    }
+
+    let read_result = tokio::time::timeout_at(deadline, async {
+        let mut reader = w
+            .get(
+                &probe_object,
+                &remote_version_id,
+                WarmBackendGetOpts {
+                    start_offset: 0,
+                    length: 7,
+                },
+            )
+            .await
+            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+        let mut body = Vec::new();
+        reader
+            .take(7)
+            .read_to_end(&mut body)
+            .await
+            .map_err(|_| ERR_TIER_PERM_ERR.clone())?;
+        if body != b"RustFS" {
+            return Err(ERR_TIER_PERM_ERR.clone());
+        }
+        Ok(())
+    })
+    .await
+    .map_err(|_| timeout_error())
+    .and_then(|result| result);
+    let cleanup_result = tokio::time::timeout_at(cleanup_deadline, async {
+        if !remove_discovered_probe_candidate(w, &probe_object, authoritative_candidate).await? {
+            return Err(std::io::Error::other("remote tier probe disappeared before cleanup"));
+        }
+        match w.probe_transition_candidate(&probe_object).await? {
+            TransitionCandidateProbe::Missing => Ok(()),
+            _ => Err(std::io::Error::other("remote tier probe remained after cleanup")),
+        }
+    })
+    .await;
+    if !matches!(cleanup_result, Ok(Ok(()))) {
+        return Err(probe_cleanup_incomplete_error());
+    }
+    if let Err(err) = read_result {
        //if is_err_bucket_not_found(&err) {
        //    return Err(ERR_TIER_BUCKET_NOT_FOUND);
        //}
@@ -477,12 +653,28 @@ pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), Admin
            return Err(ERR_TIER_MISSING_CREDENTIALS);
        }*/
        //else {
-        return Err(ERR_TIER_PERM_ERR.clone());
+        return Err(err);
        //}
    }
    Ok(())
 }

+/// Validate a backend using a caller-owned deadline while retaining a bounded
+/// reconciliation window for an uncertain probe PUT. The validation future is
+/// kept alive through cleanup so an outer timeout cannot abandon the remote
+/// probe object.
+pub(crate) async fn check_warm_backend_until(
+    w: Option<&WarmBackendImpl>,
+    deadline: tokio::time::Instant,
+) -> Result<(), AdminError> {
+    check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_FINAL_RECONCILE_TIMEOUT).await
+}
+
+pub async fn check_warm_backend(w: Option<&WarmBackendImpl>) -> Result<(), AdminError> {
+    let deadline = tokio::time::Instant::now() + WARM_BACKEND_PROBE_TIMEOUT;
+    check_warm_backend_with_deadlines(w, deadline, deadline + WARM_BACKEND_PROBE_TIMEOUT).await
+}
+
 pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBackendImpl, AdminError> {
    let mut d: Option<WarmBackendImpl> = None;
    match tier.tier_type {
@@ -701,7 +893,7 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
    let d: WarmBackendImpl = Box::new(MeteredWarmBackend { inner: d });

    if probe {
-        d.validate().await.map_err(|_| ERR_TIER_INVALID_CONFIG.clone())?;
+        check_warm_backend(Some(&d)).await?;
    }
    Ok(d)
 }
@@ -754,6 +946,7 @@ pub(crate) async fn new_transition_candidate_reconciler(
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::services::tier::test_util::{MockWarmBackend, MockWarmOp};
    use crate::services::tier::tier_config::TierWasabi;
    use std::sync::{
        Arc,
@@ -920,13 +1113,38 @@ mod tests {

    struct RejectingProbeVersionBackend {
        gets: Arc<AtomicUsize>,
+        present: Arc<std::sync::atomic::AtomicBool>,
        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        returned_version: String,
    }

    struct RecordingProbeBackend {
        get_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        present: Arc<std::sync::atomic::AtomicBool>,
        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+        remove_clears_candidate: bool,
        fail_get: bool,
+        body: ProbeBody,
+    }
+
+    struct HangingProbePutBackend {
+        put_started: Arc<tokio::sync::Notify>,
+        present: Arc<std::sync::atomic::AtomicBool>,
+        probes: Arc<AtomicUsize>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+    }
+
+    struct LateVisibleProbeBackend {
+        visible_at: tokio::time::Instant,
+        removed: Arc<std::sync::atomic::AtomicBool>,
+        probes: Arc<AtomicUsize>,
+        removed_versions: Arc<tokio::sync::Mutex<Vec<String>>>,
+    }
+
+    #[derive(Clone, Copy)]
+    enum ProbeBody {
+        Exact,
+        Mismatch,
    }

    #[async_trait::async_trait]
@@ -976,7 +1194,7 @@ mod tests {
        }

        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
-            Ok(uuid::Uuid::nil().to_string())
+            Ok(self.returned_version.clone())
        }

        async fn put_with_meta(
@@ -999,10 +1217,19 @@ mod tests {
        }

        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.present.store(false, Ordering::SeqCst);
            self.removed_versions.lock().await.push(rv.to_string());
            Ok(())
        }

+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
        async fn in_use(&self) -> Result<bool, std::io::Error> {
            Ok(false)
        }
@@ -1029,15 +1256,121 @@ mod tests {
            if self.fail_get {
                Err(std::io::Error::other("probe GET failed"))
            } else {
-                Ok(ReadCloser::new(std::io::Cursor::new(Vec::new())))
+                match self.body {
+                    ProbeBody::Exact => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFS".to_vec()))),
+                    ProbeBody::Mismatch => Ok(ReadCloser::new(std::io::Cursor::new(b"RustFT".to_vec()))),
+                }
            }
        }

        async fn remove(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            if self.remove_clears_candidate {
+                self.present.store(false, Ordering::SeqCst);
+            }
            self.removed_versions.lock().await.push(rv.to_string());
            Ok(())
        }

+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for HangingProbePutBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            self.put_started.notify_one();
+            std::future::pending().await
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("GET must not run after a timed out probe PUT"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
+        }
+
+        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.present.store(false, Ordering::SeqCst);
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            self.probes.fetch_add(1, Ordering::SeqCst);
+            if self.present.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
+        async fn in_use(&self) -> Result<bool, std::io::Error> {
+            Ok(false)
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl WarmBackend for LateVisibleProbeBackend {
+        async fn put(&self, _object: &str, _r: ReaderImpl, _length: i64) -> Result<String, std::io::Error> {
+            Err(std::io::Error::new(
+                std::io::ErrorKind::ConnectionReset,
+                "probe PUT response was lost before the object became visible",
+            ))
+        }
+
+        async fn put_with_meta(
+            &self,
+            object: &str,
+            r: ReaderImpl,
+            length: i64,
+            _meta: HashMap<String, String>,
+        ) -> Result<String, std::io::Error> {
+            self.put(object, r, length).await
+        }
+
+        async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
+            Err(std::io::Error::other("GET must not run after a lost probe PUT response"))
+        }
+
+        async fn remove(&self, _object: &str, _rv: &str) -> Result<(), std::io::Error> {
+            Err(std::io::Error::other("generic remove must not replace exact probe cleanup"))
+        }
+
+        async fn remove_exact(&self, _object: &str, rv: &str) -> Result<(), std::io::Error> {
+            self.removed.store(true, Ordering::SeqCst);
+            self.removed_versions.lock().await.push(rv.to_string());
+            Ok(())
+        }
+
+        async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+            self.probes.fetch_add(1, Ordering::SeqCst);
+            if tokio::time::Instant::now() >= self.visible_at && !self.removed.load(Ordering::SeqCst) {
+                Ok(TransitionCandidateProbe::VersionedPresent(PROBE_VERSION.to_string()))
+            } else {
+                Ok(TransitionCandidateProbe::Missing)
+            }
+        }
+
        async fn in_use(&self) -> Result<bool, std::io::Error> {
            Ok(false)
        }
@@ -1098,13 +1431,15 @@ mod tests {
        assert_eq!(probe, TransitionCandidateProbe::Unsupported);
    }

-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
    async fn check_warm_backend_removes_exact_probe_when_versioning_drifts() {
        let gets = Arc::new(AtomicUsize::new(0));
        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
        let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
            gets: gets.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
            removed_versions: removed_versions.clone(),
+            returned_version: uuid::Uuid::nil().to_string(),
        });

        let err = check_warm_backend(Some(&backend))
@@ -1113,7 +1448,27 @@ mod tests {

        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
        assert_eq!(gets.load(Ordering::SeqCst), 0);
-        assert_eq!(removed_versions.lock().await.as_slice(), [uuid::Uuid::nil().to_string()]);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_rejects_empty_put_version_for_a_versioned_candidate() {
+        let gets = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RejectingProbeVersionBackend {
+            gets: gets.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            returned_version: String::new(),
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("an empty PUT version must not read or generically delete a versioned object");
+
+        assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
+        assert_eq!(gets.load(Ordering::SeqCst), 0);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
    }

    #[tokio::test]
@@ -1122,8 +1477,11 @@ mod tests {
        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
            get_versions: get_versions.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
            fail_get: false,
+            body: ProbeBody::Exact,
        });

        check_warm_backend(Some(&backend))
@@ -1140,8 +1498,11 @@ mod tests {
        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
            get_versions: get_versions.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
            fail_get: true,
+            body: ProbeBody::Exact,
        });

        let err = check_warm_backend(Some(&backend))
@@ -1153,6 +1514,169 @@ mod tests {
        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
    }

+    #[tokio::test]
+    async fn check_warm_backend_removes_probe_after_body_mismatch() {
+        let get_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions,
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: true,
+            fail_get: false,
+            body: ProbeBody::Mismatch,
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("a mismatched body should fail after cleanup");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_rejects_a_stale_candidate_after_successful_delete() {
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let backend: WarmBackendImpl = Box::new(RecordingProbeBackend {
+            get_versions: Arc::new(tokio::sync::Mutex::new(Vec::new())),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            removed_versions: removed_versions.clone(),
+            remove_clears_candidate: false,
+            fail_get: false,
+            body: ProbeBody::Exact,
+        });
+
+        let err = check_warm_backend(Some(&backend))
+            .await
+            .expect_err("cleanup must not succeed while the deleted candidate remains visible");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(err.message.contains("cleanup is incomplete"));
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_reconciles_a_lost_put_response() {
+        let backend = MockWarmBackend::new();
+        backend.lose_next_put_response();
+        let driver: WarmBackendImpl = Box::new(backend.clone());
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("a lost probe PUT response must fail after compensation");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert_eq!(backend.object_count().await, 0);
+        assert_eq!(backend.exact_remove_count(), 1);
+        let operations = backend.op_log().await;
+        let put = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Put { object } => Some(object),
+            _ => None,
+        });
+        let probe = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Probe { object } => Some(object),
+            _ => None,
+        });
+        let remove = operations.iter().find_map(|operation| match operation {
+            MockWarmOp::Remove { object } => Some(object),
+            _ => None,
+        });
+        let (Some(put), Some(probe), Some(remove)) = (put, probe, remove) else {
+            panic!("lost-response compensation should PUT, probe, and remove");
+        };
+        assert_eq!(put, probe);
+        assert_eq!(probe, remove);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_retries_until_a_late_put_becomes_visible() {
+        let probes = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let driver: WarmBackendImpl = Box::new(LateVisibleProbeBackend {
+            visible_at: tokio::time::Instant::now() + Duration::from_secs(5),
+            removed: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+            probes: probes.clone(),
+            removed_versions: removed_versions.clone(),
+        });
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("a late-visible probe PUT must still report the lost response");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(
+            probes.load(Ordering::SeqCst) > 5,
+            "reconciliation must not stop at the first Missing result"
+        );
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_reports_incomplete_cleanup_without_guessing() {
+        for candidate in [TransitionCandidateProbe::Unsupported, TransitionCandidateProbe::Ambiguous] {
+            let backend = MockWarmBackend::new();
+            backend.set_transition_candidate_probe_override(Some(candidate)).await;
+            backend.lose_next_put_response();
+            let driver: WarmBackendImpl = Box::new(backend.clone());
+
+            let err = check_warm_backend(Some(&driver))
+                .await
+                .expect_err("an uncertain candidate must fail without a guessed delete");
+
+            assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+            assert!(err.message.contains("cleanup is incomplete"));
+            assert_eq!(backend.remove_count().await, 0);
+            assert_eq!(backend.object_count().await, 1);
+        }
+    }
+
+    #[tokio::test]
+    async fn check_warm_backend_reports_an_exact_cleanup_failure() {
+        let backend = MockWarmBackend::new();
+        backend.set_remove_failure(true);
+        backend.lose_next_put_response();
+        let driver: WarmBackendImpl = Box::new(backend.clone());
+
+        let err = check_warm_backend(Some(&driver))
+            .await
+            .expect_err("an exact cleanup failure must replace the ambiguous PUT error");
+
+        assert_eq!(err.code, ERR_TIER_PERM_ERR.code);
+        assert!(err.message.contains("cleanup is incomplete"));
+        assert_eq!(backend.exact_remove_count(), 1);
+        assert_eq!(backend.object_count().await, 1);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn check_warm_backend_reconciles_a_timed_out_put() {
+        let put_started = Arc::new(tokio::sync::Notify::new());
+        let probes = Arc::new(AtomicUsize::new(0));
+        let removed_versions = Arc::new(tokio::sync::Mutex::new(Vec::new()));
+        let driver: WarmBackendImpl = Box::new(HangingProbePutBackend {
+            put_started: put_started.clone(),
+            present: Arc::new(std::sync::atomic::AtomicBool::new(true)),
+            probes: probes.clone(),
+            removed_versions: removed_versions.clone(),
+        });
+        let check = check_warm_backend(Some(&driver));
+        tokio::pin!(check);
+        tokio::select! {
+            _ = put_started.notified() => {}
+            result = &mut check => panic!("probe completed before the PUT timeout: {result:?}"),
+        }
+
+        tokio::time::advance(WARM_BACKEND_PROBE_TIMEOUT + Duration::from_millis(1)).await;
+        let err = check.await.expect_err("a timed out probe PUT must fail after compensation");
+
+        assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
+        assert!(
+            probes.load(Ordering::SeqCst) > 1,
+            "timed-out PUT reconciliation must keep checking through the visibility window"
+        );
+        assert_eq!(removed_versions.lock().await.as_slice(), [PROBE_VERSION]);
+    }
+
    #[tokio::test]
    async fn new_wasabi_backend_honors_probe_flag() {
        let tier = TierConfig {
@@ -1298,6 +1822,15 @@ mod tests {
        assert_eq!(insecure.client.endpoint_url.port_or_known_default(), Some(80));
    }

+    #[test]
+    fn endpoint_authority_preserves_ipv6_brackets_and_explicit_port() {
+        let url = url::Url::parse("https://[2001:db8::1]:9443").expect("the IPv6 endpoint should parse");
+        assert_eq!(
+            endpoint_authority(&url).expect("the endpoint should have an authority"),
+            "[2001:db8::1]:9443"
+        );
+    }
+
    #[tokio::test]
    async fn s3_compatible_backend_strips_only_a_trailing_prefix_separator() {
        let mut params = s3_compatible_params("http://tier.example.com:9000");
@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
    tier_config::TierAliyun,
    warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
        new_s3_compatible_warm_backend, optimal_part_size,
    },
    warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendAliyun {
        self.0.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.0.in_use().await
    }
@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
    tier_config::TierAzure,
    warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
        new_s3_compatible_warm_backend, optimal_part_size,
    },
    warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,16 @@ impl WarmBackend for WarmBackendAzure {
        self.0.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        // Azure currently uses the shared S3/SigV4 transport, but its normal
+        // object path cannot persist exact remote versions across mixed
+        // RustFS releases. The mutation probe may still detect and precisely
+        // remove a versioned test object before rejecting that configuration.
+        self.0
+            .probe_transition_candidate_with_raw_version_header(object, "x-amz-version-id")
+            .await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.0.in_use().await
    }
@@ -98,6 +108,23 @@ impl WarmBackend for WarmBackendAzure {
 mod tests {
    use super::*;
    use crate::services::tier::tier_config::TierAzure;
+    use rustfs_s3_client::{
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{Options, TransitionClient, TransitionCore},
+    };
+    use std::sync::Arc;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+    async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String {
+        let mut request = Vec::new();
+        let mut buffer = [0_u8; 1024];
+        while !request.windows(4).any(|window| window == b"\r\n\r\n") {
+            let read = stream.read(&mut buffer).await.expect("fixture request should be readable");
+            assert_ne!(read, 0, "connection closed before request headers were received");
+            request.extend_from_slice(&buffer[..read]);
+        }
+        String::from_utf8_lossy(&request).into_owned()
+    }

    /// The SSRF guard itself is exercised once, generically, in
    /// `warm_backend::tests` (see backlog#2040/backlog#2041 and
@@ -119,4 +146,85 @@ mod tests {
            Err(err) => assert!(err.to_string().contains("not allowed")),
        }
    }
+
+    #[tokio::test]
+    async fn versioned_candidate_cleanup_uses_the_exact_s3_version_without_enabling_data_versions() {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let (mut get_stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
+            let get_request = read_request_head(&mut get_stream).await;
+            get_stream
+                .write_all(
+                    b"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: azure-version\r\nConnection: close\r\n\r\nx",
+                )
+                .await
+                .expect("fixture should write candidate GET response");
+
+            let (mut delete_stream, _) = listener.accept().await.expect("fixture should accept exact DELETE");
+            let delete_request = read_request_head(&mut delete_stream).await;
+            delete_stream
+                .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+                .await
+                .expect("fixture should write exact DELETE response");
+            (get_request, delete_request)
+        });
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "azure",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        let backend = WarmBackendAzure(WarmBackendS3 {
+            core: TransitionCore(Arc::clone(&client)),
+            client,
+            bucket: "bucket".to_string(),
+            prefix: String::new(),
+            storage_class: String::new(),
+        });
+        assert!(
+            !backend.0.client.provider_version_capabilities().exact_get_delete,
+            "probe-only version discovery must not change Azure's persisted data-path contract"
+        );
+
+        let candidate = backend
+            .probe_transition_candidate("probe")
+            .await
+            .expect("Azure candidate should be discovered");
+        assert_eq!(candidate, TransitionCandidateProbe::VersionedPresent("azure-version".to_string()));
+        backend
+            .remove_exact("probe", "azure-version")
+            .await
+            .expect("Azure candidate should be deleted by exact version");
+
+        let (get_request, delete_request) = fixture.await.expect("fixture should join");
+        assert!(get_request.to_ascii_lowercase().contains("\r\nrange: bytes=0-0\r\n"));
+        assert!(
+            delete_request
+                .lines()
+                .next()
+                .is_some_and(|line| line.contains("DELETE /bucket/probe?versionId=azure-version "))
+        );
+    }
 }
@@ -18,13 +18,13 @@
 #![allow(unused_must_use)]
 #![allow(clippy::all)]

-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use std::future::Future;
 use std::io::{Error, ErrorKind};
 use std::sync::Arc;

 use bytes::Bytes;
-use google_cloud_auth::credentials::Credentials;
-use google_cloud_auth::credentials::user_account::Builder;
+use google_cloud_auth::credentials::service_account::Builder;
 use google_cloud_storage as gcs;
 use google_cloud_storage::client::Storage;
 use google_cloud_storage::client::StorageControl;
@@ -32,7 +32,7 @@ use std::convert::TryFrom;

 use crate::services::tier::{
    tier_config::TierGCS,
-    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
 };
 use rustfs_s3_client::{
    admin_handler_utils::AdminError,
@@ -43,6 +43,7 @@ use rustfs_utils::egress::validate_outbound_url;
 use tracing::warn;

 const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
+const MAX_GCS_CANDIDATE_PAGES: usize = 64;

 fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
    if remote_version.is_empty() {
@@ -57,6 +58,85 @@ fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
    Ok(Some(generation))
 }

+fn append_gcs_chunk<E: std::fmt::Display>(
+    contents: &mut Vec<u8>,
+    chunk: Result<Bytes, E>,
+    max_response_bytes: Option<usize>,
+) -> std::io::Result<()> {
+    let chunk = chunk.map_err(|err| std::io::Error::other(err.to_string()))?;
+    if max_response_bytes.is_some_and(|limit| contents.len().saturating_add(chunk.len()) > limit) {
+        return Err(std::io::Error::new(
+            ErrorKind::InvalidData,
+            "GCS object response exceeded the configured byte limit",
+        ));
+    }
+    contents.extend_from_slice(&chunk);
+    Ok(())
+}
+
+fn gcs_bucket_resource_name(bucket: &str) -> String {
+    format!("projects/_/buckets/{bucket}")
+}
+
+struct GcsCandidateObject {
+    name: String,
+    generation: i64,
+}
+
+struct GcsCandidatePage {
+    objects: Vec<GcsCandidateObject>,
+    next_page_token: String,
+}
+
+async fn probe_exact_gcs_candidate<F, Fut>(
+    remote_object: &str,
+    mut fetch_page: F,
+) -> Result<TransitionCandidateProbe, std::io::Error>
+where
+    F: FnMut(String) -> Fut,
+    Fut: Future<Output = Result<GcsCandidatePage, std::io::Error>>,
+{
+    let mut page_token = String::new();
+    let mut seen_page_tokens = HashSet::new();
+    let mut generation = None;
+    let mut pages_seen = 0_usize;
+
+    loop {
+        pages_seen += 1;
+        if pages_seen > MAX_GCS_CANDIDATE_PAGES {
+            return Err(std::io::Error::new(
+                ErrorKind::InvalidData,
+                "GCS candidate listing exceeded the page limit",
+            ));
+        }
+        let response = fetch_page(page_token.clone()).await?;
+        for candidate in response.objects.iter().filter(|candidate| candidate.name == remote_object) {
+            if candidate.generation <= 0 {
+                return Err(std::io::Error::new(
+                    ErrorKind::InvalidData,
+                    "GCS candidate listing returned a non-positive generation",
+                ));
+            }
+            if generation.replace(candidate.generation).is_some() {
+                return Ok(TransitionCandidateProbe::Ambiguous);
+            }
+        }
+
+        if response.next_page_token.is_empty() {
+            break;
+        }
+        if !seen_page_tokens.insert(response.next_page_token.clone()) {
+            return Err(std::io::Error::new(ErrorKind::InvalidData, "GCS candidate listing repeated a page token"));
+        }
+        page_token = response.next_page_token;
+    }
+
+    Ok(match generation {
+        Some(generation) => TransitionCandidateProbe::VersionedPresent(generation.to_string()),
+        None => TransitionCandidateProbe::Missing,
+    })
+}
+
 pub struct WarmBackendGCS {
    pub client: Arc<Storage>,
    pub control: Arc<StorageControl>,
@@ -80,8 +160,8 @@ impl WarmBackendGCS {
                .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
        }

-        let authorized_user = serde_json::from_str(&conf.creds)?;
-        let credentials = Builder::new(authorized_user)
+        let service_account = serde_json::from_str(&conf.creds)?;
+        let credentials = Builder::new(service_account)
            //.with_retry_policy(AlwaysRetry.with_attempt_limit(3))
            //.with_backoff_policy(backoff)
            .build()
@@ -98,7 +178,11 @@ impl WarmBackendGCS {
        let client = Arc::new(client);
        // Control-plane client: the data-plane `Storage` client cannot delete or list objects;
        // delete_object/list_objects live on StorageControl.
-        let Ok(control) = StorageControl::builder().with_credentials(credentials).build().await else {
+        let mut control_builder = StorageControl::builder().with_credentials(credentials);
+        if !conf.endpoint.is_empty() {
+            control_builder = control_builder.with_endpoint(conf.endpoint.clone());
+        }
+        let Ok(control) = control_builder.build().await else {
            return Err(std::io::Error::other("StorageControl::builder error"));
        };
        let control = Arc::new(control);
@@ -136,9 +220,10 @@ impl WarmBackend for WarmBackendGCS {
            ReaderImpl::Body(content_body) => content_body.to_vec(),
            ReaderImpl::ObjectBody(mut content_body) => content_body.read_all().await?,
        };
+        let bucket = gcs_bucket_resource_name(&self.bucket);
        let Ok(res) = Box::pin(
            self.client
-                .write_object(&self.bucket, &self.get_dest(object), Bytes::from(d))
+                .write_object(&bucket, &self.get_dest(object), Bytes::from(d))
                .send_buffered(),
        )
        .await
@@ -154,7 +239,9 @@ impl WarmBackend for WarmBackendGCS {
    }

    async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
-        let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
+        let bucket = gcs_bucket_resource_name(&self.bucket);
+        let mut req = self.client.read_object(&bucket, &self.get_dest(object));
+        let mut max_response_bytes = None;
        if let Some(generation) = parse_generation(rv)? {
            req = req.set_generation(generation);
        }
@@ -170,6 +257,11 @@ impl WarmBackend for WarmBackendGCS {
                .length
                .try_into()
                .map_err(|_| std::io::Error::other("invalid range: negative length"))?;
+            max_response_bytes = Some(
+                opts.length
+                    .try_into()
+                    .map_err(|_| std::io::Error::other("invalid range: length does not fit in memory"))?,
+            );
            req = req.set_read_range(google_cloud_storage::model_ext::ReadRange::segment(offset, count));
        }

@@ -177,8 +269,8 @@ impl WarmBackend for WarmBackendGCS {
            return Err(std::io::Error::other("read_object error"));
        };
        let mut contents = Vec::new();
-        while let Ok(Some(chunk)) = reader.next().await.transpose() {
-            contents.extend_from_slice(&chunk);
+        while let Some(chunk) = reader.next().await {
+            append_gcs_chunk(&mut contents, chunk, max_response_bytes)?;
        }
        Ok(ReadCloser::new(std::io::Cursor::new(contents)))
    }
@@ -190,7 +282,7 @@ impl WarmBackend for WarmBackendGCS {
        let mut req = self
            .control
            .delete_object()
-            .set_bucket(format!("projects/_/buckets/{}", self.bucket))
+            .set_bucket(gcs_bucket_resource_name(&self.bucket))
            .set_object(self.get_dest(object));
        if let Some(generation) = parse_generation(rv)? {
            req = req.set_generation(generation);
@@ -199,13 +291,47 @@ impl WarmBackend for WarmBackendGCS {
        Ok(())
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        let remote_object = self.get_dest(object);
+        let parent = gcs_bucket_resource_name(&self.bucket);
+        probe_exact_gcs_candidate(&remote_object, |page_token| {
+            let control = self.control.clone();
+            let parent = parent.clone();
+            let prefix = remote_object.clone();
+            async move {
+                let response = control
+                    .list_objects()
+                    .set_parent(parent)
+                    .set_prefix(prefix)
+                    .set_versions(true)
+                    .set_page_size(2)
+                    .set_page_token(page_token)
+                    .send()
+                    .await
+                    .map_err(|err| std::io::Error::other(err.to_string()))?;
+                Ok(GcsCandidatePage {
+                    objects: response
+                        .objects
+                        .into_iter()
+                        .map(|candidate| GcsCandidateObject {
+                            name: candidate.name,
+                            generation: candidate.generation,
+                        })
+                        .collect(),
+                    next_page_token: response.next_page_token,
+                })
+            }
+        })
+        .await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        // Scope the listing to this tier's prefix (matching the other warm backends) and only
        // need to know whether a single object exists.
        let resp = self
            .control
            .list_objects()
-            .set_parent(format!("projects/_/buckets/{}", self.bucket))
+            .set_parent(gcs_bucket_resource_name(&self.bucket))
            .set_prefix(self.prefix.clone())
            .set_page_size(1)
            .send()
@@ -218,10 +344,126 @@ impl WarmBackend for WarmBackendGCS {

 #[cfg(test)]
 mod tests {
+    use super::GcsCandidateObject;
+    use super::GcsCandidatePage;
+    use super::MAX_GCS_CANDIDATE_PAGES;
    use super::WarmBackendGCS;
+    use super::append_gcs_chunk;
+    use super::gcs_bucket_resource_name;
    use super::parse_generation;
+    use super::probe_exact_gcs_candidate;
    use crate::services::tier::tier_config::TierGCS;
+    use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts};
+    use google_cloud_auth::credentials::anonymous::Builder as Anonymous;
+    use google_cloud_storage::client::{Storage, StorageControl};
    use std::io::ErrorKind;
+    use std::sync::Arc;
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+    use tokio::net::TcpListener;
+
+    async fn serve_data_plane_fixture(listener: TcpListener) -> Vec<String> {
+        let upload_body = r#"{"name":"probe","bucket":"tier-bucket","generation":"123"}"#;
+        let responses = [
+            format!(
+                "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{upload_body}",
+                upload_body.len()
+            ),
+            "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-6/7\r\nx-goog-generation: 123\r\ncontent-length: 7\r\nconnection: close\r\n\r\nRustFS!"
+                .to_string(),
+            "HTTP/1.1 206 Partial Content\r\ncontent-type: application/octet-stream\r\ncontent-range: bytes 0-7/8\r\nx-goog-generation: 123\r\ncontent-length: 8\r\nconnection: close\r\n\r\nRustFS!!"
+                .to_string(),
+        ];
+        let mut requests = Vec::new();
+
+        for response in responses {
+            let (mut stream, _) = listener.accept().await.expect("the GCS fixture should accept a request");
+            let mut request = Vec::new();
+            loop {
+                let mut chunk = [0_u8; 1024];
+                let count = stream
+                    .read(&mut chunk)
+                    .await
+                    .expect("the GCS fixture should read request headers");
+                if count == 0 {
+                    break;
+                }
+                request.extend_from_slice(&chunk[..count]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            let header_end = request
+                .windows(4)
+                .position(|window| window == b"\r\n\r\n")
+                .map(|position| position + 4)
+                .expect("the GCS fixture should receive complete request headers");
+            let headers = String::from_utf8_lossy(&request[..header_end]);
+            if headers.lines().any(|line| line.eq_ignore_ascii_case("expect: 100-continue")) {
+                stream
+                    .write_all(b"HTTP/1.1 100 Continue\r\n\r\n")
+                    .await
+                    .expect("the GCS fixture should acknowledge 100-continue");
+            }
+            let content_length = headers
+                .lines()
+                .find_map(|line| {
+                    let (name, value) = line.split_once(':')?;
+                    name.eq_ignore_ascii_case("content-length")
+                        .then(|| value.trim().parse::<usize>().expect("content-length should be numeric"))
+                })
+                .unwrap_or_default();
+            while request.len() < header_end.saturating_add(content_length) {
+                let mut chunk = [0_u8; 1024];
+                let count = stream
+                    .read(&mut chunk)
+                    .await
+                    .expect("the GCS fixture should read the request body");
+                if count == 0 {
+                    break;
+                }
+                request.extend_from_slice(&chunk[..count]);
+            }
+            requests.push(String::from_utf8_lossy(&request).into_owned());
+            stream
+                .write_all(response.as_bytes())
+                .await
+                .expect("the GCS fixture should write its response");
+        }
+
+        requests
+    }
+
+    fn candidate_page(objects: &[(&str, i64)], next_page_token: &str) -> GcsCandidatePage {
+        GcsCandidatePage {
+            objects: objects
+                .iter()
+                .map(|(name, generation)| GcsCandidateObject {
+                    name: (*name).to_string(),
+                    generation: *generation,
+                })
+                .collect(),
+            next_page_token: next_page_token.to_string(),
+        }
+    }
+
+    async fn probe_candidate_pages(
+        remote_object: &str,
+        pages: Vec<GcsCandidatePage>,
+    ) -> (Result<TransitionCandidateProbe, std::io::Error>, Vec<String>) {
+        let mut pages = pages.into_iter();
+        let mut requested_tokens = Vec::new();
+        let result = probe_exact_gcs_candidate(remote_object, |page_token| {
+            requested_tokens.push(page_token);
+            std::future::ready(
+                pages
+                    .next()
+                    .ok_or_else(|| std::io::Error::new(ErrorKind::UnexpectedEof, "test fixture ran out of GCS pages")),
+            )
+        })
+        .await;
+        (result, requested_tokens)
+    }

    #[test]
    fn generation_parser_preserves_exact_numeric_versions() {
@@ -241,6 +483,243 @@ mod tests {
        }
    }

+    #[test]
+    fn body_collection_propagates_an_error_after_a_complete_prefix() {
+        let mut contents = Vec::new();
+        append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFS")), Some(7))
+            .expect("the prefix chunk should be collected");
+        let err = append_gcs_chunk(&mut contents, Err(std::io::Error::other("trailing stream failure")), Some(7))
+            .expect_err("a trailing stream error must not be mistaken for EOF");
+
+        assert_eq!(contents, b"RustFS");
+        assert!(err.to_string().contains("trailing stream failure"));
+    }
+
+    #[test]
+    fn body_collection_rejects_a_chunk_that_exceeds_the_probe_limit() {
+        let mut contents = Vec::new();
+        let err = append_gcs_chunk::<std::io::Error>(&mut contents, Ok(bytes::Bytes::from_static(b"RustFSxx")), Some(7))
+            .expect_err("the GCS collection layer must reject an oversized probe response");
+
+        assert!(contents.is_empty());
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_finds_exact_name_on_first_or_later_page() {
+        let (first, first_tokens) =
+            probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", 7)], "")]).await;
+        assert_eq!(
+            first.expect("an exact first-page object should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("7".to_string())
+        );
+        assert_eq!(first_tokens, [""]);
+
+        let (later, later_tokens) = probe_candidate_pages(
+            "prefix/object",
+            vec![
+                candidate_page(&[("prefix/object-shadow", 8)], "next"),
+                candidate_page(&[("prefix/object", 9)], ""),
+            ],
+        )
+        .await;
+        assert_eq!(
+            later.expect("an exact later-page object should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("9".to_string())
+        );
+        assert_eq!(later_tokens, ["", "next"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_ignores_non_exact_prefix_matches() {
+        let (probe, _) = probe_candidate_pages(
+            "prefix/object",
+            vec![candidate_page(
+                &[("prefix/object-shadow", 8), ("prefix/object/child", 9), ("prefix/object", 7)],
+                "",
+            )],
+        )
+        .await;
+
+        assert_eq!(
+            probe.expect("prefix-only matches should not hide the exact object"),
+            TransitionCandidateProbe::VersionedPresent("7".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_reports_duplicate_exact_names_as_ambiguous() {
+        let (probe, _) = probe_candidate_pages(
+            "prefix/object",
+            vec![
+                candidate_page(&[("prefix/object", 7)], "next"),
+                candidate_page(&[("prefix/object", 8)], ""),
+            ],
+        )
+        .await;
+
+        assert_eq!(
+            probe.expect("multiple exact generations should produce a conservative result"),
+            TransitionCandidateProbe::Ambiguous
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_reports_missing_without_an_exact_name() {
+        let (probe, _) = probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object-shadow", 8)], "")]).await;
+
+        assert_eq!(
+            probe.expect("a complete listing without an exact name should be definitive"),
+            TransitionCandidateProbe::Missing
+        );
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_non_positive_generations() {
+        for generation in [0, -1] {
+            let (probe, _) =
+                probe_candidate_pages("prefix/object", vec![candidate_page(&[("prefix/object", generation)], "")]).await;
+            let err = probe.expect_err("a non-positive GCS generation must fail closed");
+            assert_eq!(err.kind(), ErrorKind::InvalidData, "generation {generation}");
+        }
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_a_page_token_that_does_not_advance() {
+        let (probe, requested_tokens) =
+            probe_candidate_pages("prefix/object", vec![candidate_page(&[], "next"), candidate_page(&[], "next")]).await;
+
+        let err = probe.expect_err("a repeated GCS page token must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens, ["", "next"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_a_non_adjacent_page_token_cycle() {
+        let (probe, requested_tokens) = probe_candidate_pages(
+            "prefix/object",
+            vec![candidate_page(&[], "a"), candidate_page(&[], "b"), candidate_page(&[], "a")],
+        )
+        .await;
+
+        let err = probe.expect_err("a non-adjacent GCS page token cycle must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens, ["", "a", "b"]);
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_rejects_an_unbounded_unique_token_chain() {
+        let pages = (0..MAX_GCS_CANDIDATE_PAGES)
+            .map(|index| candidate_page(&[], &format!("token-{index}")))
+            .collect();
+        let (probe, requested_tokens) = probe_candidate_pages("prefix/object", pages).await;
+
+        let err = probe.expect_err("an unbounded unique page-token chain must fail closed");
+        assert_eq!(err.kind(), ErrorKind::InvalidData);
+        assert_eq!(requested_tokens.len(), MAX_GCS_CANDIDATE_PAGES);
+    }
+
+    #[tokio::test]
+    async fn plain_bucket_reaches_gcs_put_and_get_transport_with_resource_name() {
+        let listener = TcpListener::bind("127.0.0.1:0")
+            .await
+            .expect("the GCS fixture should bind a loopback port");
+        let endpoint = format!("http://{}", listener.local_addr().expect("the GCS fixture should have a local address"));
+        let fixture = tokio::spawn(serve_data_plane_fixture(listener));
+        let credentials = Anonymous::new().build();
+        let client = Storage::builder()
+            .with_endpoint(endpoint.clone())
+            .with_credentials(credentials.clone())
+            .build()
+            .await
+            .expect("the GCS data client should build");
+        let control = StorageControl::builder()
+            .with_endpoint(endpoint)
+            .with_credentials(credentials)
+            .build()
+            .await
+            .expect("the GCS control client should build");
+        let backend = WarmBackendGCS {
+            client: Arc::new(client),
+            control: Arc::new(control),
+            bucket: "tier-bucket".to_string(),
+            prefix: String::new(),
+        };
+
+        let (version, body, oversized_error_kind, requests) = tokio::time::timeout(Duration::from_secs(5), async {
+            let version = backend
+                .put(
+                    "probe",
+                    rustfs_s3_client::transition_api::ReaderImpl::Body(bytes::Bytes::from_static(b"RustFS")),
+                    6,
+                )
+                .await
+                .expect("a plain configured bucket should reach the GCS upload transport");
+            let mut reader = backend
+                .get(
+                    "probe",
+                    &version,
+                    WarmBackendGetOpts {
+                        start_offset: 0,
+                        length: 7,
+                    },
+                )
+                .await
+                .expect("a plain configured bucket should reach the GCS read transport");
+            let mut body = Vec::new();
+            reader
+                .read_to_end(&mut body)
+                .await
+                .expect("the fixture body should be readable");
+            let oversized_error = match backend
+                .get(
+                    "probe",
+                    &version,
+                    WarmBackendGetOpts {
+                        start_offset: 0,
+                        length: 7,
+                    },
+                )
+                .await
+            {
+                Ok(_) => panic!("an eight-byte response must not pass a seven-byte collection limit"),
+                Err(err) => err,
+            };
+            let requests = fixture.await.expect("the GCS fixture task should finish");
+            (version, body, oversized_error.kind(), requests)
+        })
+        .await
+        .expect("the GCS data-plane requests should not be rejected before transport");
+
+        assert_eq!(gcs_bucket_resource_name("tier-bucket"), "projects/_/buckets/tier-bucket");
+        assert_eq!(version, "123");
+        assert_eq!(body, b"RustFS!");
+        assert_eq!(oversized_error_kind, ErrorKind::InvalidData);
+        assert!(
+            requests[0].starts_with("POST /upload/storage/v1/b/tier-bucket/o?"),
+            "unexpected upload request line: {}",
+            requests[0].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[1].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
+            "unexpected read request line: {}",
+            requests[1].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[1].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
+            "the GCS probe read must preserve its seven-byte range"
+        );
+        assert!(
+            requests[2].starts_with("GET /storage/v1/b/tier-bucket/o/probe?"),
+            "unexpected oversized read request line: {}",
+            requests[2].lines().next().unwrap_or_default()
+        );
+        assert!(
+            requests[2].to_ascii_lowercase().contains("\r\nrange: bytes=0-6\r\n"),
+            "the oversized response must be fetched under the same seven-byte request boundary"
+        );
+    }
+
    #[tokio::test]
    async fn new_rejects_loopback_endpoint_before_credential_setup() {
        let conf = TierGCS {
@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
    tier_config::TierHuaweicloud,
    warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
        new_s3_compatible_warm_backend, optimal_part_size,
    },
    warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendHuaweicloud {
        self.0.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.0.in_use().await
    }
@@ -26,11 +26,12 @@ use crate::services::tier::{
    tier_config::TierS3,
    warm_backend::{
        TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
-        build_transition_put_options,
+        build_transition_put_options, endpoint_authority,
    },
 };
 use http::HeaderMap;
 use rustfs_s3_client::{
+    api_error_response::to_error_response,
    api_get_options::GetObjectOptions,
    api_list::ListObjectsOptions,
    api_put_object::PutObjectOptions,
@@ -43,7 +44,7 @@ use rustfs_s3_client::{
 };
 use rustfs_utils::egress::validate_outbound_url;
 use rustfs_utils::path::SLASH_SEPARATOR;
-use s3s::dto::BucketVersioningStatus;
+use s3s::{S3ErrorCode, dto::BucketVersioningStatus};

 pub struct WarmBackendS3 {
    pub client: Arc<TransitionClient>,
@@ -74,6 +75,19 @@ fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBu
    })
 }

+fn bounded_get_range(opts: &WarmBackendGetOpts) -> Result<Option<(i64, i64)>, std::io::Error> {
+    if opts.start_offset < 0 || opts.length <= 0 {
+        return Ok(None);
+    }
+    usize::try_from(opts.length)
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: length does not fit in memory"))?;
+    let end_offset = opts
+        .start_offset
+        .checked_add(opts.length - 1)
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid range: end offset overflow"))?;
+    Ok(Some((opts.start_offset, end_offset)))
+}
+
 impl WarmBackendS3 {
    pub async fn new(conf: &TierS3, _tier: &str) -> Result<Self, std::io::Error> {
        Self::new_with_bucket_lookup(conf, BucketLookupType::BucketLookupAuto, "s3").await
@@ -132,10 +146,8 @@ impl WarmBackendS3 {
            bucket_lookup,
            ..Default::default()
        };
-        let host = u
-            .host()
-            .ok_or_else(|| std::io::Error::other("Invalid endpoint URL: missing host"))?;
-        let client = TransitionClient::new(&host.to_string(), opts, tier_type).await?;
+        let endpoint = endpoint_authority(&u)?;
+        let client = TransitionClient::new(&endpoint, opts, tier_type).await?;

        let client = Arc::new(client);
        let core = TransitionCore(Arc::clone(&client));
@@ -177,10 +189,8 @@ impl WarmBackendS3 {
        if !rv.is_empty() {
            gopts.version_id = rv.to_string();
        }
-        if opts.start_offset >= 0 && opts.length > 0 {
-            gopts
-                .set_range(opts.start_offset, opts.start_offset + opts.length - 1)
-                .map_err(std::io::Error::other)?;
+        if let Some((start_offset, end_offset)) = bounded_get_range(&opts)? {
+            gopts.set_range(start_offset, end_offset)?;
        }
        let (_, headers, reader) = self.core.get_object(&self.bucket, &self.get_dest(object), &gopts).await?;
        Ok((headers, reader))
@@ -191,36 +201,64 @@ impl WarmBackendS3 {
        remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
    }

-    async fn probe_transition_candidate_versions(
+    async fn probe_current_transition_candidate_with_header(
        &self,
        object: &str,
-        bucket_versioning: RemoteBucketVersioning,
+        raw_version_header: Option<&'static str>,
    ) -> Result<TransitionCandidateProbe, std::io::Error> {
-        let remote_object = self.get_dest(object);
-        let mut opts = ListObjectsOptions::default();
-        opts.set("prefix", &remote_object);
-        opts.set("max-keys", "1000");
-
-        let mut key_marker = String::new();
-        let mut version_id_marker = String::new();
-        let mut candidates = TransitionCandidateVersions::default();
-        loop {
-            let versions = self
-                .client
-                .list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
-                .await?;
-            candidates.extend(&remote_object, &versions);
-            if candidates.is_ambiguous() {
-                return Ok(TransitionCandidateProbe::Ambiguous);
+        match self
+            .get_with_headers(
+                object,
+                "",
+                WarmBackendGetOpts {
+                    start_offset: 0,
+                    length: 1,
+                },
+            )
+            .await
+        {
+            Ok((headers, _)) => {
+                let version_id = match raw_version_header {
+                    Some(header_name) => match headers.get(header_name) {
+                        Some(value) => {
+                            let version_id = value.to_str().map_err(|_| {
+                                std::io::Error::new(
+                                    std::io::ErrorKind::InvalidData,
+                                    "remote object version id is not valid ASCII",
+                                )
+                            })?;
+                            validate_remote_version_id(version_id)?;
+                            Some(version_id)
+                        }
+                        None => None,
+                    },
+                    None => self.client.raw_version_id(&headers)?,
+                };
+                Ok(match version_id {
+                    Some(version_id) => TransitionCandidateProbe::VersionedPresent(version_id.to_string()),
+                    None => TransitionCandidateProbe::UnversionedPresent,
+                })
            }
-            if !versions.is_truncated {
-                return classify_transition_candidates(candidates, bucket_versioning);
+            Err(err) => {
+                let response = to_error_response(&err);
+                if response.code == S3ErrorCode::NoSuchKey {
+                    Ok(TransitionCandidateProbe::Missing)
+                } else {
+                    Err(err)
+                }
            }
-
-            advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
        }
    }

+    pub(crate) async fn probe_transition_candidate_with_raw_version_header(
+        &self,
+        object: &str,
+        raw_version_header: &'static str,
+    ) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.probe_current_transition_candidate_with_header(object, Some(raw_version_header))
+            .await
+    }
+
    async fn probe_transition_candidate_identity(
        &self,
        object: &str,
@@ -343,6 +381,7 @@ struct TransitionCandidateVersions {
 }

 impl TransitionCandidateVersions {
+    #[cfg(test)]
    fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
        for version in versions.versions.iter().filter(|version| version.key == remote_object) {
            if self.version_id.is_some() {
@@ -353,10 +392,6 @@ impl TransitionCandidateVersions {
        }
    }

-    fn is_ambiguous(&self) -> bool {
-        self.ambiguous
-    }
-
    fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
        if self.ambiguous {
            return TransitionCandidateProbe::Ambiguous;
@@ -380,6 +415,8 @@ impl TransitionCandidateVersions {
 mod tests {
    use super::*;
    use rustfs_s3_client::api_s3_datatypes::{ListVersionsResult, Version};
+    use std::time::Duration;
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};

    #[tokio::test]
    async fn new_rejects_loopback_endpoint_before_network_setup() {
@@ -398,6 +435,204 @@ mod tests {
        }
    }

+    #[tokio::test]
+    async fn new_preserves_an_explicit_endpoint_port() {
+        let conf = TierS3 {
+            endpoint: "https://tier.example.com:9443".to_string(),
+            bucket: "tier-bucket".to_string(),
+            access_key: "access".to_string(),
+            secret_key: "secret".to_string(),
+            region: "us-east-1".to_string(),
+            ..Default::default()
+        };
+
+        let backend = WarmBackendS3::new(&conf, "tier")
+            .await
+            .expect("a well-formed S3 endpoint should initialize without network I/O");
+        assert_eq!(backend.client.endpoint_url.host_str(), Some("tier.example.com"));
+        assert_eq!(backend.client.endpoint_url.port(), Some(9443));
+    }
+
+    #[tokio::test]
+    async fn overflowing_get_range_is_rejected_before_network_io() {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "s3",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        let backend = WarmBackendS3 {
+            core: TransitionCore(Arc::clone(&client)),
+            client,
+            bucket: "bucket".to_string(),
+            prefix: String::new(),
+            storage_class: String::new(),
+        };
+
+        let err = backend
+            .get_with_headers(
+                "probe",
+                "",
+                WarmBackendGetOpts {
+                    start_offset: i64::MAX,
+                    length: 2,
+                },
+            )
+            .await
+            .expect_err("an overflowing range must fail before issuing a GET");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), listener.accept())
+                .await
+                .is_err()
+        );
+    }
+
+    async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
+        let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let fixture = tokio::spawn(async move {
+            let responses = [
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
+                "HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
+            ];
+            let mut requests = Vec::new();
+            for response in responses {
+                let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
+                let mut request = Vec::new();
+                let mut buffer = [0; 1024];
+                loop {
+                    let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                    assert_ne!(read, 0, "connection closed before request headers were received");
+                    request.extend_from_slice(&buffer[..read]);
+                    if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                        break;
+                    }
+                }
+                requests.push(String::from_utf8_lossy(&request).into_owned());
+                stream
+                    .write_all(response.as_bytes())
+                    .await
+                    .expect("fixture should write candidate response");
+            }
+            requests
+        });
+        let client = Arc::new(
+            TransitionClient::new(
+                &endpoint,
+                Options {
+                    creds: Credentials::new(Static(Value {
+                        access_key_id: "access-key".to_string(),
+                        secret_access_key: "secret-key".to_string(),
+                        signer_type: SignatureType::SignatureV4,
+                        ..Default::default()
+                    })),
+                    region: "us-east-1".to_string(),
+                    bucket_lookup: BucketLookupType::BucketLookupPath,
+                    max_retries: 1,
+                    ..Default::default()
+                },
+                "s3",
+            )
+            .await
+            .expect("fixture client should build"),
+        );
+        Some((
+            WarmBackendS3 {
+                core: TransitionCore(Arc::clone(&client)),
+                client,
+                bucket: "bucket".to_string(),
+                prefix: String::new(),
+                storage_class: String::new(),
+            },
+            fixture,
+        ))
+    }
+
+    #[tokio::test]
+    async fn candidate_probe_uses_only_exact_bounded_get_permissions() {
+        let Some((backend, fixture)) = candidate_probe_fixture().await else {
+            return;
+        };
+
+        assert_eq!(
+            backend
+                .probe_transition_candidate("versioned-probe")
+                .await
+                .expect("versioned candidate should be discovered"),
+            TransitionCandidateProbe::VersionedPresent("opaque-version".to_string())
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("unversioned-probe")
+                .await
+                .expect("unversioned candidate should be discovered"),
+            TransitionCandidateProbe::UnversionedPresent
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("missing-probe")
+                .await
+                .expect("a missing key should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+        assert_eq!(
+            backend
+                .probe_transition_candidate("provider-missing-probe")
+                .await
+                .expect("a provider-specific missing code should be classified"),
+            TransitionCandidateProbe::Missing
+        );
+        let err = backend
+            .probe_transition_candidate("forbidden-probe")
+            .await
+            .expect_err("an authorization failure must not be mistaken for a missing key");
+        assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
+
+        let requests = fixture.await.expect("candidate fixture should join");
+        for request in requests {
+            let request = request.to_ascii_lowercase();
+            assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
+            assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
+            assert!(!request.contains("?versioning"));
+            assert!(!request.contains("?versions"));
+        }
+    }
+
    fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
        ListVersionsResult {
            versions: versions
@@ -631,8 +866,7 @@ impl WarmBackend for WarmBackendS3 {
    }

    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
-        let bucket_versioning = self.remote_bucket_versioning().await?;
-        self.probe_transition_candidate_versions(object, bucket_versioning).await
+        self.probe_current_transition_candidate_with_header(object, None).await
    }

    async fn in_use(&self) -> Result<bool, std::io::Error> {
@@ -23,7 +23,7 @@ use std::collections::HashMap;
 use crate::services::tier::{
    tier_config::TierTencent,
    warm_backend::{
-        S3CompatibleWarmBackendParams, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
+        S3CompatibleWarmBackendParams, TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
        new_s3_compatible_warm_backend, optimal_part_size,
    },
    warm_backend_s3::WarmBackendS3,
@@ -89,6 +89,10 @@ impl WarmBackend for WarmBackendTencent {
        self.0.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
+        self.0.probe_transition_candidate(object).await
+    }
+
    async fn in_use(&self) -> Result<bool, std::io::Error> {
        self.0.in_use().await
    }
@@ -23,7 +23,7 @@ use uuid::Uuid;

 use crate::services::tier::{
    tier_config::{TierS3, TierWasabi},
-    warm_backend::{WarmBackend, WarmBackendGetOpts},
+    warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts},
    warm_backend_s3::WarmBackendS3,
 };
 use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl};
@@ -169,6 +169,10 @@ impl WarmBackend for WarmBackendWasabi {
        self.s3.remove(object, rv).await
    }

+    async fn probe_transition_candidate(&self, object: &str) -> io::Result<TransitionCandidateProbe> {
+        self.s3.probe_transition_candidate(object).await
+    }
+
    async fn in_use(&self) -> io::Result<bool> {
        self.check_remote_bucket_unversioned().await?;
        let in_use = self.s3.in_use().await?;
@@ -10339,9 +10339,17 @@ mod tests {
    }

    #[cfg(feature = "test-util")]
-    #[tokio::test]
+    #[test]
    #[serial_test::serial(storage_class_env)]
-    async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
+    fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence() {
+        run_large_stack_async_test(
+            "v6-checkpoint-fence-loss",
+            v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case,
+        );
+    }
+
+    #[cfg(feature = "test-util")]
+    async fn v6_decommission_checkpoint_no_lock_put_rejects_lost_publication_fence_case() {
        let temp_dir = tempfile::tempdir().expect("create v6 checkpoint fence-loss store dir");
        let (_ctx, store, _shutdown) =
            without_storage_class_env(build_isolated_test_store(temp_dir.path(), "v6-checkpoint-fence-loss", &[4, 4])).await;
@@ -57,7 +57,11 @@ fn deserialize_code<'de, D>(d: D) -> Result<S3ErrorCode, D::Error>
 where
    D: Deserializer<'de>,
 {
-    Ok(S3ErrorCode::from_bytes(String::deserialize(d)?.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
+    let code = String::deserialize(d)?;
+    if code == "NoSuchObject" {
+        return Ok(S3ErrorCode::NoSuchKey);
+    }
+    Ok(S3ErrorCode::from_bytes(code.as_bytes()).unwrap_or(S3ErrorCode::Custom("".into())))
 }

 impl Default for ErrorResponse {
@@ -325,4 +329,21 @@ mod tests {
        assert_eq!(response.code, S3ErrorCode::NoSuchVersion);
        assert_eq!(response.status_code, StatusCode::NOT_FOUND);
    }
+
+    #[test]
+    fn normalizes_provider_specific_missing_object_code() {
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-request-id", "request-id".parse().expect("request ID header should parse"));
+
+        let response = http_resp_to_error_response(
+            StatusCode::NOT_FOUND,
+            &headers,
+            b"<Error><Code>NoSuchObject</Code><Message>remote detail</Message></Error>".to_vec(),
+            "bucket",
+            "object",
+        );
+
+        assert_eq!(response.code, S3ErrorCode::NoSuchKey);
+        assert_eq!(response.status_code, StatusCode::NOT_FOUND);
+    }
 }
@@ -30,7 +30,9 @@ use tokio_util::io::StreamReader;
 use crate::{
    api_error_response::err_invalid_argument,
    api_get_options::GetObjectOptions,
-    transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info_for_provider},
+    transition_api::{
+        ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, collect_response_body, to_object_info_for_provider,
+    },
 };
 use futures_util::StreamExt;
 use http_body_util::BodyExt;
@@ -39,6 +41,42 @@ use hyper::body::Bytes;
 use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
 use tokio_util::io::ReaderStream;

+fn response_limit_from_range(opts: &GetObjectOptions) -> Result<Option<usize>, std::io::Error> {
+    let Some(range) = opts
+        .headers
+        .iter()
+        .find_map(|(name, value)| name.eq_ignore_ascii_case("range").then_some(value.as_str()))
+    else {
+        return Ok(None);
+    };
+    let Some((unit, bounds)) = range.split_once('=') else {
+        return Ok(None);
+    };
+    if !unit.eq_ignore_ascii_case("bytes") {
+        return Ok(None);
+    }
+    let Some((start, end)) = bounds.split_once('-') else {
+        return Ok(None);
+    };
+    if start.is_empty() || end.is_empty() || end.contains(',') {
+        return Ok(None);
+    }
+    let start = start
+        .parse::<u64>()
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range start is invalid"))?;
+    let end = end
+        .parse::<u64>()
+        .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range end is invalid"))?;
+    let length = end
+        .checked_sub(start)
+        .and_then(|length| length.checked_add(1))
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length overflows"))?;
+    let limit = usize::try_from(length).map_err(|_| {
+        std::io::Error::new(std::io::ErrorKind::InvalidInput, "closed response range length does not fit in memory")
+    })?;
+    Ok(Some(limit))
+}
+
 impl TransitionClient {
    pub fn get_object(&self, bucket_name: &str, object_name: &str, opts: &GetObjectOptions) -> Result<Object, std::io::Error> {
        let _ = opts;
@@ -54,6 +92,7 @@ impl TransitionClient {
        object_name: &str,
        opts: &GetObjectOptions,
    ) -> Result<(ObjectInfo, HeaderMap, ReadCloser), std::io::Error> {
+        let max_response_bytes = response_limit_from_range(opts)?;
        let resp = self
            .execute_method(
                http::Method::GET,
@@ -81,18 +120,214 @@ impl TransitionClient {

        let h = resp.headers().clone();

-        let mut body_vec = Vec::new();
        let mut body = resp.into_body();
-        while let Some(frame) = body.frame().await {
-            let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
-            if let Some(data) = frame.data_ref() {
-                body_vec.extend_from_slice(data);
+        let body_vec = if let Some(limit) = max_response_bytes {
+            collect_response_body(body, limit).await?
+        } else {
+            let mut body_vec = Vec::new();
+            while let Some(frame) = body.frame().await {
+                let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+                if let Some(data) = frame.data_ref() {
+                    body_vec.extend_from_slice(data);
+                }
            }
-        }
+            body_vec
+        };
        Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
    }
 }

+#[cfg(test)]
+mod bounded_response_tests {
+    use super::response_limit_from_range;
+    use crate::{
+        api_get_options::GetObjectOptions,
+        credentials::{Credentials, SignatureType, Static, Value},
+        transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
+    };
+    use http_body_util::Full;
+    use hyper::body::Bytes;
+    use std::time::Duration;
+    use tokio::{
+        io::{AsyncReadExt, AsyncWriteExt},
+        net::TcpListener,
+    };
+
+    #[test]
+    fn closed_range_derives_a_collection_limit_without_new_public_options() {
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(5, 11).expect("the closed range should be valid");
+
+        assert_eq!(response_limit_from_range(&opts).expect("the range should parse"), Some(7));
+    }
+
+    #[tokio::test]
+    async fn response_collection_rejects_the_body_that_exceeds_its_range_limit() {
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+        let max_response_bytes = response_limit_from_range(&opts)
+            .expect("the range should parse")
+            .expect("the closed range should have a limit");
+        let err = collect_response_body(Full::new(Bytes::from_static(b"RustFSxx")), max_response_bytes)
+            .await
+            .expect_err("the collection layer must reject a response larger than its limit");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+    }
+
+    async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let request = tokio::spawn(async move {
+            let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
+            let mut request = Vec::new();
+            let mut buffer = [0; 1024];
+            loop {
+                let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
+                assert_ne!(read, 0, "connection closed before request headers were received");
+                request.extend_from_slice(&buffer[..read]);
+                if request.windows(4).any(|window| window == b"\r\n\r\n") {
+                    break;
+                }
+            }
+            let request = String::from_utf8_lossy(&request).into_owned();
+            let response = format!(
+                "HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+                body.len()
+            );
+            stream
+                .write_all(response.as_bytes())
+                .await
+                .expect("fixture should write response headers");
+            stream.write_all(body).await.expect("fixture should write response body");
+            request
+        });
+        let client = TransitionClient::new(
+            &endpoint,
+            Options {
+                creds: Credentials::new(Static(Value {
+                    access_key_id: "access-key".to_string(),
+                    secret_access_key: "secret-key".to_string(),
+                    signer_type: SignatureType::SignatureV4,
+                    ..Default::default()
+                })),
+                region: "us-east-1".to_string(),
+                bucket_lookup: BucketLookupType::BucketLookupPath,
+                max_retries: 1,
+                ..Default::default()
+            },
+            "",
+        )
+        .await
+        .expect("fixture client should build");
+        Some((client, request))
+    }
+
+    #[tokio::test]
+    async fn real_transport_accepts_the_exact_closed_range_length() {
+        let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
+            return;
+        };
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let (_, _, mut reader) = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect("a seven-byte response should fit the requested range");
+        let mut body = Vec::new();
+        reader
+            .read_to_end(&mut body)
+            .await
+            .expect("bounded response should be readable");
+
+        assert_eq!(body, b"RustFS!");
+        assert!(
+            request
+                .await
+                .expect("fixture should join")
+                .to_ascii_lowercase()
+                .contains("\r\nrange: bytes=0-6\r\n")
+        );
+    }
+
+    #[tokio::test]
+    async fn real_transport_rejects_a_body_larger_than_the_closed_range() {
+        let Some((client, request)) = bounded_get_fixture(b"RustFS!!").await else {
+            return;
+        };
+        let mut opts = GetObjectOptions::default();
+        opts.set_range(0, 6).expect("the probe range should be valid");
+
+        let err = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect_err("an eight-byte response must exceed the seven-byte range limit");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
+        assert!(
+            request
+                .await
+                .expect("fixture should join")
+                .to_ascii_lowercase()
+                .contains("\r\nrange: bytes=0-6\r\n")
+        );
+    }
+
+    #[tokio::test]
+    async fn overflowing_closed_range_is_rejected_before_network_io() {
+        let listener = match TcpListener::bind("127.0.0.1:0").await {
+            Ok(listener) => listener,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("test listener should bind: {err}"),
+        };
+        let endpoint = listener
+            .local_addr()
+            .expect("listener local address should be available")
+            .to_string();
+        let client = TransitionClient::new(
+            &endpoint,
+            Options {
+                creds: Credentials::new(Static(Value {
+                    access_key_id: "access-key".to_string(),
+                    secret_access_key: "secret-key".to_string(),
+                    signer_type: SignatureType::SignatureV4,
+                    ..Default::default()
+                })),
+                region: "us-east-1".to_string(),
+                bucket_lookup: BucketLookupType::BucketLookupPath,
+                max_retries: 1,
+                ..Default::default()
+            },
+            "",
+        )
+        .await
+        .expect("fixture client should build");
+        let mut opts = GetObjectOptions::default();
+        opts.headers
+            .insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
+
+        let err = client
+            .get_object_inner("bucket", "probe", &opts)
+            .await
+            .expect_err("an overflowing closed range must be rejected locally");
+
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+        assert!(
+            tokio::time::timeout(Duration::from_millis(100), listener.accept())
+                .await
+                .is_err()
+        );
+    }
+}
+
 #[derive(Default)]
 pub struct GetRequest {
    pub buffer: Vec<u8>,
@@ -105,6 +105,12 @@ struct DirtyUsageSnapshot {
    covers_all_pending: bool,
 }

+#[derive(Clone, Debug, Default)]
+pub(crate) struct ScannerBucketScanScope {
+    selected_buckets: Option<Arc<HashSet<String>>>,
+    baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
+}
+
 pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
    matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
 }
@@ -146,6 +152,7 @@ fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool {
 pub struct ScannerBucketScanPlan {
    buckets: Vec<BucketInfo>,
    all_buckets: Arc<Vec<BucketInfo>>,
+    scope: ScannerBucketScanScope,
    digest: DataUsageScanPlanDigest,
    leader_epoch: u64,
    tier_registry_generation: u64,
@@ -732,6 +739,8 @@ mod dirty_usage;
 mod guards;
 mod io_cache;
 mod io_cycle;
+#[cfg(test)]
+use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
 pub(crate) use io_cycle::nsscanner_with_storage_status;
 mod io_disk;
 #[cfg(test)]
@@ -14,6 +14,93 @@
 /// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
 use super::*;

+#[derive(Clone, Copy)]
+pub(super) struct ScannerSetCacheGeneration {
+    pub(super) want_cycle: u64,
+    pub(super) leader_epoch: u64,
+    pub(super) tier_registry_generation: u64,
+    pub(super) source: DataUsageCacheSource,
+    pub(super) scan_plan_digest: DataUsageScanPlanDigest,
+}
+
+pub(super) struct PreparedScopedSetScan {
+    pub(super) buckets: Vec<BucketInfo>,
+    pub(super) cache: DataUsageCache,
+}
+
+pub(super) fn prepare_scoped_set_scan(
+    old_cache: &DataUsageCache,
+    set_buckets: &[BucketInfo],
+    all_buckets: &[BucketInfo],
+    scope: &ScannerBucketScanScope,
+    generation: ScannerSetCacheGeneration,
+) -> Option<PreparedScopedSetScan> {
+    let (Some(selected_buckets), Some(baseline_scan_plan_digest)) = (&scope.selected_buckets, scope.baseline_scan_plan_digest)
+    else {
+        return None;
+    };
+    if selected_buckets.is_empty()
+        || !old_cache.info.snapshot_complete
+        || old_cache.info.last_update.is_none()
+        || old_cache.info.name != DATA_USAGE_ROOT
+        || old_cache.info.next_cycle > generation.want_cycle
+        || old_cache.info.leader_epoch != generation.leader_epoch
+        || old_cache.info.tier_registry_generation != Some(generation.tier_registry_generation)
+        || old_cache.info.source != Some(generation.source)
+        || old_cache.info.scan_plan_digest != Some(baseline_scan_plan_digest)
+        || old_cache.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT
+        || old_cache.checked_flatten_complete_scope(DATA_USAGE_ROOT).is_none()
+    {
+        return None;
+    }
+
+    let mut cache = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: DATA_USAGE_ROOT.to_string(),
+            next_cycle: generation.want_cycle,
+            leader_epoch: generation.leader_epoch,
+            tier_registry_generation: Some(generation.tier_registry_generation),
+            source: Some(generation.source),
+            snapshot_complete: false,
+            scan_plan_digest: Some(generation.scan_plan_digest),
+            cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+            lkg_snapshot_complete: true,
+            lkg_next_cycle: Some(old_cache.info.next_cycle),
+            lkg_last_update: old_cache.info.last_update,
+            lkg_leader_epoch: Some(old_cache.info.leader_epoch),
+            lkg_scan_plan_digest: old_cache.info.scan_plan_digest,
+            ..Default::default()
+        },
+        cache: HashMap::new(),
+    };
+    cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
+    let root_hash = crate::hash_path(DATA_USAGE_ROOT);
+    let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
+    for bucket in all_buckets {
+        if !current_bucket_names.insert(bucket.name.as_str()) {
+            return None;
+        }
+        if selected_buckets.contains(&bucket.name) {
+            cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
+            continue;
+        }
+
+        let bucket_hash = crate::hash_path(&bucket.name);
+        old_cache.find(&bucket.name)?;
+        cache.copy_with_children(old_cache, &bucket_hash, &Some(root_hash.clone()));
+        cache.find(&bucket.name)?;
+    }
+
+    Some(PreparedScopedSetScan {
+        buckets: set_buckets
+            .iter()
+            .filter(|bucket| selected_buckets.contains(&bucket.name))
+            .cloned()
+            .collect(),
+        cache,
+    })
+}
+
 #[async_trait::async_trait]
 impl ScannerIOCache for SetDisks {
    #[tracing::instrument(skip(self, budget, scan_plan, updates))]
@@ -27,8 +114,9 @@ impl ScannerIOCache for SetDisks {
        scan_mode: HealScanMode,
    ) -> Result<()> {
        let ScannerBucketScanPlan {
-            buckets,
+            mut buckets,
            all_buckets,
+            scope,
            digest: scan_plan_digest,
            leader_epoch,
            tier_registry_generation,
@@ -63,26 +151,57 @@ impl ScannerIOCache for SetDisks {
                "Scanner old data usage cache load failed; rebuilding from bucket caches"
            );
        }
+        let scoped_scan = prepare_scoped_set_scan(
+            &old_cache,
+            &buckets,
+            &all_buckets,
+            &scope,
+            ScannerSetCacheGeneration {
+                want_cycle,
+                leader_epoch,
+                tier_registry_generation,
+                source,
+                scan_plan_digest,
+            },
+        );
+        let mut scoped_cache = scoped_scan.map(|prepared| {
+            buckets = prepared.buckets;
+            prepared.cache
+        });
        if buckets.is_empty() {
            let now = SystemTime::now();
-            let mut cache = DataUsageCache {
-                info: DataUsageCacheInfo {
-                    name: DATA_USAGE_ROOT.to_string(),
-                    next_cycle: want_cycle,
-                    last_update: Some(now),
-                    leader_epoch,
-                    tier_registry_generation: Some(tier_registry_generation),
-                    source: Some(source),
-                    snapshot_complete: true,
-                    scan_plan_digest: Some(scan_plan_digest),
-                    cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
-                    ..Default::default()
-                },
-                cache: HashMap::new(),
+            let mut cache = match scoped_cache.take() {
+                Some(cache) => cache,
+                None => {
+                    let mut cache = DataUsageCache {
+                        info: DataUsageCacheInfo {
+                            name: DATA_USAGE_ROOT.to_string(),
+                            next_cycle: want_cycle,
+                            leader_epoch,
+                            tier_registry_generation: Some(tier_registry_generation),
+                            source: Some(source),
+                            scan_plan_digest: Some(scan_plan_digest),
+                            cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+                            ..Default::default()
+                        },
+                        cache: HashMap::new(),
+                    };
+                    cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
+                    for bucket in all_buckets.iter() {
+                        cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
+                    }
+                    cache
+                }
            };
-            cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
-            for bucket in all_buckets.iter() {
-                cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
+            cache.info.last_update = Some(now);
+            cache.info.snapshot_complete = true;
+            cache.info.lkg_snapshot_complete = false;
+            cache.info.lkg_next_cycle = None;
+            cache.info.lkg_last_update = None;
+            cache.info.lkg_leader_epoch = None;
+            cache.info.lkg_scan_plan_digest = None;
+            if cache.find(DATA_USAGE_ROOT).is_none() {
+                cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
            }
            reset_disk_bucket_scan_gauges(&pool_label, &set_label);
            return persist_and_publish_cache_snapshot(
@@ -269,92 +388,102 @@ impl ScannerIOCache for SetDisks {
        record_disk_bucket_scans_active(0, &pool_label, &set_label);
        let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone());

-        // Fence a stale set aggregate before copying entries into per-bucket work caches.
-        if old_cache.info.next_cycle <= want_cycle
-            && old_cache.info.leader_epoch <= leader_epoch
-            && old_cache.info.tier_registry_generation != Some(tier_registry_generation)
-        {
-            old_cache.info.scan_plan_digest = None;
-        }
-        let old_lkg = old_cache.info.snapshot_complete.then_some({
-            (
-                old_cache.info.next_cycle,
-                old_cache.info.last_update,
-                old_cache.info.leader_epoch,
-                old_cache.info.scan_plan_digest,
-            )
-        });
-        let prepare_outcome = match old_cache.prepare_for_scan(
-            DATA_USAGE_ROOT,
-            want_cycle,
-            leader_epoch,
-            source,
-            scan_plan_digest,
-            require_cache_source,
-        ) {
-            DataUsageCachePrepareOutcome::RejectedNewerCycle => {
-                cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel);
-                warn!(
-                    target: "rustfs::scanner::io",
-                    event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                    component = LOG_COMPONENT_SCANNER,
-                    subsystem = LOG_SUBSYSTEM_IO,
-                    pool = self.pool_index,
-                    set = self.set_index,
-                    cache_name = DATA_USAGE_CACHE_NAME,
-                    requested_cycle = want_cycle,
-                    cached_cycle = old_cache.info.next_cycle,
-                    state = "stale_cycle_rejected",
-                    "Scanner rejected a set cache cycle regression"
-                );
-                return Ok(());
+        let mut cache = if let Some(cache) = scoped_cache.take() {
+            cache
+        } else {
+            // Fence a stale set aggregate before copying entries into per-bucket work caches.
+            if old_cache.info.next_cycle <= want_cycle
+                && old_cache.info.leader_epoch <= leader_epoch
+                && old_cache.info.tier_registry_generation != Some(tier_registry_generation)
+            {
+                old_cache.info.scan_plan_digest = None;
            }
-            DataUsageCachePrepareOutcome::RejectedNewerLeader => {
-                warn!(
-                    target: "rustfs::scanner::io",
-                    event = EVENT_SCANNER_CACHE_PERSIST_STATE,
-                    component = LOG_COMPONENT_SCANNER,
-                    subsystem = LOG_SUBSYSTEM_IO,
-                    pool = self.pool_index,
-                    set = self.set_index,
-                    cache_name = DATA_USAGE_CACHE_NAME,
-                    requested_epoch = leader_epoch,
-                    cached_epoch = old_cache.info.leader_epoch,
-                    state = "stale_leader_rejected",
-                    "Scanner rejected work from an older leader epoch"
-                );
-                return Ok(());
-            }
-            outcome => outcome,
-        };
-        if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused)
-            && let Some((cycle, last_update, epoch, digest)) = old_lkg
-        {
-            old_cache.info.lkg_snapshot_complete = true;
-            old_cache.info.lkg_next_cycle = Some(cycle);
-            old_cache.info.lkg_last_update = last_update;
-            old_cache.info.lkg_leader_epoch = Some(epoch);
-            old_cache.info.lkg_scan_plan_digest = digest;
-        }
-
-        let mut cache = DataUsageCache {
-            info: DataUsageCacheInfo {
-                name: DATA_USAGE_ROOT.to_string(),
-                next_cycle: want_cycle,
+            let old_lkg = old_cache.info.snapshot_complete.then_some({
+                (
+                    old_cache.info.next_cycle,
+                    old_cache.info.last_update,
+                    old_cache.info.leader_epoch,
+                    old_cache.info.scan_plan_digest,
+                )
+            });
+            let prepare_outcome = match old_cache.prepare_for_scan(
+                DATA_USAGE_ROOT,
+                want_cycle,
                leader_epoch,
-                tier_registry_generation: Some(tier_registry_generation),
-                source: Some(source),
-                snapshot_complete: false,
-                scan_plan_digest: Some(scan_plan_digest),
-                cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
-                ..Default::default()
-            },
-            cache: HashMap::new(),
+                source,
+                scan_plan_digest,
+                require_cache_source,
+            ) {
+                DataUsageCachePrepareOutcome::RejectedNewerCycle => {
+                    cache_cycle_floor.fetch_max(old_cache.info.next_cycle, Ordering::AcqRel);
+                    warn!(
+                        target: "rustfs::scanner::io",
+                        event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_IO,
+                        pool = self.pool_index,
+                        set = self.set_index,
+                        cache_name = DATA_USAGE_CACHE_NAME,
+                        requested_cycle = want_cycle,
+                        cached_cycle = old_cache.info.next_cycle,
+                        state = "stale_cycle_rejected",
+                        "Scanner rejected a set cache cycle regression"
+                    );
+                    return Ok(());
+                }
+                DataUsageCachePrepareOutcome::RejectedNewerLeader => {
+                    warn!(
+                        target: "rustfs::scanner::io",
+                        event = EVENT_SCANNER_CACHE_PERSIST_STATE,
+                        component = LOG_COMPONENT_SCANNER,
+                        subsystem = LOG_SUBSYSTEM_IO,
+                        pool = self.pool_index,
+                        set = self.set_index,
+                        cache_name = DATA_USAGE_CACHE_NAME,
+                        requested_epoch = leader_epoch,
+                        cached_epoch = old_cache.info.leader_epoch,
+                        state = "stale_leader_rejected",
+                        "Scanner rejected work from an older leader epoch"
+                    );
+                    return Ok(());
+                }
+                outcome => outcome,
+            };
+            if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused)
+                && let Some((cycle, last_update, epoch, digest)) = old_lkg
+            {
+                old_cache.info.lkg_snapshot_complete = true;
+                old_cache.info.lkg_next_cycle = Some(cycle);
+                old_cache.info.lkg_last_update = last_update;
+                old_cache.info.lkg_leader_epoch = Some(epoch);
+                old_cache.info.lkg_scan_plan_digest = digest;
+            }
+
+            let mut cache = DataUsageCache {
+                info: DataUsageCacheInfo {
+                    name: DATA_USAGE_ROOT.to_string(),
+                    next_cycle: want_cycle,
+                    leader_epoch,
+                    tier_registry_generation: Some(tier_registry_generation),
+                    source: Some(source),
+                    snapshot_complete: false,
+                    scan_plan_digest: Some(scan_plan_digest),
+                    cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+                    lkg_snapshot_complete: old_cache.info.lkg_snapshot_complete,
+                    lkg_next_cycle: old_cache.info.lkg_next_cycle,
+                    lkg_last_update: old_cache.info.lkg_last_update,
+                    lkg_leader_epoch: old_cache.info.lkg_leader_epoch,
+                    lkg_scan_plan_digest: old_cache.info.lkg_scan_plan_digest,
+                    ..Default::default()
+                },
+                cache: HashMap::new(),
+            };
+            cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
+            for bucket in all_buckets.iter() {
+                cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
+            }
+            cache
        };
-        cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
-        for bucket in all_buckets.iter() {
-            cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
-        }

        let (bucket_tx, bucket_rx) = mpsc::channel::<BucketInfo>(buckets.len());

@@ -1257,11 +1386,6 @@ impl ScannerIOCache for SetDisks {
            incomplete_scope.info.snapshot_complete = false;
            incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest);
            incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
-            incomplete_scope.info.lkg_snapshot_complete = old_cache.info.lkg_snapshot_complete;
-            incomplete_scope.info.lkg_next_cycle = old_cache.info.lkg_next_cycle;
-            incomplete_scope.info.lkg_last_update = old_cache.info.lkg_last_update;
-            incomplete_scope.info.lkg_leader_epoch = old_cache.info.lkg_leader_epoch;
-            incomplete_scope.info.lkg_scan_plan_digest = old_cache.info.lkg_scan_plan_digest;
            if let Err(e) = updates.send(incomplete_scope).await {
                error!(
                    target: "rustfs::scanner::io",
@@ -63,6 +63,41 @@ pub(crate) async fn nsscanner_with_storage_status<S>(
 where
    S: ScannerStorage,
 {
+    let request = ScannerCycleRequest {
+        ctx,
+        budget,
+        updates,
+        want_cycle,
+        leader_epoch,
+        scan_mode,
+        scan_scope: ScannerBucketScanScope::default(),
+    };
+    nsscanner_with_storage_status_scoped(store, request).await
+}
+
+pub(crate) struct ScannerCycleRequest {
+    pub(crate) ctx: CancellationToken,
+    pub(crate) budget: Arc<ScannerCycleBudget>,
+    pub(crate) updates: mpsc::Sender<DataUsageInfo>,
+    pub(crate) want_cycle: u64,
+    pub(crate) leader_epoch: u64,
+    pub(crate) scan_mode: HealScanMode,
+    pub(crate) scan_scope: ScannerBucketScanScope,
+}
+
+pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
+where
+    S: ScannerStorage,
+{
+    let ScannerCycleRequest {
+        ctx,
+        budget,
+        updates,
+        want_cycle,
+        leader_epoch,
+        scan_mode,
+        scan_scope,
+    } = request;
    let child_token = ctx.child_token();
    let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);

@@ -280,6 +315,7 @@ where
        let scan_plan = ScannerBucketScanPlan {
            buckets: set_buckets,
            all_buckets: Arc::clone(&all_buckets),
+            scope: scan_scope.clone(),
            digest: scan_plan_digest,
            leader_epoch,
            tier_registry_generation,
@@ -765,6 +765,155 @@ fn bucket_usage_scan_order_prioritizes_dirty_buckets() {
    assert_eq!(names, vec!["dirty", "missing", "cached"]);
 }

+fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsageScanPlanDigest) -> DataUsageCache {
+    let mut cache = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: DATA_USAGE_ROOT.to_string(),
+            next_cycle: 7,
+            last_update: Some(SystemTime::now()),
+            leader_epoch: 11,
+            source: Some(DataUsageCacheSource::new(1, 2)),
+            snapshot_complete: true,
+            scan_plan_digest: Some(scan_plan_digest),
+            cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
+            tier_registry_generation: Some(13),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
+    for (bucket, size) in buckets {
+        cache.replace(
+            bucket,
+            DATA_USAGE_ROOT,
+            DataUsageEntry {
+                size: *size,
+                objects: 1,
+                ..Default::default()
+            },
+        );
+    }
+    cache
+}
+
+#[test]
+fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
+    let baseline_digest = DataUsageScanPlanDigest([1; 32]);
+    let current_digest = DataUsageScanPlanDigest([2; 32]);
+    let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20), ("deleted", 30)], baseline_digest);
+    old_cache.replace(
+        "stable/prefix",
+        "stable",
+        DataUsageEntry {
+            size: 5,
+            objects: 1,
+            ..Default::default()
+        },
+    );
+    let all_buckets = vec![bucket_info("stable"), bucket_info("dirty")];
+    let selected_buckets = Arc::new(HashSet::from(["dirty".to_string(), "deleted".to_string()]));
+
+    let prepared = prepare_scoped_set_scan(
+        &old_cache,
+        &all_buckets,
+        &all_buckets,
+        &ScannerBucketScanScope {
+            selected_buckets: Some(selected_buckets),
+            baseline_scan_plan_digest: Some(baseline_digest),
+        },
+        ScannerSetCacheGeneration {
+            want_cycle: 8,
+            leader_epoch: 11,
+            tier_registry_generation: 13,
+            source: DataUsageCacheSource::new(1, 2),
+            scan_plan_digest: current_digest,
+        },
+    )
+    .expect("complete matching set cache should support a scoped scan");
+
+    assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
+    let stable = prepared
+        .cache
+        .checked_flatten("stable")
+        .expect("unselected bucket subtree should be retained");
+    assert_eq!((stable.size, stable.objects), (15, 2));
+    assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
+    assert!(prepared.cache.find("deleted").is_none());
+    assert_eq!(prepared.cache.info.scan_plan_digest, Some(current_digest));
+    assert_eq!(prepared.cache.info.next_cycle, 8);
+    assert!(!prepared.cache.info.snapshot_complete);
+    assert!(prepared.cache.info.lkg_snapshot_complete);
+    assert_eq!(prepared.cache.info.lkg_next_cycle, Some(7));
+    assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
+}
+
+#[test]
+fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() {
+    let baseline_digest = DataUsageScanPlanDigest([3; 32]);
+    let old_cache = complete_set_usage_cache(&[("stable", 10)], baseline_digest);
+    let all_buckets = vec![bucket_info("stable"), bucket_info("new")];
+
+    assert!(
+        prepare_scoped_set_scan(
+            &old_cache,
+            &all_buckets,
+            &all_buckets,
+            &ScannerBucketScanScope {
+                selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
+                baseline_scan_plan_digest: Some(baseline_digest),
+            },
+            ScannerSetCacheGeneration {
+                want_cycle: 8,
+                leader_epoch: 11,
+                tier_registry_generation: 13,
+                source: DataUsageCacheSource::new(1, 2),
+                scan_plan_digest: DataUsageScanPlanDigest([4; 32]),
+            },
+        )
+        .is_none()
+    );
+}
+
+#[test]
+fn scoped_set_scan_requires_an_exact_complete_baseline() {
+    let baseline_digest = DataUsageScanPlanDigest([5; 32]);
+    let all_buckets = vec![bucket_info("dirty")];
+    let scope = ScannerBucketScanScope {
+        selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
+        baseline_scan_plan_digest: Some(baseline_digest),
+    };
+    let generation = ScannerSetCacheGeneration {
+        want_cycle: 8,
+        leader_epoch: 11,
+        tier_registry_generation: 13,
+        source: DataUsageCacheSource::new(1, 2),
+        scan_plan_digest: DataUsageScanPlanDigest([6; 32]),
+    };
+
+    let mut incomplete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
+    incomplete.info.snapshot_complete = false;
+    assert!(prepare_scoped_set_scan(&incomplete, &all_buckets, &all_buckets, &scope, generation).is_none());
+
+    let mut not_durable = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
+    not_durable.info.last_update = None;
+    assert!(prepare_scoped_set_scan(&not_durable, &all_buckets, &all_buckets, &scope, generation).is_none());
+
+    let mut wrong_digest = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
+    wrong_digest.info.scan_plan_digest = Some(DataUsageScanPlanDigest([7; 32]));
+    assert!(prepare_scoped_set_scan(&wrong_digest, &all_buckets, &all_buckets, &scope, generation).is_none());
+
+    let empty_scope = ScannerBucketScanScope {
+        selected_buckets: Some(Arc::new(HashSet::new())),
+        baseline_scan_plan_digest: Some(baseline_digest),
+    };
+    let complete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
+    assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &empty_scope, generation).is_none());
+
+    let mut future_cache = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
+    future_cache.info.next_cycle = generation.want_cycle.saturating_add(1);
+    assert!(prepare_scoped_set_scan(&future_cache, &all_buckets, &all_buckets, &scope, generation).is_none());
+}
+
 #[test]
 fn record_set_scan_failure_preserves_first_error() {
    let mut first = None;
@@ -12,10 +12,10 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
 |---|---|---|---|
 | Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p <crate>`); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) |
 | ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) |
-| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) |
+| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`, `e2e-distributed`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); 4-node 4-disk map: [distributed-e2e.md](distributed-e2e.md) |
 | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` | PR, report-only (second half of the `End-to-End Tests` job) |
 | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only |
-| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree |
+| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/offline-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos with the `e2e-distributed` nightly lane |
 | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus |
 | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p <crate>` | On demand; never a gate |

@@ -61,6 +61,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the
 | `e2e-full` | Merge-queue / main-push single-node e2e lane |
 | `e2e-repl-nightly` | Nightly slow / cross-process replication lane |
 | `e2e-nightly` | Nightly serial multi-process cluster fault lane |
+| `e2e-distributed` | Nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
 | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` |

 Membership of each e2e profile is pinned by a digest in `.config/e2e-<profile>-selection.txt` and checked by `scripts/check_test_wiring.py --check-profile <profile>` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only):
@@ -74,6 +74,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` |
 | `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
+| `e2e-distributed.yml` (nightly) | `distributed` | 4-node 4-disk e2e gate including direct/rolling upgrade of historical objects and IAM AK/SK; JUnit, membership listing, server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` |
 | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
 | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
 | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |
@@ -0,0 +1,70 @@
+# Distributed 4-node 4-disk e2e
+
+**Use this when:** adding or diagnosing GitHub Actions coverage for a 4-node cluster, or deciding whether a behaviour belongs in `e2e-distributed` versus the single-node `e2e-full` lane, the nightly cluster-fault lane, or the hardware functional chain.
+**Source of truth:** `crates/e2e_test/src/distributed/`, `[profile.e2e-distributed]` in `.config/nextest.toml`, `.github/workflows/e2e-distributed.yml`.
+
+## Topology
+
+The in-tree harness runs every node on `127.0.0.1` with a distinct port. That matches `RustFSTestClusterEnvironment` in `crates/e2e_test/src/common.rs`:
+
+| Layout | Constructor | Use |
+|---|---|---|
+| 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos |
+| 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release |
+| 2 single-node pools × 4 drives | `ClusterTopology::per_node_pools(4, [[0],[1]])` | Harness-only: `append_single_node_pool` unit tests. Live multi-pool expand/restart currently dies with `pool metadata recovery required`; this lane does not change that production gate |
+
+A pool striped across several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped pools remain the hardware functional-chain / backlog #1313 / #1314 lane.
+
+Decommission and rebalance POST on the 4×4 single-pool layout is refused by the current product (`single pool deployments do not support decommission`, NotImplemented, or opaque 500 InternalError when the inner pool-meta fence is wrapped). 502/503 are not treated as a product refusal. Those cases still assert object bytes and SHA-256; when the API starts they wait for completion and assert post-move integrity. They do not treat a refusal as a successful move. This lane does not change production pool-meta bootstrap, write-fence, or decommission policy; it only observes the current server behavior.
+
+## What this lane covers
+
+`cargo nextest run --profile e2e-distributed -p e2e_test` selects `distributed::*`:
+
+- S3 put / get / head / list / copy / rename / delete / presign / empty object
+- Object Lock COMPLIANCE, GOVERNANCE (with bypass), legal hold
+- Versioning, version GET, delete marker
+- Bucket replication between two 4-node clusters; hard quota
+- Health / admin info / storageinfo / audit target list
+- Pool restart, decommission/rebalance *attempts*, checksum integrity, S3 during those attempts on 4×4. Live multi-pool expand/restart is a production pool-meta bootstrap limitation and is not patched here; `append_single_node_pool` is covered by harness unit tests
+- Site replication object convergence
+- High-concurrency PUT/GET; concurrent PUT during decommission
+- Node kill/restart, full process restart, drive offline (4×4). Volume-proxy blackhole stays in `cluster_volume_fault_proxy_pass_smoke` (2×2); a 4-node volume proxy cannot format because RPC audience is the listen port
+- Multipart, cross-node listing, list-buckets agreement
+- Concurrent GET while a peer node is killed
+- Direct and rolling upgrade from the pinned previous release: historical objects, versioned history, and IAM user AK/SK still work afterwards
+
+## Existing Actions gaps this lane does not replace
+
+Those suites stay in place; this lane fills the in-tree 4×4 hole they leave.
+
+| Existing lane | Gap |
+|---|---|
+| `rustfs-*-test.yml` functional chain | Clones private `rustfs/auto-testing`, runs on three shared VMs (`vm000`–`vm002`), `continue-on-error: true`, not a merge signal, not 4 nodes. Hardware `rustfs-upgrade-test.yml` stays there |
+| `e2e-upgrade.yml` | Single-node SSE/multipart/delete-marker contracts plus mixed-version listing; does not pin IAM user AK/SK on a 4-node cluster |
+| `e2e-smoke` / `e2e-full` | Almost all cases are single-node |
+| `e2e-nightly` | 4-node cluster faults and heal, not S3/lock/versioning/quota/decommission matrix |
+| `e2e-repl-nightly` | Site and bucket replication on 1–3 *single-node* processes |
+| `e2e-s3tests.yml` `multi` | Weekly ceph/s3-tests against Docker 4-node; not lock/WORM, decommission, chaos, or checksum integrity |
+| `crates/e2e_test/src/chaos.rs` | Single-node disk faults only |
+
+Hardware power-loss, NIC pull, and real disk replacement still belong on the smoke-testing VMs. This lane simulates those with SIGKILL, directory rename, and `FaultProxy` blackhole.
+
+## Run
+
+```bash
+cargo build -p rustfs --bins
+# Upgrade cases require the pinned previous binary (CI downloads it).
+export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2
+cargo nextest run --profile e2e-distributed -p e2e_test
+```
+
+Without `RUSTFS_UPGRADE_SOURCE_BINARY` the two `distributed::upgrade_test::*` cases fail closed. Filter them out for a local run that is not checking upgrade:
+
+```bash
+cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distributed::upgrade_test::/)'
+```
+
+The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check.
+
+Membership is pinned by `.config/e2e-distributed-selection.txt`. Update it with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed <listing.json> linux` after adding or renaming a case.
@@ -80,6 +80,82 @@ fn wasabi_payload_name(config: &TierConfig) -> S3Result<String> {
        .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Wasabi configuration"))
 }

+fn normalize_add_tier_payload_name(config: &mut TierConfig) -> S3Result<()> {
+    match config.tier_type {
+        TierType::S3 => {
+            let _ = config
+                .s3
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?;
+        }
+        TierType::Wasabi => config.name = wasabi_payload_name(config)?,
+        TierType::RustFS => {
+            config.name = config
+                .rustfs
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
+                .name
+                .clone();
+        }
+        TierType::MinIO => {
+            config.name = config
+                .minio
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
+                .name
+                .clone();
+        }
+        TierType::Aliyun => {
+            config.name = config
+                .aliyun
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
+                .name
+                .clone();
+        }
+        TierType::Tencent => {
+            config.name = config
+                .tencent
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
+                .name
+                .clone();
+        }
+        TierType::Huaweicloud => {
+            config.name = config
+                .huaweicloud
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
+                .name
+                .clone();
+        }
+        TierType::Azure => {
+            config.name = config
+                .azure
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
+                .name
+                .clone();
+        }
+        TierType::GCS => {
+            let _ = config
+                .gcs
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?;
+        }
+        TierType::R2 => {
+            config.name = config
+                .r2
+                .as_ref()
+                .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
+                .name
+                .clone();
+        }
+        TierType::Unsupported => {}
+    }
+    Ok(())
+}
+
 fn spawn_transition_tier_config_propagation(action: &'static str) {
    if let Some(notification_sys) = current_notification_system() {
        debug!(
@@ -263,75 +339,7 @@ impl Operation for AddTier {
        let mut args: TierConfig = serde_json::from_slice(&body)
            .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("invalid JSON: {e}")))?;

-        match args.tier_type {
-            TierType::S3 => {
-                args.name = args
-                    .s3
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing S3 configuration"))?
-                    .name;
-            }
-            TierType::Wasabi => {
-                args.name = wasabi_payload_name(&args)?;
-            }
-            TierType::RustFS => {
-                args.name = args
-                    .rustfs
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing RustFS configuration"))?
-                    .name;
-            }
-            TierType::MinIO => {
-                args.name = args
-                    .minio
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing MinIO configuration"))?
-                    .name;
-            }
-            TierType::Aliyun => {
-                args.name = args
-                    .aliyun
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Aliyun configuration"))?
-                    .name;
-            }
-            TierType::Tencent => {
-                args.name = args
-                    .tencent
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Tencent configuration"))?
-                    .name;
-            }
-            TierType::Huaweicloud => {
-                args.name = args
-                    .huaweicloud
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Huawei Cloud configuration"))?
-                    .name;
-            }
-            TierType::Azure => {
-                args.name = args
-                    .azure
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing Azure configuration"))?
-                    .name;
-            }
-            TierType::GCS => {
-                args.name = args
-                    .gcs
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing GCS configuration"))?
-                    .name;
-            }
-            TierType::R2 => {
-                args.name = args
-                    .r2
-                    .clone()
-                    .ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "missing R2 configuration"))?
-                    .name;
-            }
-            _ => (),
-        }
+        normalize_add_tier_payload_name(&mut args)?;
        debug!(
            event = EVENT_ADMIN_TIER_STATE,
            component = LOG_COMPONENT_ADMIN,
@@ -1150,6 +1158,44 @@ mod tests {
        assert_eq!(err.message(), Some("missing Wasabi configuration"));
    }

+    #[test]
+    fn add_tier_payload_preserves_canonical_madmin_s3_and_gcs_names() {
+        for (provider, wire) in [
+            (
+                "S3",
+                serde_json::json!({
+                    "Type": "s3",
+                    "Name": "COLD-S3",
+                    "S3": {
+                        "Endpoint": "https://s3.example.invalid",
+                        "AccessKey": "access",
+                        "SecretKey": "secret",
+                        "Bucket": "archive"
+                    }
+                }),
+            ),
+            (
+                "GCS",
+                serde_json::json!({
+                    "Type": "gcs",
+                    "Name": "COLD-GCS",
+                    "GCS": {
+                        "Endpoint": "https://storage.googleapis.com",
+                        "Creds": "e30=",
+                        "Bucket": "archive"
+                    }
+                }),
+            ),
+        ] {
+            let mut config: TierConfig = serde_json::from_value(wire).expect("canonical madmin payload should decode");
+            let expected = config.name.clone();
+
+            normalize_add_tier_payload_name(&mut config).expect("canonical madmin payload should pass the handler boundary");
+
+            assert_eq!(config.name, expected, "{provider} top-level Name must not be cleared");
+        }
+    }
+
    #[test]
    fn resolve_tier_name_prefers_path_parameter() {
        let uri: Uri = "/rustfs/admin/v3/tier/HOT?tier=COLD".parse().expect("uri should parse");
@@ -1756,5 +1802,11 @@ mod tests {
        }

        assert!(!production.contains("check_key_valid(get_session_token"));
+
+        let add_tier = source_block(production, "impl Operation for AddTier");
+        assert!(
+            add_tier.contains("normalize_add_tier_payload_name(&mut args)?;"),
+            "AddTier must preserve canonical top-level provider names through the tested boundary helper"
+        );
    }
 }
@@ -170,12 +170,12 @@ use s3s::dto::{
    DeleteObjectsOutput, DeletedObject, ETag, GetObjectAttributesInput, GetObjectAttributesOutput, GetObjectAttributesParts,
    GetObjectInput, GetObjectOutput, HeadObjectInput, HeadObjectOutput, MetadataDirective, ObjectAttributes, ObjectLockLegalHold,
    ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput,
-    PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm,
-    SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption,
-    ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat,
-    WebsiteRedirectLocation,
+    PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreRequestType, RestoreStatus,
+    SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput,
+    ServerSideEncryption, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader,
+    Timestamp, TimestampFormat, WebsiteRedirectLocation,
 };
-use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
+use s3s::header::X_AMZ_RESTORE;
 use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
 use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};

@@ -165,18 +165,49 @@ impl DefaultObjectUsecase {

        validate_table_catalog_object_mutation(&bucket, &object).await?;

-        let rreq = rreq.ok_or_else(|| {
-            S3Error::with_message(S3ErrorCode::Custom("ErrValidRestoreObject".into()), "restore request is required")
-        })?;
+        // Typed S3 errors on every RestoreObject failure (backlog#2205): a
+        // `Custom` code serializes as a generic 500, which makes SDK clients
+        // retry client errors and conflicts alike.
+        let rreq = rreq.ok_or_else(|| S3Error::with_message(S3ErrorCode::MalformedXML, "restore request is required"))?;
+
+        // SELECT-type restore is not supported (backlog#1341). The restore
+        // path can only write the retrieved bytes back to the source key, so
+        // honouring a SELECT request overwrote the source object with
+        // SELECT-only metadata (dropping `x-amz-restore`, user metadata and
+        // tags on an unversioned bucket, or publishing a bogus latest version
+        // on a versioned one) while never writing anything to
+        // `OutputLocation.S3`. Reject before any guard, metadata write or
+        // fabricated `x-amz-restore-output-path` response header.
+        if rreq
+            .type_
+            .as_ref()
+            .is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
+        {
+            return Err(S3Error::with_message(
+                S3ErrorCode::NotImplemented,
+                "SELECT restore requests are not supported.",
+            ));
+        }

        let Some(store) = self.object_store() else {
            return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
        };

+        // Validate the request shape before taking any lock or reading the
+        // object: a malformed request or an illegal `Days` value is a client
+        // error, and the validator messages are static — they carry no
+        // backend or credential detail.
+        if let Err(e) = validate_restore_request(&rreq, store.clone()) {
+            return Err(S3Error::with_message(
+                S3ErrorCode::InvalidRequest,
+                format!("Restore object validation failed: {e}"),
+            ));
+        }
+
        let version_id_str = version_id.clone().unwrap_or_default();
        let mut opts = post_restore_opts(&version_id_str, &bucket, &object)
            .await
-            .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrPostRestoreOpts".into()), "restore object failed."))?;
+            .map_err(ApiError::from)?;
        apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
        // `apply_bucket_generation_guard` deliberately tolerates a missing guard
        // (only the S3 access layer installs one), so this must not hard-require
@@ -192,11 +223,7 @@ impl DefaultObjectUsecase {
            }
        };

-        // SELECT-type restores skip both the ongoing check and the metadata
-        // write below, so the accept guard would protect nothing for them —
-        // they keep the plain (read-locked) accept path.
-        let is_select = rreq.type_.as_ref().is_some_and(|t| t.as_str() == "SELECT");
-        let restore_operation_id = (!is_select).then(Uuid::new_v4);
+        let restore_operation_id = Some(Uuid::new_v4());
        let mut restore_worker_guard = if let Some(operation_id) = restore_operation_id {
            Some(
                store
@@ -210,10 +237,10 @@ impl DefaultObjectUsecase {

        // Hold the restore-accept guard across the restore-status read, the
        // ongoing/already-restored decision, and the metadata write below, so
-        // two concurrent (non-SELECT) POST ?restore cannot both observe
-        // ongoing=false and both start a copy-back (backlog#1304). Reads and
-        // writes inside this scope run with no_lock; the guard is dropped
-        // before the copy-back is spawned so it never blocks readers.
+        // two concurrent POST ?restore cannot both observe ongoing=false and
+        // both start a copy-back (backlog#1304). Reads and writes inside this
+        // scope run with no_lock; the guard is dropped before the copy-back is
+        // spawned so it never blocks readers.
        // Contention on the accept guard (e.g. a concurrent accept or an
        // in-flight commit on the same object) is transient — answer 503
        // SlowDown so SDK clients back off and retry instead of treating it
@@ -222,9 +249,7 @@ impl DefaultObjectUsecase {
        if store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)? != restore_bucket_incarnation_id {
            return Err(ApiError::from(StorageError::BucketNotFound(bucket.clone())).into());
        }
-        let mut accept_guard = if is_select {
-            None
-        } else {
+        let mut accept_guard = {
            let guard = store
                .acquire_restore_accept_guard(&bucket, &object)
                .await
@@ -233,24 +258,17 @@ impl DefaultObjectUsecase {
            Some(guard)
        };

-        let mut obj_info = store
-            .get_object_info(&bucket, &object, &opts)
-            .await
-            .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed."))?;
+        // A missing key or version must stay NoSuchKey / NoSuchVersion, and an
+        // authorization or storage failure must keep its own identity, so map
+        // the storage error instead of flattening it (backlog#2205).
+        let mut obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;

-        // Check if object is in a transitioned state
+        // Restoring an object that was never transitioned is the S3
+        // InvalidObjectState case, not an internal error.
        if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
            return Err(S3Error::with_message(
-                S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
-                "restore object failed.",
-            ));
-        }
-
-        // Validate restore request
-        if let Err(e) = validate_restore_request(&rreq, store.clone()) {
-            return Err(S3Error::with_message(
-                S3ErrorCode::Custom("ErrValidRestoreObject".into()),
-                format!("Restore object validation failed: {}", e),
+                S3ErrorCode::InvalidObjectState,
+                "The operation is not valid for the object's storage class.",
            ));
        }

@@ -260,7 +278,7 @@ impl DefaultObjectUsecase {
        // would create an ABBA cycle. If the probe succeeds, reacquire and
        // re-read the object before replacing the exact orphan generation.
        let mut superseded_worker_guard = None;
-        if obj_info.restore_ongoing && !is_select {
+        if obj_info.restore_ongoing {
            match classify_ongoing_restore(obj_info.user_defined.as_ref(), OffsetDateTime::now_utc()) {
                OngoingRestoreRecovery::ActiveOrUnsafe => {
                    return Err(S3Error::with_message(
@@ -293,13 +311,11 @@ impl DefaultObjectUsecase {
                            .map_err(|_| S3Error::with_message(S3ErrorCode::SlowDown, "restore object failed."))?,
                    );
                    opts.no_lock = true;
-                    obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(|_| {
-                        S3Error::with_message(S3ErrorCode::Custom("ErrInvalidObjectState".into()), "restore object failed.")
-                    })?;
+                    obj_info = store.get_object_info(&bucket, &object, &opts).await.map_err(ApiError::from)?;
                    if obj_info.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
                        return Err(S3Error::with_message(
-                            S3ErrorCode::Custom("ErrInvalidTransitionedState".into()),
-                            "restore object failed.",
+                            S3ErrorCode::InvalidObjectState,
+                            "The operation is not valid for the object's storage class.",
                        ));
                    }
                    if obj_info.restore_ongoing {
@@ -327,11 +343,11 @@ impl DefaultObjectUsecase {
        remove_str(&mut metadata, SUFFIX_RESTORE_OPERATION_ID);
        remove_str(&mut metadata, SUFFIX_RESTORE_WORKER_LOCK);

-        let mut header = HeaderMap::new();
-
        let event_object_info = obj_info.clone();
        let obj_info_ = obj_info.clone();
-        if !is_select {
+        // Scopes the accept-guarded metadata write: everything below runs
+        // inside the accept critical section, which is released right after.
+        {
            obj_info.metadata_only = true;
            metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), rreq.days.unwrap_or(1).to_string());
            let request_date = OffsetDateTime::now_utc().format(&Rfc3339).map_err(|e| {
@@ -403,7 +419,7 @@ impl DefaultObjectUsecase {
                    &restore_dst_opts,
                )
                .await
-                .map_err(|_| S3Error::with_message(S3ErrorCode::Custom("ErrCopyObject".into()), "restore object failed."))?;
+                .map_err(ApiError::from)?;
            rustfs_scanner::record_dirty_usage_bucket(&bucket);
            #[cfg(test)]
            maybe_pause_after_restore_status_commit(&bucket, &object).await;
@@ -429,17 +445,6 @@ impl DefaultObjectUsecase {
        drop(accept_guard);
        drop(restore_bucket_lifecycle_guard);

-        // Handle output location for SELECT requests
-        if let Some(output_location) = &rreq.output_location
-            && let Some(s3) = &output_location.s3
-            && !s3.bucket_name.is_empty()
-        {
-            let restore_object = Uuid::new_v4().to_string();
-            if let Ok(header_value) = format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse() {
-                header.insert(X_AMZ_RESTORE_OUTPUT_PATH, header_value);
-            }
-        }
-
        // Spawn restoration task in the background. Pin the copy-back to the
        // version the accept resolved and flagged: with a versionless request
        // on a versioned bucket, a PUT landing between the accept and the
@@ -499,7 +504,7 @@ impl DefaultObjectUsecase {
            restore_output_path: None,
        };
        helper = helper.object(event_object_info).version_id(version_id_str);
-        let result = Ok(S3Response::with_headers(output, header));
+        let result = Ok(S3Response::new(output));
        let _ = helper.complete(&result);
        result
    }
@@ -629,6 +634,28 @@ mod tests {
        assert_eq!(classify_ongoing_restore(&conflicting_date, now), OngoingRestoreRecovery::ActiveOrUnsafe);
    }

+    fn restore_request(days: Option<i32>) -> RestoreRequest {
+        RestoreRequest {
+            days,
+            description: None,
+            glacier_job_parameters: None,
+            output_location: None,
+            select_parameters: None,
+            tier: None,
+            type_: None,
+        }
+    }
+
+    fn restore_input(bucket: &str, key: &str, rreq: RestoreRequest) -> RestoreObjectInput {
+        RestoreObjectInput::builder()
+            .bucket(bucket.to_string())
+            .key(key.to_string())
+            .restore_request(Some(rreq))
+            .build()
+            .expect("restore input should build")
+    }
+
+    /// backlog#2205: a missing restore body is a client error, not a 500.
    #[tokio::test]
    async fn execute_restore_object_rejects_missing_restore_request() {
        let input = RestoreObjectInput::builder()
@@ -641,10 +668,92 @@ mod tests {
        let usecase = DefaultObjectUsecase::without_context();

        let err = usecase.execute_restore_object(req).await.unwrap_err();
-        match err.code() {
-            S3ErrorCode::Custom(code) => assert_eq!(code, "ErrValidRestoreObject"),
-            code => panic!("unexpected error code: {:?}", code),
-        }
+        assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
+    }
+
+    /// backlog#1341: a SELECT restore must be rejected outright — the restore
+    /// path can only write back to the source key, never to
+    /// `OutputLocation.S3`. Rejection happens before the store is resolved, so
+    /// an uninitialized usecase still answers NotImplemented rather than the
+    /// InternalError every request that gets past this point returns.
+    #[tokio::test]
+    async fn execute_restore_object_rejects_select_type() {
+        let mut rreq = restore_request(None);
+        rreq.type_ = Some(s3s::dto::RestoreRequestType::from_static(s3s::dto::RestoreRequestType::SELECT));
+
+        let req = build_request(restore_input("test-bucket", "test-key", rreq), Method::POST);
+        let usecase = DefaultObjectUsecase::without_context();
+
+        let err = usecase.execute_restore_object(req).await.unwrap_err();
+        assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
+    }
+
+    /// backlog#2205: every RestoreObject failure that reaches storage must
+    /// keep its typed S3 identity. Before this, a missing key, a malformed
+    /// version-id, an illegal `Days` and an object that was never transitioned
+    /// all collapsed into `Custom(...)` codes, which serialize as a retryable
+    /// HTTP 500.
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn execute_restore_object_maps_failures_to_typed_s3_errors() {
+        use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
+
+        let store = crate::app::gating_test_env::shared_gating_ecstore().await;
+        let context = crate::app::gating_test_env::shared_gating_ambient().await;
+        let bucket = format!("restore-typed-errors-{}", Uuid::new_v4().simple());
+        store
+            .make_bucket(&bucket, &MakeBucketOptions::default())
+            .await
+            .expect("create restore test bucket");
+        let mut reader = PutObjReader::from_vec(b"never transitioned".to_vec());
+        store
+            .put_object(&bucket, "local-object", &mut reader, &ObjectOptions::default())
+            .await
+            .expect("put untransitioned test object");
+
+        let usecase = DefaultObjectUsecase::with_context(Some(context));
+
+        // An illegal `Days` is a client error, rejected before any lock or
+        // object read.
+        let err = usecase
+            .execute_restore_object(build_request(
+                restore_input(&bucket, "local-object", restore_request(Some(0))),
+                Method::POST,
+            ))
+            .await
+            .expect_err("days=0 must be rejected");
+        assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
+
+        // A malformed version-id keeps InvalidArgument instead of being
+        // flattened inside `post_restore_opts`.
+        let mut input = restore_input(&bucket, "local-object", restore_request(Some(1)));
+        input.version_id = Some("not-a-uuid".to_string());
+        let err = usecase
+            .execute_restore_object(build_request(input, Method::POST))
+            .await
+            .expect_err("malformed version-id must be rejected");
+        assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
+
+        // A missing key stays NoSuchKey.
+        let err = usecase
+            .execute_restore_object(build_request(
+                restore_input(&bucket, "missing-object", restore_request(Some(1))),
+                Method::POST,
+            ))
+            .await
+            .expect_err("missing key must be rejected");
+        assert_eq!(err.code(), &S3ErrorCode::NoSuchKey);
+
+        // Restoring an object that was never transitioned is the S3
+        // InvalidObjectState case, not an internal error.
+        let err = usecase
+            .execute_restore_object(build_request(
+                restore_input(&bucket, "local-object", restore_request(Some(1))),
+                Method::POST,
+            ))
+            .await
+            .expect_err("untransitioned object must be rejected");
+        assert_eq!(err.code(), &S3ErrorCode::InvalidObjectState);
    }

    #[tokio::test]
@@ -368,7 +368,8 @@ pub(crate) mod bucket {
                version_id: &str,
                bucket: &str,
                object: &str,
-            ) -> Result<crate::storage::storage_api::StorageObjectOptions, std::io::Error> {
+            ) -> Result<crate::storage::storage_api::StorageObjectOptions, crate::storage::storage_api::StorageError>
+            {
                crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts(
                    version_id, bucket, object,
                )
@@ -14,6 +14,8 @@

 #[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
 use std::alloc::{GlobalAlloc, Layout};
+#[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
+use std::ptr::NonNull;

 #[cfg(all(feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
 #[derive(Default)]
@@ -35,8 +37,10 @@ unsafe impl GlobalAlloc for MiMallocAllocator {
    }

    unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
-        // SAFETY: ptr came from this allocator and layout.size() is the original allocation size.
-        unsafe { rustfs_mimalloc::MiMalloc::free_csize(ptr, layout.size()) }
+        // SAFETY: ptr came from this allocator, is non-null by GlobalAlloc's
+        // dealloc contract, and layout.size() is the original allocation size.
+        let ptr = unsafe { NonNull::new_unchecked(ptr) };
+        unsafe { rustfs_mimalloc::MiMalloc::free_csize_nonnull(ptr, layout.size()) }
    }

    unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
@@ -14,8 +14,6 @@

 use const_str::concat;
 use shadow_rs::shadow;
-use std::path::Path;
-use std::process::Command;

 shadow!(build);

@@ -47,10 +45,6 @@ pub const DISPLAY_VERSION: &str = {

 type VersionParseResult = Result<(u32, u32, u32, Option<String>), Box<dyn std::error::Error>>;

-fn build_version_override() -> Option<&'static str> {
-    BUILD_VERSION_OVERRIDE.filter(|version| !version.is_empty())
-}
-
 fn version_ref(version: &str) -> String {
    if version.starts_with("refs/tags/") || version.starts_with('@') {
        version.to_string()
@@ -61,91 +55,7 @@ fn version_ref(version: &str) -> String {

 #[allow(clippy::const_is_empty)]
 pub fn get_version() -> String {
-    if let Some(version) = build_version_override() {
-        return version_ref(version);
-    }
-
-    // Get the latest tag
-    if let Ok(latest_tag) = get_latest_tag() {
-        // Check if current commit is newer than the latest tag
-        if is_head_newer_than_tag(&latest_tag) {
-            // If current commit is newer, increment the version number
-            if let Ok(new_version) = increment_version(&latest_tag) {
-                return format!("refs/tags/{new_version}");
-            }
-        }
-
-        // If current commit is the latest tag, or version increment failed, return current tag
-        return format!("refs/tags/{latest_tag}");
-    }
-
-    // If no tag exists, use original logic
-    if !build::TAG.is_empty() {
-        format!("refs/tags/{}", build::TAG)
-    } else if !build::SHORT_COMMIT.is_empty() {
-        format!("@{}", build::SHORT_COMMIT)
-    } else {
-        format!("refs/tags/{}", build::PKG_VERSION)
-    }
-}
-
-/// Get the latest git tag
-fn get_latest_tag() -> Result<String, Box<dyn std::error::Error>> {
-    let output = Command::new("git").args(["describe", "--tags", "--abbrev=0"]).output()?;
-
-    if output.status.success() {
-        let tag = String::from_utf8(output.stdout)?;
-        Ok(tag.trim().to_string())
-    } else {
-        Err("Failed to get latest tag".into())
-    }
-}
-
-/// Check if current HEAD is newer than specified tag
-fn is_head_newer_than_tag(tag: &str) -> bool {
-    is_head_newer_than_tag_in(Path::new("."), tag)
-}
-
-fn is_head_newer_than_tag_in(repo: &Path, tag: &str) -> bool {
-    let head = Command::new("git").current_dir(repo).args(["rev-parse", "HEAD"]).output();
-    let tag_commit = Command::new("git")
-        .current_dir(repo)
-        .args(["rev-list", "-n", "1", tag])
-        .output();
-
-    let (Ok(head), Ok(tag_commit)) = (head, tag_commit) else {
-        return false;
-    };
-
-    if !head.status.success() || !tag_commit.status.success() || head.stdout == tag_commit.stdout {
-        return false;
-    }
-
-    let output = Command::new("git")
-        .current_dir(repo)
-        .args(["merge-base", "--is-ancestor", tag, "HEAD"])
-        .output();
-
-    match output {
-        Ok(result) => result.status.success(),
-        Err(_) => false,
-    }
-}
-
-/// Increment version number (increase patch version)
-fn increment_version(version: &str) -> Result<String, Box<dyn std::error::Error>> {
-    // Parse version number, e.g. "1.0.0-alpha.19" -> (1, 0, 0, Some("alpha.19"))
-    let (major, minor, patch, pre_release) = parse_version(version)?;
-
-    // If there's a pre-release identifier, increment the pre-release version number
-    if let Some(pre) = pre_release
-        && let Some(new_pre) = increment_pre_release(&pre)
-    {
-        return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
-    }
-
-    // Otherwise increment patch version number
-    Ok(format!("{major}.{minor}.{}", patch + 1))
+    version_ref(DISPLAY_VERSION)
 }

 /// Parse version number
@@ -166,28 +76,6 @@ pub fn parse_version(version: &str) -> VersionParseResult {
    Ok((major, minor, patch, pre_release))
 }

-/// Increment pre-release version number
-fn increment_pre_release(pre_release: &str) -> Option<String> {
-    // Handle pre-release versions like "alpha.19"
-    let parts: Vec<&str> = pre_release.split('.').collect();
-    if parts.len() == 2
-        && let Ok(num) = parts[1].parse::<u32>()
-    {
-        return Some(format!("{}.{}", parts[0], num + 1));
-    }
-
-    // Handle pre-release versions like "alpha19"
-    if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {
-        let prefix = &pre_release[..=pos];
-        let suffix = &pre_release[pos + 1..];
-        if let Ok(num) = suffix.parse::<u32>() {
-            return Some(format!("{prefix}{}", num + 1));
-        }
-    }
-
-    None
-}
-
 /// Clean version string - removes common prefixes
 pub fn clean_version(version: &str) -> String {
    version
@@ -284,34 +172,6 @@ mod tests {
    use super::*;
    use tracing::debug;

-    fn run_git(repo: &Path, args: &[&str]) {
-        let status = Command::new("git").current_dir(repo).args(args).status().unwrap();
-        assert!(status.success(), "git command failed: git {}", args.join(" "));
-    }
-
-    #[test]
-    fn test_is_head_newer_than_tag_requires_strict_descendant() {
-        let repo = tempfile::tempdir().unwrap();
-        run_git(repo.path(), &["init", "--quiet"]);
-        run_git(repo.path(), &["config", "user.name", "RustFS Tests"]);
-        run_git(repo.path(), &["config", "user.email", "rustfs@example.com"]);
-        run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "tagged commit"]);
-        run_git(repo.path(), &["tag", "--annotate", "1.2.3", "--message", "1.2.3"]);
-
-        assert!(!is_head_newer_than_tag_in(repo.path(), "1.2.3"));
-
-        run_git(repo.path(), &["commit", "--allow-empty", "--quiet", "-m", "newer commit"]);
-
-        assert!(is_head_newer_than_tag_in(repo.path(), "1.2.3"));
-    }
-
-    #[test]
-    fn build_version_override_is_used_for_current_version_when_set() {
-        if let Some(version) = build_version_override() {
-            assert_eq!(get_version(), version_ref(version));
-        }
-    }
-
    #[test]
    fn version_ref_keeps_existing_ref_prefixes() {
        assert_eq!(version_ref("1.2.3"), "refs/tags/1.2.3");
@@ -319,6 +179,11 @@ mod tests {
        assert_eq!(version_ref("@abc123"), "@abc123");
    }

+    #[test]
+    fn get_version_uses_build_metadata() {
+        assert_eq!(get_version(), version_ref(DISPLAY_VERSION));
+    }
+
    #[test]
    fn test_parse_version() {
        // Test standard version parsing
@@ -336,27 +201,6 @@ mod tests {
        assert_eq!(pre_release, Some("alpha.19".to_string()));
    }

-    #[test]
-    fn test_increment_pre_release() {
-        // Test alpha.19 -> alpha.20
-        assert_eq!(increment_pre_release("alpha.19"), Some("alpha.20".to_string()));
-
-        // Test beta.5 -> beta.6
-        assert_eq!(increment_pre_release("beta.5"), Some("beta.6".to_string()));
-
-        // Test unparsable case
-        assert_eq!(increment_pre_release("unknown"), None);
-    }
-
-    #[test]
-    fn test_increment_version() {
-        // Test pre-release version increment
-        assert_eq!(increment_version("1.0.0-alpha.19").unwrap(), "1.0.0-alpha.20");
-
-        // Test standard version increment
-        assert_eq!(increment_version("1.0.0").unwrap(), "1.0.1");
-    }
-
    #[test]
    fn test_version_format() {
        // Test if version format starts with refs/tags/
Author	SHA1	Message	Date
Cursor Agent	f597cdf505	test(e2e): require quota admission on over-limit 4x4 PUT Do not treat SlowDown or AccessDenied as a hard-quota rejection. Match InvalidRequest/QuotaExceeded only with a Bucket quota exceeded message, matching the single-node quota e2e contract. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 16:22:07 +00:00
Cursor Agent	58c350b615	test(e2e): tighten movement refusal and cluster start fail-fast Treat only product strings, 501, and opaque 500 as a refused move. Fail cluster start promptly when a node exits, and overlap GETs with peer kill. Drop the unused TwoPoolFourDrive live-start layout. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 16:16:41 +00:00
Cursor Agent	431011f592	test(e2e): observe 4x4 movement refusals without production changes Keep the distributed lane on 4x4 single-pool layouts. Classify decommission/rebalance product refusals in tests only, and do not change ecstore pool-meta bootstrap. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 16:04:25 +00:00
Cursor Agent	5a3b3a86a2	merge(main): sync distributed e2e branch with origin/main Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 15:59:21 +00:00
Cursor Agent	ff632794a4	test(e2e): observe pool-meta expand refusal without changing production Two-pool restart and decommission attempts stay on current server behavior. Appending pools and restarting is asserted to fail closed on pool-meta recovery. Cluster wait loops fail fast if a node process exits. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 15:48:48 +00:00
Cursor Agent	7f15ac86c3	revert(ecstore): drop pool-meta bootstrap change from test PR Distributed e2e must observe current server behavior, including the localhost pool.bin write fence. Restore combine_across_pools to the equality merge on origin/main. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 15:35:14 +00:00
Henry Guo	923bde6904	feat(scanner): prepare scoped bucket cache scans (#7136 ) Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>	2026-09-04 15:09:58 +00:00
cui fliter	10ccf7c31a	fix(version): avoid deriving RustFS version from runtime working directory (#7118 ) Signed-off-by: cuishuang <imcusg@gmail.com>	2026-09-04 23:03:11 +08:00
RustFS	f878a53e80	Merge branch 'main' into cursor/distributed-e2e-actions-cf91	2026-09-04 22:36:27 +08:00
Cursor Agent	e4dcc21206	test(e2e): add 4-node upgrade coverage for history and IAM Seed a 4-node cluster on the pinned previous release, then prove direct and rolling upgrades keep historical objects and IAM AK/SK working. The distributed Actions lane now downloads that binary. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 14:18:12 +00:00
Cursor Agent	04c70231ab	test(e2e): assert 4-node list-buckets agreement without recreate Delete-then-recreate races with peer bucket cache (NoSuchBucket). Cross-node list and GET of a live object is the distributed property this case can assert reliably. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 14:06:30 +00:00
Cursor Agent	cc8fba3a91	test(e2e): accept 501 on optional observability probes Allow NotImplemented on log/search, wait for recreated buckets to be visible on every node, and treat NoSuchBucket as a gone object. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 14:04:18 +00:00
Cursor Agent	c3f02346bf	test(e2e): make distributed movement tests survive pool-meta fence Treat decommission/rebalance 5xx as the localhost pool.bin write fence, start four-pool clusters via expand so S3 PUTs succeed, and drop the 4-node volume-proxy case that cannot format through FaultProxy. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 14:01:04 +00:00
Cursor Agent	0d7f907e9f	test(e2e): harden 4-node distributed Actions coverage Classify localhost DistErasure pool-meta write fences instead of failing the suite when decommission/rebalance POST is blocked, use the proven 2x2 volume-proxy topology, and add peer-kill GET plus bucket recreate cases. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 13:51:06 +00:00
Zhengchao An	146003a426	refactor(odm): separate source backends from namespace mapping (#7129 ) refactor(odm): isolate read-only source backend contract	2026-09-04 21:40:07 +08:00
Cursor Agent	c2c8d016db	fix(ecstore): keep multi-pool fresh bootstrap proof Combining a Fresh format-load proof with None from a peer-formatted pool was collapsing to no authority, so localhost multi-pool clusters never wrote pool.bin. Treat None as no opinion. Also stabilize the 4-node quota and volume-proxy e2e cases. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 13:13:51 +00:00
RustFS	ab05d958d8	Merge branch 'main' into cursor/distributed-e2e-actions-cf91	2026-09-04 20:45:29 +08:00
cxymds	65ed86f76e	test(ecstore): run checkpoint publication test on large stack (#7134 )	2026-09-04 12:39:18 +00:00
Cursor Agent	47cf4272ec	test(e2e): versioned WORM deletes on 4-node object lock Object Lock plus versioning allows unversioned DELETE as a delete marker; WORM must be asserted against the retained version id. Also pin localhost proxy bypass for cluster processes and a longer nextest slow-timeout for decommission polls. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 12:38:57 +00:00
cxymds	3da3bcd9ee	fix(tier): preserve credentials during backend validation (#7127 ) * fix(tier): preserve credentials during backend validation * test(tier): make clone redaction assertion clippy-clean	2026-09-04 12:24:02 +00:00
Zhengchao An	20e4fd7de6	feat(ecstore): add the sealed remote credential seam (#7137 ) * feat(ecstore): add the sealed remote credential seam Replication targets, remote tiers and on-demand migration sources will all seal their stored secrets through one envelope rather than three (rustfs/backlog#2168, design in docs/architecture/remote-credential-sealing-adr.md). Adds the versioned envelope, the seal scope that binds a ciphertext to the store, owner and field it belongs to, the sealer registration point, and the fail-closed error type. ECStore still has no rustfs-kms dependency: the binary installs a sealer the way it installs the event dispatch hook. Nothing is wired to a consumer yet, so no stored format changes. * docs(ecstore): name the event dispatch hook by module, not by symbol The architecture guard keeps EVENT_DISPATCH_HOOK references inside the event-notification owner module; the module doc cited the symbol only as an example of the hook shape, so cite its file instead.	2026-09-04 12:11:49 +00:00
Cursor Agent	a4795e6b0c	test(e2e): add 4-node 4-disk distributed Actions suite Add a nightly e2e-distributed lane that boots localhost 4-node clusters and covers S3, object lock, versioning, replication, quota, expand, decommission, rebalance, site replication, concurrency, and chaos. Co-authored-by: RustFS <hello@rustfs.com>	2026-09-04 12:10:32 +00:00
houseme	da28f8c843	chore(deps): refresh hotpath, mimalloc, and kafka (#7135 ) * chore(deps): refresh hotpath and mimalloc Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * chore(deps): bump rustfs kafka crates Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: zhi22915 <qiuzgang@gmail.com>	2026-09-04 12:10:16 +00:00
Zhengchao An	507447da12	fix(restore): reject SELECT restore and keep typed S3 errors (#7113 ) * fix(restore): reject SELECT restore and keep typed S3 errors RestoreObject accepted `Type=SELECT` requests, but the restore path can only write the retrieved bytes back to the source key: `put_restore_opts` built SELECT output options and `restore_transitioned_object` then PUT them over the source bucket/object. On an unversioned bucket that dropped `x-amz-restore`, user metadata and tags from the live object; on a versioned bucket it published a bogus latest version. Nothing was ever written to `OutputLocation.S3`, yet the response still carried a fabricated `x-amz-restore-output-path`. Reject SELECT at the API boundary with a typed NotImplemented, before any guard or metadata write, and fail closed in `put_restore_opts` as the backstop for any other caller. Every other RestoreObject failure was collapsed into a `Custom` error code, which serializes as a generic retryable 500: a missing key or version, a malformed version-id, an object that was never transitioned, an illegal `Days`, and authorization or storage failures all looked the same to a client. Map them to their S3 identities instead — NoSuchKey, NoSuchVersion, InvalidArgument, InvalidObjectState, InvalidRequest, MalformedXML — by preserving `StorageError` through `post_restore_opts` and letting `ApiError` do the mapping. The intentional 409 RestoreAlreadyInProgress and 503 SlowDown behaviour is unchanged, and request validation now runs before any lock is taken. backlog#1341, backlog#2205 * test(restore): give the typed-error regression the ecstore test stack `execute_restore_object_maps_failures_to_typed_s3_errors` builds a real ECStore fixture, and under nextest each test runs in a spawned thread with libtest's 2 MiB stack. On Linux CI that overflowed: the test aborted with SIGABRT / "fatal runtime error: stack overflow" while every other test in the run passed. Add it to the `ecstore-base-stack` filter in both the default and ci profiles, alongside the other `package(rustfs)` tests that drive the same store fixture. 4 MiB matches what the deeper multipart and access roundtrips already use.	2026-09-04 12:04:28 +00:00