test(e2e): require quota admission on over-limit 4x4 PUT

Do not treat SlowDown or AccessDenied as a hard-quota rejection.
Match InvalidRequest/QuotaExceeded only with a Bucket quota exceeded
message, matching the single-node quota e2e contract.

Co-authored-by: RustFS <hello@rustfs.com>

.config/e2e-distributed-selection.txt

@@ -0,0 +1,2 @@
+sha256-linux=4988bad7f5929152e0744f07393bc5d24aba2726e5daafa0eeca9a8c6a1f5683
+sha256-darwin=4988bad7f5929152e0744f07393bc5d24aba2726e5daafa0eeca9a8c6a1f5683

.config/nextest.toml

@@ -183,6 +183,13 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'
 
+# 4-node 4-drive distributed Actions suite: each case starts four rustfs
+# processes and up to sixteen data directories. Serialize across nextest's
+# process boundary so several 4x4 clusters never overlap.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^distributed::/)'
+test-group = 'e2e-cluster-nightly'
+
 # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
 # does not cross nextest process boundaries, so keep every Vault-backed test in
 # one group.
@@ -526,6 +533,26 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'
 
+# ---------------------------------------------------------------------------
+# e2e-distributed profile — 4-node 4-disk Actions suite
+# ---------------------------------------------------------------------------
+# Nightly / dispatch lane owned by .github/workflows/e2e-distributed.yml.
+# Each case starts four rustfs processes (and for site replication, two
+# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
+# Serialized via e2e-cluster-nightly. Not a PR merge gate.
+[profile.e2e-distributed]
+default-filter = 'package(e2e_test) & test(/^distributed::/)'
+fail-fast = false
+# Decommission / rebalance cases poll for up to 180s with little stdout.
+slow-timeout = { period = "120s", terminate-after = 6 }
+
+[profile.e2e-distributed.junit]
+path = "junit.xml"
+
+[[profile.e2e-distributed.overrides]]
+filter = 'package(e2e_test)'
+test-group = 'e2e-cluster-nightly'
+
 # ---------------------------------------------------------------------------
 # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
 # ---------------------------------------------------------------------------
@@ -586,6 +613,10 @@ path = "junit.xml"
 #     cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
 #     excluded here because backlog#2213 promotes core heal rebuild coverage to
 #     this merge/main lane while retaining nightly coverage.
+#   * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
+#     replication, quota, observability, expand/decommission/rebalance, site
+#     replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
+#     .github/workflows/e2e-distributed.yml.
 #   * on_demand_migration::interop_test — the ODM-20 provider interoperability
 #     cases, which are meaningless without a source: they run in the dedicated
 #     [profile.e2e-odm-interop] lane below, where the workflow points them at a
@@ -607,6 +638,7 @@ default-filter = """
     package(e2e_test)
     & !test(/^protocols::/)
     & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^distributed::/)
     & !test(/^replication_extension_test::/)
     & !test(/^replication_target_matrix_test::/)
     & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)

.github/scheduled-validations.json

@@ -4,6 +4,11 @@
   { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  {
+    "workflow": ".github/workflows/e2e-distributed.yml",
+    "max_age_hours": 36,
+    "never_ran_grace_until": "2026-09-18T00:00:00Z"
+  },
   { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
   { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },

.github/workflows/e2e-distributed.yml

@@ -0,0 +1,140 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 4-node 4-disk distributed e2e lane.
+#
+# Each selected test starts a real localhost cluster via
+# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
+# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. This is not a required
+# merge check: it is the scheduled/dispatch counterpart to the hardware
+# functional chain that currently clones rustfs/auto-testing onto three VMs.
+# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
+
+name: e2e-distributed
+
+on:
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
+        required: false
+        default: ""
+  schedule:
+    # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
+    - cron: "53 5 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
+jobs:
+  distributed:
+    name: Distributed 4-node 4-disk e2e
+    runs-on: sm-standard-4
+    timeout-minutes: 180
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      NO_PROXY: 127.0.0.1,localhost
+      HTTP_PROXY: ""
+      HTTPS_PROXY: ""
+      # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
+      UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
+      UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
+      UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-e2e-distributed
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
+
+      - name: Download pinned previous release
+        env:
+          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
+        run: |
+          set -euo pipefail
+          mkdir -p "$SOURCE_DIR"
+          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
+          curl --fail --location --retry 3 --output "$archive" \
+            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
+          unzip -q "$archive" -d "$SOURCE_DIR"
+          chmod +x "$SOURCE_DIR/rustfs"
+          test -x "$SOURCE_DIR/rustfs"
+          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
+
+      - name: Build rustfs binary
+        run: |
+          cargo build -p rustfs --bins
+          : > target/debug/rustfs.features
+
+      - name: Verify distributed e2e membership
+        env:
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+        run: |
+          cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
+
+      - name: Run distributed 4-node e2e suite
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
+          NEXTEST_FILTER: ${{ github.event.inputs.filter }}
+        run: |
+          set -euo pipefail
+          if [ -n "${NEXTEST_FILTER}" ]; then
+            cargo nextest run --profile e2e-distributed -p e2e_test -E "${NEXTEST_FILTER}" --no-tests=fail
+          else
+            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+          fi
+
+      - name: Upload distributed e2e diagnostics
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-distributed-${{ github.run_number }}
+          path: |
+            target/nextest/e2e-distributed/junit.xml
+            ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+            ${{ runner.temp }}/rustfs-e2e-distributed-logs/
+          retention-days: 7
+          if-no-files-found: warn
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [distributed]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/rustfs-heal-test.yml

@@ -152,60 +152,6 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          STEPS_TABLE="/tmp/rustfs-heal-steps.md"
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          step_re = re.compile(r'^\[HEAL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
-          ver_re = re.compile(r'^\[HEAL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
-          result_re = re.compile(r'^\[HEAL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
-
-          steps = {}
-          order = []
-          version = None
-          version_node = None
-          verdict = None
-          verdict_detail = ''
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = step_re.match(line)
-                      if m:
-                          n, desc, status = m.group(1), m.group(2), m.group(3)
-                          if n not in steps:
-                              order.append(n)
-                          steps[n] = (desc, status)  # later lines win (fail after pass)
-                          continue
-                      m = ver_re.match(line)
-                      if m:
-                          version, version_node = m.group(1), m.group(2)
-                          continue
-                      m = result_re.match(line)
-                      if m:
-                          verdict, verdict_detail = m.group(1), m.group(2)
-          except FileNotFoundError:
-              pass
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Step Results\n\n')
-              if version:
-                  node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
-                  out.write(f'- Version under test: **{version}**{node_note}\n')
-              if verdict:
-                  out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
-              out.write('\n')
-              out.write('| Step | Description | Result |\n')
-              out.write('| --- | --- | --- |\n')
-              for n in sorted(order, key=int):
-                  desc, status = steps[n]
-                  out.write(f'| {n} | {desc} | {status} |\n')
-              if not order:
-                  out.write('| - | - | NOT RUN (no step result lines found) |\n')
-          PY
           {
             echo "# RustFS heal test report"
             echo ""
@@ -214,8 +160,6 @@ jobs:
             echo "- Package: ${PACKAGE_SOURCE}"
             echo "- Test Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${STEPS_TABLE}" || true
-            echo ""
             echo "## Log tail"
             echo '```text'
             tail -n 200 "${LOG_FILE}" || true

.github/workflows/rustfs-pool-expand-test.yml

@@ -380,60 +380,6 @@ jobs:
           else
             PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
-          STEPS_TABLE="${POOL_ARTIFACT_DIR}/pool-steps.md"
-          python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
-          import re
-          import sys
-
-          log_file, out_file = sys.argv[1], sys.argv[2]
-          ansi = re.compile(r'\x1b\[[0-9;]*m')
-          step_re = re.compile(r'^\[POOL-STEP\]\s+(\d+)\s+(.+?)\s+(PASS|FAIL|SKIP)\s*$')
-          ver_re = re.compile(r'^\[POOL-VERSION\]\s+(\S+)(?:\s+\(node\s+(\S+)\))?\s*$')
-          result_re = re.compile(r'^\[POOL-RESULT\]\s+(PASS|FAIL)\s+(.*)$')
-
-          steps = {}
-          order = []
-          version = None
-          version_node = None
-          verdict = None
-          verdict_detail = ''
-          try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = step_re.match(line)
-                      if m:
-                          n, desc, status = m.group(1), m.group(2), m.group(3)
-                          if n not in steps:
-                              order.append(n)
-                          steps[n] = (desc, status)  # later lines win (fail after pass)
-                          continue
-                      m = ver_re.match(line)
-                      if m:
-                          version, version_node = m.group(1), m.group(2)
-                          continue
-                      m = result_re.match(line)
-                      if m:
-                          verdict, verdict_detail = m.group(1), m.group(2)
-          except FileNotFoundError:
-              pass
-
-          with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Step Results\n\n')
-              if version:
-                  node_note = f' (captured via `rustfs --version` on {version_node})' if version_node else ''
-                  out.write(f'- Version under test: **{version}**{node_note}\n')
-              if verdict:
-                  out.write(f'- Overall result: **{verdict}** — {verdict_detail}\n')
-              out.write('\n')
-              out.write('| Step | Description | Result |\n')
-              out.write('| --- | --- | --- |\n')
-              for n in sorted(order, key=int):
-                  desc, status = steps[n]
-                  out.write(f'| {n} | {desc} | {status} |\n')
-              if not order:
-                  out.write('| - | - | NOT RUN (no step result lines found) |\n')
-          PY
           {
             echo "# RustFS pool expansion test report"
             echo ""
@@ -443,8 +389,6 @@ jobs:
             echo "- Warp concurrent: ${{ inputs.warp_concurrent || '32' }}"
             echo "- Test Step Outcome: ${{ steps.pool_test.outcome }}"
             echo ""
-            cat "${STEPS_TABLE}" || true
-            echo ""
             echo "## Log tail"
             echo '```text'
             tail -n 200 "${LOG_FILE}" || true

.github/workflows/rustfs-upgrade-test.yml

@@ -18,9 +18,9 @@ on:
   workflow_dispatch:
     inputs:
       from_version:
-        description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)'
+        description: 'OLD RustFS release tag (e.g. 1.0.0-rc.4-preview.1)'
         required: false
-        default: '1.0.0-rc.3'
+        default: '1.0.0-rc.4-preview.1'
       from_url:
         description: 'OLD .deb URL. Overrides from_version.'
         required: false
@@ -203,75 +203,54 @@ jobs:
             TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
           fi
           CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
-          MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
-          python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
+          python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
           import re
           import sys
 
-          log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
+          log_file, out_file = sys.argv[1], sys.argv[2]
           ansi = re.compile(r'\x1b\[[0-9;]*m')
           start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
           done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
-          topo_re = re.compile(
-              r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
 
           rows = []
           index = {}
-          topo_rows = []
           try:
-              with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
-                  for raw in fh:
-                      line = ansi.sub('', raw).strip()
-                      m = topo_re.match(line)
-                      if m:
-                          topo_rows.append(m.groups())
-                          continue
-                      m = start_re.match(line)
-                      if m:
-                          case_id, name = m.group(1), m.group(2)
-                          if case_id not in index:
-                              index[case_id] = len(rows)
-                              rows.append([case_id, name, 'RUNNING'])
-                          continue
-                      m = done_re.match(line)
-                      if m:
-                          status, case_id = m.group(1), m.group(2)
-                          if case_id in index:
-                              rows[index[case_id]][2] = status
-                          else:
-                              rows.append([case_id, case_id, status])
-                              index[case_id] = len(rows) - 1
+            with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
+              for raw in fh:
+                line = ansi.sub('', raw).strip()
+                m = start_re.match(line)
+                if m:
+                  case_id, name = m.group(1), m.group(2)
+                  if case_id not in index:
+                    index[case_id] = len(rows)
+                    rows.append([case_id, name, 'RUNNING'])
+                  continue
+                m = done_re.match(line)
+                if m:
+                  status, case_id = m.group(1), m.group(2)
+                  if case_id in index:
+                    rows[index[case_id]][2] = status
+                  else:
+                    rows.append([case_id, case_id, status])
+                    index[case_id] = len(rows) - 1
           except FileNotFoundError:
-              rows = []
+            rows = []
 
           counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
           for _, _, status in rows:
-              counts[status] = counts.get(status, 0) + 1
+            counts[status] = counts.get(status, 0) + 1
 
           with open(out_file, 'w', encoding='utf-8') as out:
-              out.write('## Case Summary\n\n')
-              out.write(f"- Total: {len(rows)}\\n")
-              out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
-              out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
-              out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
-              out.write('\\n')
-              out.write('| Case | Name | Status |\\n')
-              out.write('| --- | --- | --- |\\n')
-              for case_id, name, status in rows:
-                  out.write(f'| {case_id} | {name} | {status} |\\n')
-
-          # Upgrade matrix: one row per topology/backend with the versions
-          # captured on the nodes (rustfs --version) and the aggregated
-          # result. The dashboard renders this table directly.
-          with open(matrix_file, 'w', encoding='utf-8') as out:
-              out.write('## Upgrade Matrix\n\n')
-              out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
-              out.write('| --- | --- | --- | --- | --- |\n')
-              for topo, backend, old_v, new_v, npass, nfail in topo_rows:
-                  result = 'PASS' if nfail == '0' else 'FAIL'
-                  out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
-              if not topo_rows:
-                  out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
+            out.write('## Case Summary\n\n')
+            out.write(f"- Total: {len(rows)}\\n")
+            out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
+            out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
+            out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
+            out.write('\\n')
+            out.write('| Case | Name | Status |\\n')
+            out.write('| --- | --- | --- |\\n')
+            for case_id, name, status in rows:
+              out.write(f'| {case_id} | {name} | {status} |\\n')
           PY
           {
             echo "# RustFS upgrade compatibility report"
@@ -282,8 +261,6 @@ jobs:
             echo "- To: ${TO_SOURCE}"
             echo "- Test Step Outcome: ${{ steps.test.outcome }}"
             echo ""
-            cat "${MATRIX_TABLE}" || true
-            echo ""
             cat "${CASE_TABLE}" || true
             echo ""
             echo "## Log tail"

.github/workflows/scheduled-validation-watchdog.yml

@@ -22,6 +22,7 @@ on:
       - "Continuous Integration"
       - "coverage"
       - "e2e-nightly"
+      - "e2e-distributed"
       - "e2e-s3tests"
       - "Fuzz"
       - "mint"

crates/e2e_test/README.md

@@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
 
@@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker.
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
 | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch; not a merge gate) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
+# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
+# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
+cargo nextest run --profile e2e-distributed -p e2e_test
 # Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane

crates/e2e_test/src/common.rs

@@ -1483,8 +1483,9 @@ impl RustFSTestClusterEnvironment {
             self.spawn_node(node_idx, binary_path, &volumes_arg)?;
         }
 
-        for (i, node) in self.nodes.iter().enumerate() {
-            self.wait_for_node_ready(&node.address, i).await?;
+        for i in 0..self.nodes.len() {
+            let address = self.nodes[i].address.clone();
+            self.wait_for_node_ready(&address, i).await?;
         }
 
         for node_idx in 0..self.nodes.len() {
@@ -1510,7 +1511,8 @@ impl RustFSTestClusterEnvironment {
         let volumes_arg = self.build_volumes_arg();
         self.spawn_node(node_idx, binary_path, &volumes_arg)?;
 
-        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
+        let address = self.nodes[node_idx].address.clone();
+        self.wait_for_node_ready(&address, node_idx).await?;
         self.wait_for_node_service_ready(node_idx).await?;
         Ok(())
     }
@@ -1559,8 +1561,18 @@ impl RustFSTestClusterEnvironment {
     ///
     /// Attempts to establish a TCP connection to the node's address, retries up to 60 times
     /// with a 1-second interval between attempts. Fails if the port is unreachable after all retries.
-    async fn wait_for_node_ready(&self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    fn node_process_exited(&mut self, idx: usize) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+        let Some(process) = self.nodes.get_mut(idx).and_then(|node| node.process.as_mut()) else {
+            return Ok(true);
+        };
+        Ok(process.try_wait()?.is_some())
+    }
+
+    async fn wait_for_node_ready(&mut self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         for attempt in 0..60 {
+            if self.node_process_exited(idx)? {
+                return Err(format!("cluster node {idx} process exited before TCP ready").into());
+            }
             if TcpStream::connect(address).await.is_ok() {
                 info!("Node {} ({}) TCP ready after {} attempts", idx, address, attempt + 1);
                 return Ok(());
@@ -1574,10 +1586,13 @@ impl RustFSTestClusterEnvironment {
     ///
     /// Verifies service availability by calling the S3 `list_buckets` API against the requested node,
     /// retries up to 120 times with a 1-second interval between attempts.
-    async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    async fn wait_for_node_service_ready(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let client = self.create_s3_client(node_idx)?;
 
         for attempt in 0..120 {
+            if self.node_process_exited(node_idx)? {
+                return Err(format!("cluster node {node_idx} process exited before S3 ready").into());
+            }
             match client.list_buckets().send().await {
                 Ok(_) => {
                     info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);
@@ -1700,6 +1715,64 @@ impl RustFSTestClusterEnvironment {
         Ok(())
     }
 
+    /// Append a new single-node erasure pool to a stopped multi-pool cluster.
+    ///
+    /// Used to simulate pool expansion on localhost: every pool already owns
+    /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
+    /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
+    /// allocated a fresh port and empty drive directories; callers must
+    /// [`Self::start`] afterwards so every process picks up the extended
+    /// volumes argument. Existing data directories are left untouched.
+    pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+        if self.nodes.iter().any(|node| node.process.is_some()) {
+            return Err("stop the cluster before appending a pool".into());
+        }
+        if self.topology.drives_per_node < 2 {
+            return Err(
+                "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
+                    .into(),
+            );
+        }
+
+        let mut pools = self.topology.normalized_pools();
+        for (pool_idx, nodes) in pools.iter().enumerate() {
+            if nodes.len() != 1 {
+                return Err(format!(
+                    "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
+                    nodes.len()
+                )
+                .into());
+            }
+        }
+
+        let new_idx = self.nodes.len();
+        let port = RustFSTestEnvironment::find_available_port().await?;
+        let address = format!("127.0.0.1:{port}");
+        let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
+            .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
+            .collect();
+        for dir in &data_dirs {
+            fs::create_dir_all(dir).await?;
+        }
+
+        self.nodes.push(ClusterNode {
+            url: format!("http://{address}"),
+            address,
+            data_dir: data_dirs[0].clone(),
+            data_dirs,
+            pool_idx: pools.len(),
+            process: None,
+        });
+        pools.push(vec![new_idx]);
+        self.topology.node_count = self.nodes.len();
+        self.topology.pools = pools;
+        self.node_extra_env.push(Vec::new());
+        self.node_capture_log_paths.push(None);
+        self.volume_proxy_addresses.push(None);
+
+        Ok(new_idx)
+    }
+
     /// Gracefully stop one cluster node and wait for its process to exit.
     ///
     /// This is intentionally separate from [`Self::stop_node`]: the latter is

crates/e2e_test/src/distributed/chaos_test.rs

@@ -0,0 +1,108 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, bring_drive_online, put_object, retrying_get_equals,
+    take_drive_offline, unique_bucket, wait_for_ready,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn kill_and_restart_node_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("killnode");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x11u8; 128 * 1024];
+    put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
+
+    dist.cluster.stop_node(3)?;
+    retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn full_cluster_restart_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("pwr");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x44u8; 64 * 1024];
+    put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn offline_drive_then_replace_keeps_object_readable() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("baddrive");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x22u8; 96 * 1024];
+    put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
+
+    take_drive_offline(&dist.cluster, 0, 0)?;
+    retrying_get_equals(&dist.client(2)?, &bucket, "durable.bin", &body, Duration::from_secs(20)).await?;
+    bring_drive_online(&dist.cluster, 0, 0)?;
+    retrying_get_equals(&dist.client(3)?, &bucket, "durable.bin", &body, Duration::from_secs(20)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("getkill");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x7Au8; 96 * 1024];
+    put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
+
+    let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
+    let start = Arc::new(Barrier::new(13));
+    let mut handles = Vec::new();
+    for idx in 0..12 {
+        let client = live[idx % live.len()].clone();
+        let bucket = bucket.clone();
+        let body = body.clone();
+        let start = start.clone();
+        handles.push(tokio::spawn(async move {
+            start.wait().await;
+            retrying_get_equals(&client, &bucket, "steady.bin", &body, Duration::from_secs(20)).await
+        }));
+    }
+    start.wait().await;
+    dist.cluster.stop_node(3)?;
+    for handle in handles {
+        handle.await??;
+    }
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/concurrency_stability_test.rs

@@ -0,0 +1,57 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use crate::common::init_logging;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn four_node_high_concurrency_puts_are_readable_from_every_node() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("conc");
+    dist.create_bucket(&bucket).await?;
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(32));
+
+    let mut handles = Vec::new();
+    for idx in 0..32 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("c/{idx:02}.bin");
+            let body = payload_for(&key, 16 * 1024);
+            put_object(client, &bucket, &key, body.clone()).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    let mut inventory = Vec::new();
+    for handle in handles {
+        inventory.push(handle.await??);
+    }
+
+    for (node_idx, client) in clients.iter().enumerate() {
+        for (key, body) in &inventory {
+            assert_object_bytes(client, &bucket, key, body)
+                .await
+                .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
+        }
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/concurrent_data_movement_test.rs

@@ -0,0 +1,67 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, payload_for, put_inventory_retrying,
+    retrying_get_equals, retrying_put, unique_bucket, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("concdecom");
+    dist.create_bucket(&bucket).await?;
+    let baseline_client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&baseline_client, &bucket, 10, 24 * 1024, Duration::from_secs(30)).await?;
+
+    let decommission_started = decommission_started_or_refused(&dist.cluster, 0).await?;
+
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(16));
+    let mut handles = Vec::new();
+    for idx in 0..16 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("live/{idx:02}.bin");
+            let body = payload_for(&key, 8 * 1024);
+            retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    let mut live_objects = Vec::new();
+    for handle in handles {
+        live_objects.push(handle.await??);
+    }
+
+    if decommission_started {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+
+    let checker = dist.client(3)?;
+    assert_inventory(&checker, &bucket, &inventory).await?;
+    for (key, body) in live_objects {
+        retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/data_integrity_movement_test.rs

@@ -0,0 +1,44 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, put_inventory_retrying, sha256_hex,
+    unique_bucket, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn decommission_attempt_does_not_alter_object_sha256() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("integrity");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 20, 64 * 1024, Duration::from_secs(30)).await?;
+    let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
+
+    if decommission_started_or_refused(&dist.cluster, 0).await? {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+
+    let after_client = dist.client(2)?;
+    assert_inventory(&after_client, &bucket, &inventory).await?;
+    for (key, expected_hash) in before {
+        let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
+        let body = got.body.collect().await?.into_bytes();
+        assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs

@@ -0,0 +1,71 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, list_pools_json, put_inventory,
+    rebalance_started_or_refused, unique_bucket, wait_for_decommission_complete, wait_for_rebalance_idle,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_restart_preserves_objects_then_rebalance_attempt() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("expand");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory(&client, &bucket, 12, 32 * 1024).await?;
+    assert_inventory(&client, &bucket, &inventory).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+
+    let after_restart = dist.client(0)?;
+    assert_inventory(&after_restart, &bucket, &inventory).await?;
+    let peer = dist.client(3)?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+
+    if rebalance_started_or_refused(&dist.cluster).await? {
+        wait_for_rebalance_idle(&dist.cluster, Duration::from_secs(90)).await?;
+    }
+    assert_inventory(&peer, &bucket, &inventory).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_decommission_attempt_does_not_lose_objects() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("decom");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(1)?;
+    let inventory = put_inventory(&client, &bucket, 16, 48 * 1024).await?;
+
+    let pools_before = list_pools_json(&dist.cluster).await?;
+    let pool_count = pools_before
+        .as_array()
+        .map(Vec::len)
+        .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
+        .unwrap_or(1);
+    assert!(pool_count >= 1, "expected at least one pool before decommission: {pools_before}");
+
+    if decommission_started_or_refused(&dist.cluster, 0).await? {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+
+    let after = dist.client(3)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/extra_test.rs

@@ -0,0 +1,149 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("extra");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+
+    let key = "multipart.bin";
+    let part1 = vec![0x41u8; 5 * 1024 * 1024];
+    let part2 = vec![0x42u8; 5 * 1024 * 1024];
+    let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
+
+    let uploaded1 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(part1.clone()))
+        .send()
+        .await?;
+    let uploaded2 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(2)
+        .body(ByteStream::from(part2.clone()))
+        .send()
+        .await?;
+
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded1.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded2.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let mut expected = part1;
+    expected.extend_from_slice(&part2);
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
+    }
+
+    put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
+    put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
+    let mut seen = Vec::new();
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let listed = dist
+            .client(node_idx)?
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("list/")
+            .send()
+            .await?;
+        let keys: Vec<String> = listed
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        seen.push(keys);
+    }
+    for keys in &seen[1..] {
+        assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
+    }
+
+    let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
+    assert_eq!(got, b"a");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("listed");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node_idx)?;
+        let name = bucket.clone();
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = client.clone();
+                let name = name.clone();
+                async move {
+                    let listed = client.list_buckets().send().await?;
+                    Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
+                }
+            },
+            &format!("node {node_idx} lists {bucket}"),
+        )
+        .await?;
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = dist.client(node_idx).expect("client");
+                let name = bucket.clone();
+                async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
+            },
+            &format!("node {node_idx} reads seed.bin"),
+        )
+        .await?;
+    }
+    Ok(())
+}

crates/e2e_test/src/distributed/harness.rs

@@ -0,0 +1,933 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Shared 4-node distributed e2e helpers.
+//!
+//! Two localhost-expressible layouts cover the suite:
+//!
+//! * **4×4 single pool** (`four_by_four`) — four processes, four drives each,
+//!   one `DistErasure` pool (16 explicit volume endpoints). This is the
+//!   default S3 / lock / versioning / chaos topology.
+//! * **4×4 four pool** — `append_single_node_pool` exists for harness unit
+//!   tests. Live expand-then-restart currently hits `pool metadata recovery
+//!   required` on localhost DistErasure. That is a production bootstrap-proof
+//!   limitation this test lane does not change. Movement tests use 4×4 single
+//!   pool and classify decommission/rebalance product refusals (and opaque
+//!   500 InternalError) as a refused move while still asserting object bytes.
+//!
+//! Genuine multi-node *striped* pools still need multi-host CI (backlog
+//! #1313 / #1314). Site replication uses two 4-node 1-drive clusters so the
+//! process count stays at eight rather than sixteen.
+
+use crate::common::{
+    ClusterTopology, FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, admin_request, build_test_s3_config,
+    local_http_client, replication_fast_env, signed_request,
+};
+use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV;
+use aws_sdk_s3::Client;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+use http::{Method, StatusCode};
+use sha2::{Digest, Sha256};
+use std::collections::BTreeMap;
+use std::path::Path;
+use std::time::Duration;
+use tokio::time::{Instant, sleep};
+use uuid::Uuid;
+
+pub(crate) type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
+
+pub(crate) const NODE_COUNT: usize = 4;
+pub(crate) const DRIVES_PER_NODE: usize = 4;
+
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum DistLayout {
+    /// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
+    FourByFour,
+    /// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
+    FourNodeFourDisk,
+}
+
+pub(crate) struct DistCluster {
+    pub cluster: RustFSTestClusterEnvironment,
+}
+
+impl DistCluster {
+    pub async fn start(layout: DistLayout) -> TestResult<Self> {
+        Self::start_with_env(layout, &[]).await
+    }
+
+    pub async fn start_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
+        let mut dist = Self::new_stopped_with_env(layout, extra_env).await?;
+        dist.cluster.start().await?;
+        Ok(dist)
+    }
+
+    /// Allocate ports and data dirs without spawning processes.
+    ///
+    /// Upgrade tests configure capture logs, then start a pinned previous
+    /// binary against the same directories.
+    pub async fn new_stopped(layout: DistLayout) -> TestResult<Self> {
+        Self::new_stopped_with_env(layout, &[]).await
+    }
+
+    pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
+        let topology = match layout {
+            DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
+            DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
+        };
+        let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
+        cluster.set_env("NO_PROXY", "127.0.0.1,localhost");
+        cluster.set_env("HTTP_PROXY", "");
+        cluster.set_env("HTTPS_PROXY", "");
+        for &(key, value) in extra_env {
+            cluster.set_env(key, value);
+        }
+        Ok(Self { cluster })
+    }
+
+    /// Start every node with a specific `rustfs` binary, keeping the allocated
+    /// data directories. Used to seed an old on-disk format before upgrading.
+    pub async fn start_from_binary(&mut self, binary: &Path) -> TestResult {
+        self.cluster.start_with_binary(binary).await?;
+        wait_for_ready(&self.cluster).await?;
+        Ok(())
+    }
+
+    /// Stop every node and bring the same data directories up on the workspace
+    /// binary (direct upgrade).
+    pub async fn restart_with_current_binary(&mut self) -> TestResult {
+        self.cluster.stop();
+        self.cluster.start().await?;
+        wait_for_ready(&self.cluster).await?;
+        Ok(())
+    }
+
+    /// Replace one running node with the workspace binary (rolling upgrade).
+    pub async fn replace_node_with_current_binary(&mut self, node_idx: usize) -> TestResult {
+        self.cluster.stop_node(node_idx)?;
+        self.cluster.start_node(node_idx).await?;
+        wait_for_ready(&self.cluster).await?;
+        Ok(())
+    }
+
+    pub fn client_with_credentials(&self, node_idx: usize, access_key: &str, secret_key: &str) -> TestResult<Client> {
+        if node_idx >= self.cluster.nodes.len() {
+            return Err("node_idx is invalid".into());
+        }
+        Ok(Client::from_conf(build_test_s3_config(
+            &self.cluster.nodes[node_idx].url,
+            access_key,
+            secret_key,
+            None,
+            "cluster-iam-test",
+        )))
+    }
+
+    pub async fn start_replication_pair() -> TestResult<(Self, Self)> {
+        let mut extra: Vec<(&str, &str)> = replication_fast_env();
+        extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
+        extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV);
+        let source = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
+        let target = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?;
+        Ok((source, target))
+    }
+
+    pub fn client(&self, node_idx: usize) -> TestResult<Client> {
+        self.cluster.create_s3_client(node_idx)
+    }
+
+    pub fn clients(&self) -> TestResult<Vec<Client>> {
+        self.cluster.create_all_clients()
+    }
+
+    pub async fn create_bucket(&self, bucket: &str) -> TestResult {
+        self.cluster.create_test_bucket(bucket).await
+    }
+}
+
+pub(crate) fn unique_bucket(prefix: &str) -> String {
+    let id = Uuid::new_v4().simple().to_string();
+    format!("{prefix}-{}", &id[..12])
+}
+
+pub(crate) fn sha256_hex(bytes: &[u8]) -> String {
+    let digest = Sha256::digest(bytes);
+    digest.iter().map(|byte| format!("{byte:02x}")).collect()
+}
+
+pub(crate) fn payload_for(key: &str, size: usize) -> Vec<u8> {
+    let seed = key.as_bytes();
+    (0..size)
+        .map(|idx| seed.get(idx % seed.len()).copied().unwrap_or(0) ^ (idx as u8))
+        .collect()
+}
+
+pub(crate) async fn put_object(client: &Client, bucket: &str, key: &str, body: Vec<u8>) -> TestResult {
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from(body))
+        .send()
+        .await?;
+    Ok(())
+}
+
+pub(crate) async fn get_object_bytes(client: &Client, bucket: &str, key: &str) -> TestResult<Vec<u8>> {
+    let output = client.get_object().bucket(bucket).key(key).send().await?;
+    Ok(output.body.collect().await?.into_bytes().to_vec())
+}
+
+pub(crate) async fn assert_object_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
+    let got = get_object_bytes(client, bucket, key).await?;
+    if got.as_slice() != expected {
+        return Err(format!(
+            "object {bucket}/{key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
+            expected.len(),
+            sha256_hex(expected),
+            got.len(),
+            sha256_hex(&got)
+        )
+        .into());
+    }
+    Ok(())
+}
+
+pub(crate) async fn put_inventory(
+    client: &Client,
+    bucket: &str,
+    count: usize,
+    size: usize,
+) -> TestResult<BTreeMap<String, Vec<u8>>> {
+    let mut inventory = BTreeMap::new();
+    for idx in 0..count {
+        let key = format!("obj-{idx:04}");
+        let body = payload_for(&key, size);
+        put_object(client, bucket, &key, body.clone()).await?;
+        inventory.insert(key, body);
+    }
+    Ok(inventory)
+}
+
+/// Localhost DistErasure can 500 a PUT while heal_bucket hits a pool-meta
+/// write fence. Retry only those transient codes.
+pub(crate) async fn put_inventory_retrying(
+    client: &Client,
+    bucket: &str,
+    count: usize,
+    size: usize,
+    timeout: Duration,
+) -> TestResult<BTreeMap<String, Vec<u8>>> {
+    let mut inventory = BTreeMap::new();
+    for idx in 0..count {
+        let key = format!("obj-{idx:04}");
+        let body = payload_for(&key, size);
+        retrying_put(client, bucket, &key, body.clone(), timeout).await?;
+        inventory.insert(key, body);
+    }
+    Ok(inventory)
+}
+
+pub(crate) async fn assert_inventory(client: &Client, bucket: &str, inventory: &BTreeMap<String, Vec<u8>>) -> TestResult {
+    for (key, expected) in inventory {
+        assert_object_bytes(client, bucket, key, expected).await?;
+    }
+    Ok(())
+}
+
+pub(crate) async fn enable_versioning(client: &Client, bucket: &str) -> TestResult {
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+    Ok(())
+}
+
+pub(crate) async fn wait_until<F, Fut>(timeout: Duration, mut probe: F, label: &str) -> TestResult
+where
+    F: FnMut() -> Fut,
+    Fut: std::future::Future<Output = TestResult<bool>>,
+{
+    let deadline = Instant::now() + timeout;
+    let mut delay = Duration::from_millis(50);
+    loop {
+        let last_error = match probe().await {
+            Ok(true) => return Ok(()),
+            Ok(false) => format!("{label} still false"),
+            Err(error) => error.to_string(),
+        };
+        if Instant::now() >= deadline {
+            return Err(format!("{label} did not become true within {timeout:?}: {last_error}").into());
+        }
+        sleep(delay).await;
+        delay = (delay * 2).min(Duration::from_secs(1));
+    }
+}
+
+pub(crate) async fn cluster_admin(
+    cluster: &RustFSTestClusterEnvironment,
+    method: Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> TestResult<(StatusCode, String)> {
+    admin_request(
+        &cluster.nodes[0].url,
+        method,
+        path_and_query,
+        body,
+        &cluster.access_key,
+        &cluster.secret_key,
+    )
+    .await
+}
+
+pub(crate) async fn cluster_admin_ok(
+    cluster: &RustFSTestClusterEnvironment,
+    method: Method,
+    path_and_query: &str,
+    body: Option<String>,
+) -> TestResult<String> {
+    let (status, response) = cluster_admin(cluster, method.clone(), path_and_query, body).await?;
+    if !status.is_success() {
+        return Err(format!("{method} {path_and_query} failed: {status} {response}").into());
+    }
+    Ok(response)
+}
+
+pub(crate) async fn wait_for_ready(cluster: &RustFSTestClusterEnvironment) -> TestResult {
+    let client = local_http_client();
+    for node in &cluster.nodes {
+        let url = format!("{}/health/ready", node.url);
+        wait_until(
+            Duration::from_secs(30),
+            || {
+                let client = client.clone();
+                let url = url.clone();
+                async move {
+                    match client.get(&url).send().await {
+                        Ok(response) if response.status().is_success() => Ok(true),
+                        _ => Ok(false),
+                    }
+                }
+            },
+            &format!("node {} ready", node.address),
+        )
+        .await?;
+    }
+    Ok(())
+}
+
+pub(crate) fn take_drive_offline(
+    cluster: &RustFSTestClusterEnvironment,
+    node_idx: usize,
+    drive_idx: usize,
+) -> TestResult<String> {
+    let dir = cluster
+        .nodes
+        .get(node_idx)
+        .and_then(|node| node.data_dirs.get(drive_idx))
+        .ok_or("invalid node/drive index")?;
+    let offline = format!("{dir}.offline");
+    if Path::new(&offline).exists() {
+        return Err(format!("drive already offline: {offline}").into());
+    }
+    std::fs::rename(dir, &offline)?;
+    Ok(offline)
+}
+
+pub(crate) fn bring_drive_online(cluster: &RustFSTestClusterEnvironment, node_idx: usize, drive_idx: usize) -> TestResult {
+    let dir = cluster
+        .nodes
+        .get(node_idx)
+        .and_then(|node| node.data_dirs.get(drive_idx))
+        .ok_or("invalid node/drive index")?;
+    let offline = format!("{dir}.offline");
+    if Path::new(dir).exists() {
+        std::fs::remove_dir_all(dir)?;
+    }
+    std::fs::rename(&offline, dir)?;
+    Ok(())
+}
+
+pub(crate) async fn set_remote_target(
+    source: &RustFSTestClusterEnvironment,
+    source_bucket: &str,
+    target: &RustFSTestClusterEnvironment,
+    target_bucket: &str,
+) -> TestResult<String> {
+    let body = serde_json::json!({
+        "endpoint": target.nodes[0].address,
+        "credentials": {
+            "accessKey": target.access_key,
+            "secretKey": target.secret_key
+        },
+        "targetbucket": target_bucket,
+        "secure": false,
+        "type": "replication"
+    });
+    let url = format!(
+        "{}/rustfs/admin/v3/set-remote-target?bucket={}",
+        source.nodes[0].url,
+        urlencoding::encode(source_bucket)
+    );
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &source.access_key,
+        &source.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("set remote target failed: {status} {body}").into());
+    }
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+pub(crate) async fn put_bucket_replication(source: &RustFSTestClusterEnvironment, bucket: &str, target_arn: &str) -> TestResult {
+    let body = format!(
+        r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Role></Role>
+  <Rule>
+    <ID>rule-1</ID>
+    <Priority>1</Priority>
+    <Status>Enabled</Status>
+    <DeleteMarkerReplication>
+      <Status>Enabled</Status>
+    </DeleteMarkerReplication>
+    <ExistingObjectReplication>
+      <Status>Enabled</Status>
+    </ExistingObjectReplication>
+    <Destination>
+      <Bucket>{target_arn}</Bucket>
+    </Destination>
+  </Rule>
+</ReplicationConfiguration>"#
+    );
+    let url = format!("{}/{bucket}?replication", source.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &source.access_key,
+        &source.secret_key,
+        Some(body.into_bytes()),
+        Some("application/xml"),
+    )
+    .await?;
+    if !response.status().is_success() {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("put bucket replication failed: {status} {body}").into());
+    }
+    Ok(())
+}
+
+pub(crate) async fn wait_for_replicated_bytes(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    expected: &[u8],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            match get_object_bytes(client, bucket, key).await {
+                Ok(got) if got.as_slice() == expected => Ok(true),
+                Ok(_) => Ok(false),
+                Err(error) => {
+                    let message = error.to_string();
+                    if message.contains("NoSuchKey") || message.contains("NotFound") {
+                        Ok(false)
+                    } else {
+                        Err(error)
+                    }
+                }
+            }
+        },
+        &format!("replicated object {bucket}/{key}"),
+    )
+    .await
+}
+
+pub(crate) async fn set_bucket_quota(cluster: &RustFSTestClusterEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, _) =
+                cluster_admin(cluster, Method::GET, &format!("/rustfs/admin/v3/quota-stats/{bucket}"), None).await?;
+            Ok(status.is_success() || status == StatusCode::NOT_FOUND)
+        },
+        "quota stats ready",
+    )
+    .await?;
+    let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, response) =
+                cluster_admin(cluster, Method::PUT, &format!("/rustfs/admin/v3/quota/{bucket}"), Some(body.clone())).await?;
+            if status.is_success() {
+                return Ok(true);
+            }
+            if status == StatusCode::SERVICE_UNAVAILABLE {
+                return Ok(false);
+            }
+            Err(format!("failed to set quota for {bucket}: {status} {response}").into())
+        },
+        "set hard quota",
+    )
+    .await
+}
+
+/// Localhost DistErasure can boot and serve S3 while refusing pool.bin
+/// mutations (`pool metadata writes remain blocked` / missing fleet
+/// capability proof). Single-pool 4×4 also rejects decommission/rebalance
+/// with a product error. Tests must not pretend a move ran.
+pub(crate) fn is_pool_meta_write_fence(body: &str) -> bool {
+    body.contains("pool metadata writes remain blocked")
+        || body.contains("pool metadata recovery required")
+        || body.contains("pool activation requires a live fleet capability proof")
+        || body.contains("pool activation fleet capability proof expired")
+        || body.contains("live fleet capability proof")
+}
+
+/// Product refusals that movement tests observe. Opaque 500 InternalError stays
+/// in [`classify_data_movement_http`] because admin often wraps the fence as
+/// InternalError XML without the inner string. 502/503 and auth failures are
+/// not refusals.
+pub(crate) fn is_known_data_movement_refusal(body: &str) -> bool {
+    is_pool_meta_write_fence(body)
+        || body.contains("NotImplemented")
+        || body.contains("single pool deployments do not support")
+        || body.contains("at least one active pool must remain")
+}
+
+#[derive(Debug)]
+pub(crate) enum DataMovementStart {
+    Started,
+    Refused(String),
+}
+
+pub(crate) fn classify_data_movement_http(status: StatusCode, body: &str) -> Result<DataMovementStart, String> {
+    if status.is_success() {
+        return Ok(DataMovementStart::Started);
+    }
+    if is_known_data_movement_refusal(body) || status.as_u16() == 501 || status == StatusCode::INTERNAL_SERVER_ERROR {
+        return Ok(DataMovementStart::Refused(format!("{status} {body}")));
+    }
+    Err(format!("{status} {body}"))
+}
+
+pub(crate) async fn try_start_decommission(
+    cluster: &RustFSTestClusterEnvironment,
+    pool_id: usize,
+) -> TestResult<DataMovementStart> {
+    let path = format!("/rustfs/admin/v3/pools/decommission?pool={pool_id}&by-id=true");
+    let (status, response) = cluster_admin(cluster, Method::POST, &path, None).await?;
+    classify_data_movement_http(status, &response).map_err(|detail| format!("POST {path} failed: {detail}").into())
+}
+
+/// Returns whether decommission actually started. A product refusal or opaque
+/// 500 InternalError is not a test failure: callers still assert object bytes.
+pub(crate) async fn decommission_started_or_refused(cluster: &RustFSTestClusterEnvironment, pool_id: usize) -> TestResult<bool> {
+    match try_start_decommission(cluster, pool_id).await? {
+        DataMovementStart::Started => Ok(true),
+        DataMovementStart::Refused(detail) => {
+            eprintln!("decommission POST refused; objects still asserted: {detail}");
+            Ok(false)
+        }
+    }
+}
+
+pub(crate) async fn decommission_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult<serde_json::Value> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/decommission/status", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+fn pool_entry(status: &serde_json::Value, pool_id: usize) -> Option<&serde_json::Value> {
+    if let Some(pools) = status.get("pools").and_then(serde_json::Value::as_array) {
+        return pools
+            .iter()
+            .find(|pool| pool.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64));
+    }
+    if status.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64) {
+        Some(status)
+    } else {
+        None
+    }
+}
+
+fn decommission_pool_failed(pool: &serde_json::Value) -> bool {
+    let info = pool.get("decommissionInfo");
+    let flagged = |key: &str| info.and_then(|value| value.get(key)).and_then(serde_json::Value::as_bool) == Some(true);
+    flagged("failed")
+        || flagged("canceled")
+        || pool
+            .get("status")
+            .and_then(serde_json::Value::as_str)
+            .is_some_and(|status| status.eq_ignore_ascii_case("failed") || status.eq_ignore_ascii_case("canceled"))
+}
+
+pub(crate) fn decommission_complete(status: &serde_json::Value, pool_id: usize) -> bool {
+    let Some(pool) = pool_entry(status, pool_id) else {
+        return false;
+    };
+    if decommission_pool_failed(pool) {
+        return false;
+    }
+    let info_complete = pool
+        .get("decommissionInfo")
+        .and_then(|value| value.get("complete"))
+        .and_then(serde_json::Value::as_bool)
+        == Some(true);
+    let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or("");
+    let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or("");
+    info_complete || status_text.eq_ignore_ascii_case("complete") || pool_status.eq_ignore_ascii_case("decommissioned")
+}
+
+pub(crate) fn decommission_failed(status: &serde_json::Value, pool_id: usize) -> bool {
+    pool_entry(status, pool_id).is_some_and(decommission_pool_failed)
+}
+
+/// `Ok(true)` complete, `Ok(false)` still running, `Err` terminal failure.
+pub(crate) fn decommission_progress(status: &serde_json::Value, pool_id: usize) -> Result<bool, String> {
+    if decommission_failed(status, pool_id) {
+        return Err(format!("decommission failed for pool {pool_id}: {status}"));
+    }
+    Ok(decommission_complete(status, pool_id))
+}
+
+pub(crate) async fn wait_for_decommission_complete(
+    cluster: &RustFSTestClusterEnvironment,
+    pool_id: usize,
+    timeout: Duration,
+) -> TestResult {
+    let deadline = Instant::now() + timeout;
+    let mut delay = Duration::from_millis(50);
+    let mut last_error;
+    loop {
+        last_error = match decommission_status_json(cluster).await {
+            Ok(status) => match decommission_progress(&status, pool_id) {
+                Ok(true) => return Ok(()),
+                Ok(false) => format!("decommission complete still false: {status}"),
+                Err(failed) => return Err(failed.into()),
+            },
+            Err(error) => error.to_string(),
+        };
+        if Instant::now() >= deadline {
+            return Err(format!("decommission complete did not become true within {timeout:?}: {last_error}").into());
+        }
+        sleep(delay).await;
+        delay = (delay * 2).min(Duration::from_secs(1));
+    }
+}
+
+pub(crate) async fn try_start_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<DataMovementStart> {
+    let path = "/rustfs/admin/v3/rebalance/start";
+    let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
+    classify_data_movement_http(status, &response).map_err(|detail| format!("POST {path} failed: {detail}").into())
+}
+
+pub(crate) async fn rebalance_started_or_refused(cluster: &RustFSTestClusterEnvironment) -> TestResult<bool> {
+    match try_start_rebalance(cluster).await? {
+        DataMovementStart::Started => Ok(true),
+        DataMovementStart::Refused(detail) => {
+            eprintln!("rebalance POST refused; objects still asserted: {detail}");
+            Ok(false)
+        }
+    }
+}
+
+pub(crate) async fn rebalance_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult<serde_json::Value> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/rebalance/status", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+pub(crate) fn rebalance_active(status: &serde_json::Value) -> bool {
+    status
+        .get("pools")
+        .and_then(serde_json::Value::as_array)
+        .is_some_and(|pools| {
+            pools.iter().any(|pool| {
+                let stopping = pool.get("stopping").and_then(serde_json::Value::as_bool) == Some(true);
+                let value = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or("");
+                stopping
+                    || value.eq_ignore_ascii_case("started")
+                    || value.eq_ignore_ascii_case("active")
+                    || value.eq_ignore_ascii_case("running")
+                    || value.eq_ignore_ascii_case("stopping")
+            })
+        })
+}
+
+pub(crate) async fn wait_for_rebalance_idle(cluster: &RustFSTestClusterEnvironment, timeout: Duration) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            match rebalance_status_json(cluster).await {
+                Ok(status) => Ok(!rebalance_active(&status)),
+                Err(error) => {
+                    let message = error.to_string();
+                    if message.contains("NoSuchResource") || message.contains("404") || message.contains("not started") {
+                        Ok(true)
+                    } else {
+                        Err(error)
+                    }
+                }
+            }
+        },
+        "rebalance idle",
+    )
+    .await
+}
+
+pub(crate) async fn list_pools_json(cluster: &RustFSTestClusterEnvironment) -> TestResult<serde_json::Value> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/pools/list", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+pub(crate) async fn retrying_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, timeout: Duration) -> TestResult {
+    wait_until(
+        timeout,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                match put_object(&client, &bucket, &key, body).await {
+                    Ok(()) => Ok(true),
+                    Err(error) => {
+                        let message = error.to_string();
+                        if message.contains("SlowDown")
+                            || message.contains("ServiceUnavailable")
+                            || message.contains("InternalError")
+                            || message.contains("503")
+                            || message.contains("500")
+                        {
+                            Ok(false)
+                        } else {
+                            Err(error)
+                        }
+                    }
+                }
+            }
+        },
+        &format!("put {bucket}/{key} during data movement"),
+    )
+    .await
+}
+
+pub(crate) async fn retrying_get_equals(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    expected: &[u8],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            match get_object_bytes(client, bucket, key).await {
+                Ok(got) if got.as_slice() == expected => Ok(true),
+                Ok(_) => Ok(false),
+                Err(error) => {
+                    let message = error.to_string();
+                    if message.contains("NoSuchKey")
+                        || message.contains("SlowDown")
+                        || message.contains("ServiceUnavailable")
+                        || message.contains("InternalError")
+                        || message.contains("503")
+                        || message.contains("500")
+                    {
+                        Ok(false)
+                    } else {
+                        Err(error)
+                    }
+                }
+            }
+        },
+        &format!("get {bucket}/{key} during data movement"),
+    )
+    .await
+}
+
+#[tokio::test]
+async fn append_single_node_pool_extends_ellipses_volumes() {
+    let mut env =
+        RustFSTestClusterEnvironment::with_topology(ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0], vec![1]]))
+            .await
+            .expect("two-pool seed topology");
+    assert_eq!(env.rustfs_volumes_arg().split(' ').count(), 2);
+
+    let added = env.append_single_node_pool().await.expect("append third pool");
+    assert_eq!(added, 2);
+    assert_eq!(env.nodes.len(), 3);
+    assert_eq!(env.nodes[2].pool_idx, 2);
+    assert_eq!(env.nodes[2].data_dirs.len(), DRIVES_PER_NODE);
+    let volumes = env.rustfs_volumes_arg();
+    assert_eq!(volumes.split(' ').count(), 3, "expected three pool arguments, got: {volumes}");
+    assert!(volumes.contains("/drive{0...3}"), "expanded layout must keep drive ellipses: {volumes}");
+}
+
+#[tokio::test]
+async fn append_single_node_pool_rejects_striped_single_pool() {
+    let mut env = RustFSTestClusterEnvironment::new(4).await.expect("four-node single pool");
+    let err = env
+        .append_single_node_pool()
+        .await
+        .expect_err("a striped single pool cannot gain a localhost pool");
+    let message = err.to_string();
+    assert!(
+        message.contains("drives_per_node") || message.contains("one node per pool"),
+        "unexpected error: {message}"
+    );
+}
+
+#[cfg(unix)]
+#[tokio::test]
+async fn cluster_start_fails_fast_when_node_process_exits() {
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk)
+        .await
+        .expect("stopped 4-node cluster");
+    let script = format!("{}/immediate-exit.sh", dist.cluster.temp_dir);
+    std::fs::write(&script, "#!/bin/sh\nexit 1\n").expect("write exit stub");
+    let mut perms = std::fs::metadata(&script).expect("stat exit stub").permissions();
+    std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o755);
+    std::fs::set_permissions(&script, perms).expect("chmod exit stub");
+
+    let started = Instant::now();
+    let err = dist
+        .start_from_binary(Path::new(&script))
+        .await
+        .expect_err("a node that exits immediately must fail start");
+    let elapsed = started.elapsed();
+    let message = err.to_string();
+    assert!(
+        message.contains("exited before TCP ready") || message.contains("exited before S3 ready"),
+        "unexpected start error: {message}"
+    );
+    assert!(
+        elapsed < Duration::from_secs(30),
+        "cluster start must fail fast when a node exits, took {elapsed:?}"
+    );
+}
+
+#[test]
+fn decommission_complete_reads_pool_status_and_info_flag() {
+    let status = serde_json::json!({
+        "pools": [
+            {
+                "id": 0,
+                "status": "complete",
+                "poolStatus": "decommissioned",
+                "decommissionInfo": { "complete": true, "failed": false, "canceled": false }
+            },
+            { "id": 1, "status": "none", "poolStatus": "active" }
+        ]
+    });
+    assert!(decommission_complete(&status, 0));
+    assert!(!decommission_complete(&status, 1));
+    assert!(!decommission_failed(&status, 0));
+    assert!(decommission_progress(&status, 0).expect("complete pool"));
+    assert!(!decommission_progress(&status, 1).expect("other pool is not complete"));
+}
+
+#[test]
+fn decommission_progress_fails_closed_on_failed_flag() {
+    let failed = serde_json::json!({
+        "pools": [{
+            "id": 0,
+            "status": "failed",
+            "decommissionInfo": { "complete": false, "failed": true, "canceled": false }
+        }]
+    });
+    let err = decommission_progress(&failed, 0).expect_err("failed decommission must not look complete");
+    assert!(err.contains("decommission failed for pool 0"), "{err}");
+    assert!(!decommission_progress(&failed, 1).expect("missing pool is still running"));
+}
+
+#[test]
+fn rebalance_active_treats_started_as_in_progress() {
+    let started = serde_json::json!({ "pools": [{ "id": 0, "status": "Started", "stopping": false }] });
+    let done = serde_json::json!({ "pools": [{ "id": 0, "status": "Completed", "stopping": false }] });
+    assert!(rebalance_active(&started));
+    assert!(!rebalance_active(&done));
+}
+
+#[test]
+fn pool_meta_write_fence_matches_known_product_gates() {
+    assert!(is_pool_meta_write_fence(
+        "heal_bucket: pool metadata writes remain blocked after a recovery-required replica state"
+    ));
+    assert!(is_pool_meta_write_fence(
+        "rebalance meta save failed: pool activation requires a live fleet capability proof"
+    ));
+    assert!(is_pool_meta_write_fence("pool metadata recovery required: no durable bootstrap identity"));
+    assert!(!is_pool_meta_write_fence("NotImplemented: single pool cannot decommission"));
+    assert!(!is_pool_meta_write_fence("AccessDenied"));
+}
+
+#[test]
+fn classify_data_movement_http_observes_product_refusals_not_auth_failures() {
+    assert!(matches!(classify_data_movement_http(StatusCode::OK, ""), Ok(DataMovementStart::Started)));
+    assert!(matches!(
+        classify_data_movement_http(
+            StatusCode::BAD_REQUEST,
+            "failed to start decommission: single pool deployments do not support decommission"
+        ),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(
+            StatusCode::BAD_REQUEST,
+            "failed to start decommission: at least one active pool must remain after decommission start"
+        ),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(StatusCode::NOT_IMPLEMENTED, "NotImplemented"),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "pool metadata writes remain blocked after a recovery-required replica state"
+        ),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    assert!(matches!(
+        classify_data_movement_http(StatusCode::INTERNAL_SERVER_ERROR, "InternalError"),
+        Ok(DataMovementStart::Refused(_))
+    ));
+    let denied = classify_data_movement_http(StatusCode::FORBIDDEN, "AccessDenied").expect_err("auth failure is not a refusal");
+    assert!(denied.contains("AccessDenied"), "{denied}");
+    let unavailable = classify_data_movement_http(StatusCode::SERVICE_UNAVAILABLE, "ServiceUnavailable")
+        .expect_err("503 is not a product refusal");
+    assert!(unavailable.contains("ServiceUnavailable"), "{unavailable}");
+    let bad_gateway =
+        classify_data_movement_http(StatusCode::BAD_GATEWAY, "Bad Gateway").expect_err("502 is not a product refusal");
+    assert!(bad_gateway.contains("502") || bad_gateway.contains("Bad Gateway"), "{bad_gateway}");
+}

crates/e2e_test/src/distributed/mod.rs

@@ -0,0 +1,35 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node 4-drive distributed e2e coverage.
+//!
+//! Selected by `[profile.e2e-distributed]` and run from
+//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
+//! each case starts four real `rustfs` processes.
+
+mod chaos_test;
+mod concurrency_stability_test;
+mod concurrent_data_movement_test;
+mod data_integrity_movement_test;
+mod expand_decommission_rebalance_test;
+mod extra_test;
+mod harness;
+mod object_lock_test;
+mod observability_test;
+mod replication_quota_test;
+mod s3_basic_test;
+mod s3_during_data_movement_test;
+mod site_replication_test;
+mod upgrade_test;
+mod versioning_test;

crates/e2e_test/src/distributed/object_lock_test.rs

@@ -0,0 +1,111 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
+use crate::common::init_logging;
+use crate::object_lock::common::{delete_object_with_bypass, put_object_with_legal_hold, put_object_with_retention};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::operation::delete_object::DeleteObjectError;
+use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
+use chrono::{Duration as ChronoDuration, Utc};
+
+fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
+    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+    if code == Some("AccessDenied") {
+        Ok(())
+    } else {
+        Err(format!("{context}: expected AccessDenied, got {error:?}").into())
+    }
+}
+
+async fn expect_versioned_delete_denied(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    bypass: bool,
+    context: &str,
+) -> TestResult {
+    match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
+        Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
+        Err(error) => delete_denied(error.as_ref(), context),
+    }
+}
+
+#[tokio::test]
+async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let client = dist.client(0)?;
+    let peer = dist.client(2)?;
+    let bucket = unique_bucket("objlock");
+
+    client
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    let retain_until = Utc::now() + ChronoDuration::days(1);
+
+    let compliance_key = "compliance.bin";
+    let compliance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        compliance_key,
+        b"locked-compliance",
+        ObjectLockRetentionMode::Compliance,
+        retain_until,
+    )
+    .await?;
+
+    // Unversioned DELETE is allowed: it only creates a delete marker. WORM
+    // applies to a specific version id.
+    let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
+    assert_eq!(
+        marker.delete_marker(),
+        Some(true),
+        "unversioned DELETE on a locked object must create a delete marker"
+    );
+
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
+        .await?;
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
+
+    let governance_key = "governance.bin";
+    let governance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        governance_key,
+        b"locked-governance",
+        ObjectLockRetentionMode::Governance,
+        retain_until,
+    )
+    .await?;
+
+    expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
+        .await?;
+    delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
+
+    let hold_key = "legal-hold.bin";
+    let hold_version =
+        put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
+
+    Ok(())
+}

crates/e2e_test/src/distributed/observability_test.rs

@@ -0,0 +1,80 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, put_object, unique_bucket, wait_for_ready,
+};
+use crate::common::{init_logging, local_http_client};
+use http::Method;
+
+#[tokio::test]
+async fn four_node_four_drive_health_admin_info_and_audit_list() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    wait_for_ready(&dist.cluster).await?;
+
+    let http = local_http_client();
+    for node in &dist.cluster.nodes {
+        let ready = http.get(format!("{}/health/ready", node.url)).send().await?;
+        assert!(ready.status().is_success(), "node {} not ready: {}", node.address, ready.status());
+        let live = http.get(format!("{}/health/live", node.url)).send().await;
+        if let Ok(response) = live {
+            assert!(
+                response.status().is_success() || response.status().as_u16() == 404,
+                "unexpected live probe on {}: {}",
+                node.address,
+                response.status()
+            );
+        }
+    }
+
+    let info = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    assert!(!info.is_empty(), "admin info was empty");
+    let storage = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    assert!(
+        storage.contains("disks") || storage.contains("backend") || storage.contains("info"),
+        "storageinfo missing expected fields: {storage}"
+    );
+
+    let audit = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
+    let trimmed = audit.trim();
+    if !trimmed.is_empty() && trimmed != "null" && !trimmed.starts_with('[') && !trimmed.starts_with('{') {
+        return Err(format!("audit target list was not machine-readable: {audit}").into());
+    }
+
+    // Optional surfaces: 404/400/501 are acceptable (route missing or stubbed);
+    // unexpected 5xx is not. A 2xx body must be non-empty.
+    for path in [
+        "/rustfs/admin/v3/log/search",
+        "/rustfs/admin/v4/runtime/capabilities",
+        "/minio/v2/metrics/cluster",
+    ] {
+        let (status, body) = cluster_admin(&dist.cluster, Method::GET, path, None).await?;
+        assert!(
+            status.is_success() || status.is_client_error() || status.as_u16() == 501,
+            "observability path {path} returned {status}: {body}"
+        );
+        if status.is_success() {
+            assert!(!body.trim().is_empty(), "empty body from {path}");
+        }
+    }
+
+    let bucket = unique_bucket("obs");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "probe.log", b"observability".to_vec()).await?;
+
+    let trace = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    assert!(!trace.is_empty());
+    Ok(())
+}

crates/e2e_test/src/distributed/replication_quota_test.rs

@@ -0,0 +1,144 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
+    set_remote_target, unique_bucket, wait_for_replicated_bytes, wait_until,
+};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use http::Method;
+use std::time::Duration;
+
+/// `Ok(true)` quota admission rejected the PUT, `Ok(false)` retry, `Err` not quota.
+fn quota_over_limit_put_outcome(code: Option<&str>, message: Option<&str>) -> Result<bool, String> {
+    let quota_message = message.is_some_and(|text| text.starts_with("Bucket quota exceeded"));
+    match code {
+        Some("InvalidRequest" | "QuotaExceeded") if quota_message => Ok(true),
+        Some("SlowDown" | "ServiceUnavailable") => Ok(false),
+        Some("AccessDenied") => Err("AccessDenied is not a quota admission rejection".to_string()),
+        Some("InvalidRequest" | "QuotaExceeded") => {
+            Err(format!("InvalidRequest/QuotaExceeded without quota admission message: {message:?}"))
+        }
+        other => Err(format!("unexpected over-quota error code {other:?} message {message:?}")),
+    }
+}
+
+#[tokio::test]
+async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
+    init_logging();
+    let (source, target) = DistCluster::start_replication_pair().await?;
+    let source_bucket = unique_bucket("replsrc");
+    let target_bucket = unique_bucket("repldst");
+    source.create_bucket(&source_bucket).await?;
+    target.create_bucket(&target_bucket).await?;
+
+    let source_client = source.client(0)?;
+    let target_client = target.client(0)?;
+    enable_versioning(&source_client, &source_bucket).await?;
+    enable_versioning(&target_client, &target_bucket).await?;
+
+    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    let key = "replicated.bin";
+    let body = b"distributed-bucket-replication".to_vec();
+    put_object(&source_client, &source_bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
+
+    let peer_read = target.client(3)?;
+    wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
+    let bucket = unique_bucket("quota");
+    dist.create_bucket(&bucket).await?;
+    set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
+
+    let client = dist.client(1)?;
+    retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, body) = super::harness::cluster_admin(
+                &dist.cluster,
+                Method::GET,
+                &format!("/rustfs/admin/v3/quota-stats/{bucket}"),
+                None,
+            )
+            .await?;
+            if !status.is_success() {
+                return Ok(false);
+            }
+            let stats: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
+            Ok(stats.get("current_usage").and_then(serde_json::Value::as_u64).unwrap_or(0) >= 1024)
+        },
+        "quota stats observe small object",
+    )
+    .await?;
+
+    let mut oversized_attempt = 0u32;
+    wait_until(
+        Duration::from_secs(30),
+        || {
+            oversized_attempt += 1;
+            let key = format!("too-big-{oversized_attempt}.bin");
+            let client = client.clone();
+            let bucket = bucket.clone();
+            async move {
+                match client
+                    .put_object()
+                    .bucket(&bucket)
+                    .key(key)
+                    .body(vec![0u8; 16 * 1024].into())
+                    .send()
+                    .await
+                {
+                    Ok(_) => Ok(false),
+                    Err(error) => {
+                        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+                        let message = error.as_service_error().and_then(ProvideErrorMetadata::message);
+                        match quota_over_limit_put_outcome(code, message) {
+                            Ok(done) => Ok(done),
+                            Err(detail) => Err(format!("{detail}: {error:?}").into()),
+                        }
+                    }
+                }
+            }
+        },
+        "hard quota rejects oversized PUT",
+    )
+    .await?;
+    Ok(())
+}
+
+#[test]
+fn quota_over_limit_put_outcome_requires_quota_admission() {
+    assert_eq!(
+        quota_over_limit_put_outcome(Some("InvalidRequest"), Some("Bucket quota exceeded for bucket x")),
+        Ok(true)
+    );
+    assert_eq!(
+        quota_over_limit_put_outcome(Some("QuotaExceeded"), Some("Bucket quota exceeded")),
+        Ok(true)
+    );
+    assert_eq!(quota_over_limit_put_outcome(Some("SlowDown"), Some("slow down")), Ok(false));
+    assert_eq!(quota_over_limit_put_outcome(Some("ServiceUnavailable"), Some("unavailable")), Ok(false));
+    assert!(quota_over_limit_put_outcome(Some("AccessDenied"), Some("Access Denied")).is_err());
+    assert!(quota_over_limit_put_outcome(Some("InvalidRequest"), Some("invalid argument")).is_err());
+}

crates/e2e_test/src/distributed/s3_basic_test.rs

@@ -0,0 +1,111 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
+use crate::common::{init_logging, local_http_client};
+use aws_sdk_s3::presigning::PresigningConfig;
+use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3basic");
+    dist.create_bucket(&bucket).await?;
+
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let key = "dir/object.bin";
+    let body = vec![0xA5u8; 256 * 1024];
+    put_object(&writer, &bucket, key, body.clone()).await?;
+
+    let head = reader.head_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(head.content_length(), Some(body.len() as i64));
+    assert_object_bytes(&reader, &bucket, key, &body).await?;
+
+    let ranged = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .range("bytes=0-15")
+        .send()
+        .await?;
+    let ranged_body = ranged.body.collect().await?.into_bytes();
+    assert_eq!(ranged_body.as_ref(), &body[..16]);
+
+    let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
+    let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+    assert_eq!(keys, vec![key]);
+
+    let copy_key = "dir/object-copy.bin";
+    reader
+        .copy_object()
+        .bucket(&bucket)
+        .key(copy_key)
+        .copy_source(format!("{bucket}/{key}"))
+        .metadata_directive(MetadataDirective::Copy)
+        .send()
+        .await?;
+    assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
+
+    let moved_key = "dir/object-moved.bin";
+    writer
+        .copy_object()
+        .bucket(&bucket)
+        .key(moved_key)
+        .copy_source(format!("{bucket}/{copy_key}"))
+        .send()
+        .await?;
+    writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
+    match writer.head_object().bucket(&bucket).key(copy_key).send().await {
+        Ok(_) => return Err("copied source still present after rename delete".into()),
+        Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
+        Err(error) => return Err(error.into()),
+    }
+    assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
+
+    let presigned = writer
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
+        .await?;
+    let response = local_http_client().get(presigned.uri().to_string()).send().await?;
+    assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
+    let presigned_body = response.bytes().await?;
+    assert_eq!(presigned_body.as_ref(), body.as_slice());
+
+    let empty_key = "empty";
+    put_object(&writer, &bucket, empty_key, Vec::new()).await?;
+    let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
+    assert!(empty.is_empty());
+
+    writer
+        .delete_objects()
+        .bucket(&bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key(key).build()?)
+                .objects(ObjectIdentifier::builder().key(moved_key).build()?)
+                .objects(ObjectIdentifier::builder().key(empty_key).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+
+    let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
+    Ok(())
+}

crates/e2e_test/src/distributed/s3_during_data_movement_test.rs

@@ -0,0 +1,82 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_refused, put_inventory_retrying,
+    rebalance_started_or_refused, retrying_get_equals, retrying_put, unique_bucket, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3move");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 8, 16 * 1024, Duration::from_secs(30)).await?;
+
+    let decommission_started = decommission_started_or_refused(&dist.cluster, 0).await?;
+    let live = dist.client(2)?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-decommission.bin")),
+        "list during decommission missed the newly written key"
+    );
+
+    if decommission_started {
+        wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
+    }
+    assert_inventory(&live, &bucket, &inventory).await?;
+
+    let _ = rebalance_started_or_refused(&dist.cluster).await?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing",
+        Duration::from_secs(30),
+    )
+    .await?;
+    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/site_replication_test.rs

@@ -0,0 +1,87 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
+};
+use crate::common::{init_logging, signed_request};
+use http::{Method, StatusCode};
+use rustfs_madmin::PeerSite;
+use std::time::Duration;
+
+async fn site_replication_add(cluster: &crate::common::RustFSTestClusterEnvironment, sites: &[PeerSite]) -> TestResult<String> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &cluster.access_key,
+        &cluster.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+    Ok(response.text().await?)
+}
+
+#[tokio::test]
+async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+    let bucket = unique_bucket("siterepl");
+    site_a.create_bucket(&bucket).await?;
+    site_b.create_bucket(&bucket).await?;
+
+    let client_a = site_a.client(0)?;
+    let client_b = site_b.client(0)?;
+    enable_versioning(&client_a, &bucket).await?;
+    enable_versioning(&client_b, &bucket).await?;
+
+    let sites = vec![
+        PeerSite {
+            name: "site-a".to_string(),
+            endpoint: site_a.cluster.nodes[0].url.clone(),
+            access_key: site_a.cluster.access_key.clone(),
+            secret_key: site_a.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+        PeerSite {
+            name: "site-b".to_string(),
+            endpoint: site_b.cluster.nodes[0].url.clone(),
+            access_key: site_b.cluster.access_key.clone(),
+            secret_key: site_b.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+    ];
+    site_replication_add(&site_a.cluster, &sites).await?;
+
+    let info = cluster_admin_ok(&site_a.cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
+    assert!(
+        info.contains("site-a") || info.contains("enabled") || info.contains("true"),
+        "site replication info did not show a configured peer: {info}"
+    );
+
+    let key = "site-object.bin";
+    let body = b"four-node-site-replication".to_vec();
+    put_object(&client_a, &bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
+
+    let peer_b = site_b.client(3)?;
+    wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
+    Ok(())
+}

crates/e2e_test/src/distributed/upgrade_test.rs

@@ -0,0 +1,363 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node upgrade coverage for historical objects and IAM AK/SK.
+//!
+//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
+//! mixed-version listing). This module pins the distributed contract the
+//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
+//! written on the previous release still read back, and IAM user credentials
+//! created before the upgrade still authenticate.
+//!
+//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
+//! release. The `e2e-distributed` workflow downloads that binary; a local run
+//! without it fails closed rather than skipping.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
+    unique_bucket, wait_until,
+};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use std::ffi::OsString;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use uuid::Uuid;
+
+const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const IAM_SECRET: &str = "UpgradeTestSecretKey1";
+const WRONG_SECRET: &str = "WrongSecretKey000000";
+const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct UpgradeSeed {
+    history_bucket: String,
+    history_key: &'static str,
+    history_body: Vec<u8>,
+    versioned_bucket: String,
+    versioned_key: &'static str,
+    version1: String,
+    version1_body: Vec<u8>,
+    version2: String,
+    version2_body: Vec<u8>,
+    iam_bucket: String,
+    iam_key: &'static str,
+    iam_body: Vec<u8>,
+    iam_user: String,
+    iam_secret: &'static str,
+}
+
+fn resolve_source_binary(value: Option<OsString>) -> TestResult<PathBuf> {
+    let path = value.map(PathBuf::from).ok_or_else(|| {
+        format!(
+            "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
+        )
+    })?;
+    if !path.is_file() {
+        return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
+    }
+    Ok(path)
+}
+
+fn source_binary() -> TestResult<PathBuf> {
+    resolve_source_binary(std::env::var_os(SOURCE_BINARY_ENV))
+}
+
+fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
+        cluster
+            .cluster
+            .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+fn iam_rw_policy(bucket: &str) -> String {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}"),
+                format!("arn:aws:s3:::{bucket}/*")
+            ]
+        }]
+    })
+    .to_string()
+}
+
+async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
+    let url = &dist.cluster.nodes[0].url;
+    let access = &dist.cluster.access_key;
+    let admin_secret = &dist.cluster.secret_key;
+    admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
+    admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
+    admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
+    Ok(())
+}
+
+async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                put_object(&client, &bucket, &key, body).await?;
+                Ok(true)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let expected = expected.to_vec();
+            async move {
+                let got = get_object_bytes(&client, &bucket, &key).await?;
+                Ok(got == expected)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
+    let history_bucket = unique_bucket("upg-hist");
+    let versioned_bucket = unique_bucket("upg-ver");
+    let iam_bucket = unique_bucket("upg-iam");
+    dist.create_bucket(&history_bucket).await?;
+    dist.create_bucket(&versioned_bucket).await?;
+    dist.create_bucket(&iam_bucket).await?;
+
+    let root = dist.client(0)?;
+    enable_versioning(&root, &versioned_bucket).await?;
+
+    let history_key = "plain-history.bin";
+    let history_body = b"written by the previous 4-node release".to_vec();
+    put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
+
+    let versioned_key = "versioned-history.txt";
+    let version1_body = b"version-one-before-upgrade".to_vec();
+    let version1 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("first versioned PUT omitted version ID")?
+        .to_string();
+    let version2_body = b"version-two-before-upgrade".to_vec();
+    let version2 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("second versioned PUT omitted version ID")?
+        .to_string();
+
+    let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
+
+    let iam_key = "iam-history.bin";
+    let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
+    let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
+    wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
+
+    Ok(UpgradeSeed {
+        history_bucket,
+        history_key,
+        history_body,
+        versioned_bucket,
+        versioned_key,
+        version1,
+        version1_body,
+        version2,
+        version2_body,
+        iam_bucket,
+        iam_key,
+        iam_body,
+        iam_user,
+        iam_secret: IAM_SECRET,
+    })
+}
+
+async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
+    let root_a = dist.client(0)?;
+    let root_b = dist.client(3)?;
+    wait_for_bytes(
+        &root_b,
+        &seed.history_bucket,
+        seed.history_key,
+        &seed.history_body,
+        &format!("{context}: root GET historical object"),
+    )
+    .await?;
+    assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
+
+    let v1 = root_b
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version1)
+        .send()
+        .await?;
+    let v1_body = v1.body.collect().await?.into_bytes();
+    if v1_body.as_ref() != seed.version1_body.as_slice() {
+        return Err(format!("{context}: version 1 bytes changed after upgrade").into());
+    }
+    let v2 = root_a
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version2)
+        .send()
+        .await?;
+    let v2_body = v2.body.collect().await?.into_bytes();
+    if v2_body.as_ref() != seed.version2_body.as_slice() {
+        return Err(format!("{context}: version 2 bytes changed after upgrade").into());
+    }
+
+    let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    if !users.contains(&seed.iam_user) {
+        return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
+    }
+
+    let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
+    let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
+    wait_for_bytes(
+        &iam_on_upgraded,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 0"),
+    )
+    .await?;
+    wait_for_bytes(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 3"),
+    )
+    .await?;
+
+    let post_key = format!("after-upgrade-{context}.txt");
+    let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
+    wait_for_put(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        &post_key,
+        post_body.clone(),
+        &format!("{context}: IAM PUT after upgrade"),
+    )
+    .await?;
+    assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
+
+    let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
+    match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
+        Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
+        Err(error) => {
+            let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+            if code == Some("SignatureDoesNotMatch")
+                || code == Some("InvalidAccessKeyId")
+                || code == Some("AccessDenied")
+                || code == Some("InvalidArgument")
+            {
+            } else if error.raw_response().is_some_and(|response| response.status().as_u16() == 403) {
+            } else {
+                return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
+            }
+        }
+    }
+
+    let post_root_key = format!("root-after-{context}.bin");
+    let post_root_body = format!("{context}: root write after upgrade").into_bytes();
+    put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
+    assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "direct-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+    dist.restart_with_current_binary().await?;
+    assert_history_and_iam(&dist, &seed, "direct").await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+
+    dist.replace_node_with_current_binary(0).await?;
+    assert_history_and_iam(&dist, &seed, "one-current-node").await?;
+
+    for node_idx in [1, 2] {
+        dist.replace_node_with_current_binary(node_idx).await?;
+    }
+    assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
+
+    dist.replace_node_with_current_binary(3).await?;
+    assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
+    Ok(())
+}
+
+#[test]
+fn missing_upgrade_source_binary_fails_closed() {
+    let err = resolve_source_binary(None).expect_err("absent env must fail closed");
+    assert!(err.to_string().contains(SOURCE_BINARY_ENV), "{err}");
+}
+
+#[test]
+fn missing_upgrade_source_binary_file_fails_closed() {
+    let err = resolve_source_binary(Some("/no/such/rustfs-upgrade-source".into())).expect_err("missing file must fail closed");
+    assert!(err.to_string().contains("does not exist"), "{err}");
+}

crates/e2e_test/src/distributed/versioning_test.rs

@@ -0,0 +1,88 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+
+#[tokio::test]
+async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("version");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "versioned.txt";
+    put_object(&writer, &bucket, key, b"v1".to_vec()).await?;
+    put_object(&writer, &bucket, key, b"v2".to_vec()).await?;
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let version_ids: Vec<String> = versions
+        .versions()
+        .iter()
+        .filter_map(|version| version.version_id().map(str::to_string))
+        .collect();
+    assert!(version_ids.len() >= 2, "expected at least two versions, got {version_ids:?}");
+
+    let latest = get_object_bytes(&reader, &bucket, key).await?;
+    assert_eq!(latest, b"v2");
+
+    let older_id = versions
+        .versions()
+        .iter()
+        .find(|version| version.is_latest() != Some(true))
+        .and_then(|version| version.version_id())
+        .ok_or("missing non-latest version id")?;
+    let older = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(older_id)
+        .send()
+        .await?;
+    let older_body = older.body.collect().await?.into_bytes();
+    assert_eq!(older_body.as_ref(), b"v1");
+
+    writer.delete_object().bucket(&bucket).key(key).send().await?;
+    let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    assert!(
+        !after_delete.delete_markers().is_empty(),
+        "delete marker missing after unversioned-style delete: {after_delete:?}"
+    );
+
+    let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
+    match latest_after_delete {
+        Ok(_) => return Err("current version should be a delete marker".into()),
+        Err(error)
+            if error
+                .as_service_error()
+                .and_then(ProvideErrorMetadata::code)
+                .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
+        Err(error) => return Err(error.into()),
+    }
+
+    let restored = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(older_id)
+        .send()
+        .await?;
+    let restored_body = restored.body.collect().await?.into_bytes();
+    assert_eq!(restored_body.as_ref(), b"v1");
+    Ok(())
+}

crates/e2e_test/src/lib.rs

@@ -378,6 +378,11 @@ mod bucket_stats_regression_test;
 #[cfg(test)]
 mod distributed_startup_regression_test;
 
+// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
+// quota, observability, expand/decommission/rebalance, site replication, chaos).
+#[cfg(test)]
+mod distributed;
+
 // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
 #[cfg(test)]
 mod tier_transition_regression_test;

docs/architecture/ilm-tiering-persistence-contracts.md

@@ -41,7 +41,7 @@ All keys below are objects in the internal metadata bucket. The table gives the
 
 | Protocol | Current schema/version | Canonical key | Creator and cleanup owner | Authoritative identity and mutable fields | Current durability point |
 |---|---|---|---|---|---|
-| Transition transaction | `rustfs-transition-transaction-v1`; successor v2 is approved below but not implemented | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed v1 `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Create-only maximum-parity write; exact record and ETag read before successor `If-Match`; terminal receipt followed by exact ETag conditional delete |
+| Transition transaction | `rustfs-transition-transaction-v1` | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Maximum-parity config write. Current create/update/delete calls do not use ETag preconditions |
 | Tier mutation peer intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/records/<aa>/<bb>/<mutation-id>.json` | The receiving peer creates and converges it; the mutation recovery path cleans it | Immutable: mutation ID/kind, old config ETag, candidate digest, sorted affected target identities, expiry. Mutable: revision, state, committed config ETag | Create with `If-None-Match: *`; transition/delete with ETag `If-Match`; maximum parity |
 | Tier mutation coordinator intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/coordinators/<aa>/<bb>/<mutation-id>.json` | The initiating node creates it; coordinator recovery cleans it after peer convergence | Same mutation identity and mutable fields as the peer record | Same conditional-write contract as the peer intent |
 | Manual job | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/jobs/<aa>/<bb>/<job-id>.json` | The admin run creates it; the active owner or recovery lease advances it. There is no current record GC owner | Immutable: job ID, bucket-level scope, options, creation time. Mutable: owner/lease, state, cancel bit, cursor, progress/report, queue snapshot, timestamps/error | Initial UUID-key write uses maximum parity without create-only precondition; later updates use ETag CAS |
@@ -54,7 +54,6 @@ All keys below are objects in the internal metadata bucket. The table gives the
 | Tier-delete chunk parent | Version 1 with `record_type = "chunked_parent"` | `ilm/tier-delete-dispatch-manifests/<scope-digest>.json` | The over-limit prefix-delete coordinator creates and advances it; parent recovery advances completed children and removes the terminal parent | Immutable operation, bucket/incarnation/prefix/topology; mutable monotonic revision, next child sequence, completed journal count, one optional exact child binding, and `Active`/`Completed` state | Create-only and fenced ETag CAS. The parent binds a `Preparing` child before it can become `DispatchAuthorized`; final `Completed` follows an error-free, non-truncated empty-candidate rescan and local prefix deletion |
 | Decommission durable-namespace receipt | `v2` | `decommission/ilm-receipts/<run-token>/<source-path>/<id-kind>/<id>.json` | The decommission coordinator writes target/source proof and is the only cleanup owner for that run | Source path, namespace and record identity, monotonic checkpoint, optional terminal checkpoint, optional v6 topology generation | Create-only then ETag CAS merge; checksum envelope; maximum parity |
 | Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/<run-token>.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal |
-| Recovery control, export, and disposition | `rustfs-ilm-recovery-control-v1`, `rustfs-ilm-recovery-export-v1`, and `rustfs-ilm-recovery-disposition-v1` are approved below but not implemented | `ilm/recovery-controls/...`, `ilm/recovery-exports/...`, and `ilm/recovery-dispositions/...` under protocol/shard/operation identities | Recovery owns one control for an exact source generation; the authenticated operator creates immutable export/disposition evidence; their collectors never own remote DELETE | Source protocol/path, all-pool copy-set manifest, ETags/content digests, owner lease, retry state, redacted error code, action, actor/reason, and terminal proof | Create-only, ETag CAS, all-pool strong readback, terminal receipt when covered by decommission, and exact conditional cleanup |
 
 `durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
 
@@ -72,7 +71,7 @@ The internal config layer supplies maximum-parity writes, create-only writes, ET
 | Conditional delete | Removes only the verified terminal generation; if an active decommission covers it, its terminal receipt is written first |
 | Strong readback | Resolves a lost response only when key, schema, full immutable identity, state, and expected successor all match |
 
-Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. Transition transaction v1 now uses create-only installation, exact record plus ETag read before each successor CAS, and exact ETag terminal deletion. Its `owner_epoch` and `not_after_unix_nanos` remain immutable, however, so an expired recovery worker claims only the next state generation rather than a renewable durable owner lease. The manual job's initial UUID-key write still lacks create-only installation, although all later owner/lease updates are CAS-protected.
+Tier mutation intents and v6 journal/manifest records use the conditional primitives. Manual job updates, scope admission, and decommission receipts also use CAS after creation. The transition transaction currently carries a fixed `owner_epoch` fence identity and a mutable `revision`, but persists with unconditional writes and deletes; those fields therefore detect some in-memory misuse but are not yet a durable exclusion fence. Changing `owner_epoch` during takeover is not current behavior and remains an open design. The manual job's initial UUID-key write has the same create-only gap, although all later owner/lease updates are CAS-protected.
 
 ### Approved target
 
@@ -82,7 +81,7 @@ Tier mutation intents and v6 journal/manifest records use the conditional primit
 - After a timeout, connection loss, or quorum-uncertain response, the caller must strongly reread. Only the exact intended successor is success; predecessor, absence, conflict, corruption, or unavailable readback retains the record and blocks destructive action.
 - A process-local mutex, cancellation token, task registry, or cached generation may reduce duplicate work but cannot authorize publication, rollback, or remote deletion.
 
-The approved transition-transaction successor, lease/takeover fields, v1 migration, and upgrade/downgrade gates are specified in [Bounded recovery control and operator disposition](#bounded-recovery-control-and-operator-disposition). They require implementation and fleet gating before any v2 writer or destructive v1 takeover is enabled.
+The exact transition-transaction lease/takeover fields and whether the existing `not_after` becomes the owner expiry are an **open design**. They must be settled with upgrade/downgrade behavior before the v1 schema changes.
 
 ## Lock and operation order
 
@@ -90,19 +89,18 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n
 
 | Path | Current acquisition order | Operations allowed while held | Operations forbidden while held |
 |---|---|---|---|
-| Tier add/edit/remove/clear | A short tier-config namespace WRITE lock captures the persisted config ETag, then releases before backend validation. After validation, namespace WRITE then `admin_updates` protect the ETag check and durable coordinator Prepared write. Both guards are released for lease drain, peer Prepare, and reference proof, then reacquired in the same order for final identity checks and config CAS. Both are released again after the coordinator becomes durably Committed | Backend validation, peer fanout, and reference proof run without either exclusive guard. Immediately before config CAS the coordinator revalidates the ETag, candidate digest, exact Prepared intent identity, and intent expiry. The durable Committed intent is recovery authority while peer Commit and local publication finish without the exclusive guards | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O. Recovery must retain an unexpired Prepared coordinator whose old ETag is still current; the old ETag alone is not abandonment proof. Remote object DELETE is never part of mutation |
+| Tier edit/remove/clear | Tier-config namespace WRITE lock; dedicated owned `admin_updates` serialization mutex; short `TierConfigMgr` state locks only while accessing manager/runtime state | The dedicated `admin_updates` guard intentionally spans awaited backend validation/probes, peer Prepare/Commit/Abort RPC, reference scans, config CAS, and candidate publication in the current protocol | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O; that rule does not prohibit the dedicated `admin_updates` guard from spanning those awaits. Remote object DELETE is never part of mutation |
 | v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition |
 | v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery |
 | v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version |
 | v6 journal destructive recovery | Synthetic per-journal recovery lock; fleet/topology proof; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Immutable manifest/topology validation, authoritative source/free-version scan, fenced state CAS, and, for an eligible terminal state, one bounded remote DELETE followed by record cleanup | Any delete when a lock, lease, or fleet proof is lost; publishing local metadata; selecting an arbitrary backend/version |
 | Free-version cleanup | Bucket-lifecycle READ lock; exact tier-generation lease; all physical object WRITE locks in stable pool/set order | Exact all-pool scan; bounded remote DELETE; local marker removal; post-delete rescan | Deleting before the free-version is the sole owner or after any fence changes |
 | Transition commit | Existing object commit locks plus exact source identity and tier-generation checks in the transition path | Publish the exact remote tuple into the matching local version | Publishing a tuple after the source identity or generation changes |
-| Transition transaction cleanup | Record validation; expiry check; a next-state ETag CAS for cleanup ownership; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit renewable owner lease or bucket-lifecycle/physical-set ownership fence | Identity-bound provider probe and deletion of a known canonical candidate | The cleanup-state CAS fences a stale predecessor, but the approved target also requires an explicit recovery lease and exact all-pool source reread before DELETE |
-| Recovery artifact quota admission (approved target) | Cluster-scoped recovery-admission WRITE lock first; then the one canonical control/source operation lock and any source-protocol metadata/physical locks in that protocol's existing stable order | Bounded internal artifact inventory, exact source/control validation, and create-only installation plus strong readback of one fully encoded export or disposition candidate | Acquiring admission while holding a control, source, disposition, bucket, physical, migration, or decommission guard; remote backend I/O; source-journal deletion; disposition `Applying`; releasing admission before candidate installation/readback converges |
+| Transition transaction cleanup | Record validation; exact backend-generation lease inside the probe/delete helper. Current recovery has no explicit bucket-lifecycle/physical-set ownership fence or durable takeover CAS | Identity-bound provider probe and deletion of a known canonical candidate | A tier lease alone does not fence the creator. The approved target requires expired ownership, durable takeover, and exact local/source reread before DELETE |
 | Manual job | Initial maximum-parity job write; then persisted bucket-level scope create/CAS; later short job/task/result metadata operations | List, checkpoint, append tasks before enqueue, append results after work, renew/take over lease | Holding metadata guards across remote transition PUT; treating the local active-job map as cluster authority. A crash between the job write and scope claim can leave `Running` without a scope record |
 | Decommission receipt | Decommission coordinator's source/target record workflow; record-specific conditional writes | Copy/validate durable record, advance receipts, construct and verify expected manifest, conditionally clean exact covered source | Remote tier DELETE; deleting an uncovered or divergent source record |
 
-Tier mutation backend validation is outside both exclusive guards and is bound to the persisted ETag snapshot. The initiating task is detached from the admin request so cancellation cannot interrupt validation cleanup. Once the coordinator Prepared record and local fence are durable, lease drain, all-node peer Prepare, and reference proof run without the tier-config namespace or `admin_updates` guard. Recovery treats an unexpired Prepared record as active even while the old config ETag remains current. Both guards are reacquired in namespace-then-admin order, and the ETag, candidate digest, exact intent identity, and expiry are revalidated immediately before config CAS. After the coordinator advances to Committed, the guards are released again for peer Commit and local publication.
+The tier mutation lock scope is intentionally recorded as **current**, not ideal. Reducing it is allowed only after a durable `Prepared` intent blocks new reference creators across the fleet, existing tier-operation leases drain, and recovery can reconstruct that block without the initiating process. Which network validation can move outside the namespace lock is an **open design**.
 
 ## Transition transaction
 
@@ -116,7 +114,7 @@ UploadStarted -> Uploaded -> LocalCommitStarted -> Committed
              \-> AbortedNoRemote
 ```
 
-Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production recovery emits `CleanupPending` after an expired `Uploaded` record wins the exact successor CAS, or when an expired `UploadOutcomeUnknown` probe returns `UnversionedPresent` or `VersionedPresent` with a non-nil identifier. `LocalCommitStarted` mismatch or missing-source recovery retains the record; that cleanup edge is currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is retained and never authorizes remote deletion.
+Separately, `mark_cleanup_pending` permits proof-checked model edges from `Uploaded`, `UploadOutcomeUnknown`, and `LocalCommitStarted`. Current production code emits `CleanupPending` only when recovery probes `UploadOutcomeUnknown` as `UnversionedPresent` or as `VersionedPresent` with a non-nil identifier. The `Uploaded` abort/recovery path deletes its candidate and transaction record directly, and `LocalCommitStarted` mismatch or missing-source recovery retains the record. The `Uploaded` and `LocalCommitStarted` cleanup edges are currently exercised through the state-machine API and tests, not produced by runtime recovery. States that require a remote delete still require a known `TransitionRemoteVersion` kind. A probed versioned candidate whose identifier parses as a nil UUID is another current special case: recovery exact-deletes it and removes the record without first persisting `CleanupPending`.
 
 The remote candidate itself is named by `canonical_transition_remote_object` under `ilm/transition-transactions/<bucket-hash>/<transaction shards>/<transaction-id>/<write-id>`. That deterministic identity is what a provider probe or exact cleanup must bind; it is distinct from the internal transaction-record key.
 
@@ -126,13 +124,13 @@ The creator owns the canonical remote candidate until local metadata commits the
 
 | Observed durable state/input | Unique current owner | Current recovery decision | Approved destructive admission |
 |---|---|---|---|
-| `UploadStarted` | Originating transition attempt; no current recovery successor is emitted | Retain | No delete. The upload may still publish |
-| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery under the exact record generation and recovery lock | Conditionally delete the record | Strong probe identity must match transaction/backend; no remote delete occurs |
+| `UploadStarted` | Originating transition attempt; current durable exclusion is incomplete | Retain | No delete. The upload may still publish |
+| `UploadOutcomeUnknown`; exact provider probe says missing | Transaction recovery, logically; current record writes do not durably exclude a concurrent worker | Delete the record | Strong probe identity must match transaction/backend; no remote delete occurs |
 | `UploadOutcomeUnknown`; probe returns `UnversionedPresent` | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, delete the unversioned candidate, delete the record | Exact transaction/canonical object/backend identity, explicitly unversioned state, durable takeover after owner expiry, current tier lease, and exact reread before cleanup |
 | `UploadOutcomeUnknown`; probe returns `VersionedPresent` with a non-nil exact identifier | Transaction recovery, with operator reconcile available after expiry | Persist `CleanupPending`, exact-delete that versioned candidate, delete the record | Exact transaction/canonical object/backend identity and remote version, durable takeover after owner expiry, current tier lease, and exact reread before cleanup |
-| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery retains ownership evidence | Retain | A nil identifier is invalid exact-version evidence and never becomes unversioned or remote-delete authority |
+| `UploadOutcomeUnknown`; probe returns `VersionedPresent` whose identifier is a nil UUID | Transaction recovery | Current code directly exact-deletes that versioned candidate and deletes the record; it does not persist `CleanupPending` | This remains a versioned exact-delete candidate and must not be treated as `UnversionedPresent`. The approved target still requires durable takeover, a current tier lease, and exact reread |
 | `UploadOutcomeUnknown`; probe ambiguous, unsupported, or errors | Transaction recovery retains ownership evidence | Retain | No destructive action; operator reconcile may inspect after expiry |
-| `Uploaded` | Originating transition attempt until expiry; after expiry, the worker that wins `Uploaded -> CleanupPending` by exact ETag CAS | Retain while active; after expiry, persist `CleanupPending`, then recheck and delete the unreferenced candidate or record | Current CAS fences the predecessor, but approved v2 also requires a durable recovery lease, full all-pool source/free-version proof, and before/after fence checks |
+| `Uploaded` | Originating transition attempt; current recovery can race it because the persisted fence is not CAS-protected | Current code immediately deletes the candidate and record | **Current safety gap:** approved behavior must first prove the creator cannot still commit by expired ownership plus durable takeover/CAS, then recheck that no matching local commit exists |
 | `LocalCommitStarted`; logical source lookup returns `TRANSITION_COMPLETE` with the same remote object, tier, and remote version | Transition committer until ownership transfers to `xl.meta` | Delete transaction record | Current recovery treats this tuple as ownership transfer. The approved target additionally compares recorded source version ID, data directory, modification time, size, and ETag before conditional terminal cleanup |
 | `LocalCommitStarted`; logical source is missing, its transition tuple differs, or the read is uncertain | Transaction record/recovery | Retain | No remote delete without a separate durable cleanup proof |
 | `CleanupPending`; logical source lookup returns the same current transition predicate | `xl.meta` is remote reachability owner; recovery owns only record cleanup | Delete transaction record | `xl.meta` is owner; do not delete remote. The approved target adds the full recorded source comparison |
@@ -150,12 +148,12 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
 
 ### Approved target and open design
 
-- Preserve the current create-only transaction installation, exact record/ETag successor CAS, and conditional terminal delete, and add mandatory exact-successor strong readback for lost or uncertain responses. No v2 work may regress the existing v1 protections.
+- Replace unconditional transaction create/update/delete with create-only, ETag CAS, and conditional terminal delete.
 - Recovery of `Uploaded` and cleanup-capable states must acquire durable ownership only after the prior owner's expiry. The recovery worker must reread the exact generation after takeover and before remote DELETE.
 - Preserve and compare source version ID, data directory, modification time, size, and ETag through local commit and recovery before accepting ownership transfer.
 - Recompute and require the exact lowercase sharded path in both transition-transaction and manual-job runtime recovery, and validate a truncated page's continuation token before processing any record from that page.
-- The approved v2 owner lease uses the duration, clock-skew allowance, takeover revision, and v1 mapping in [Transition transaction v2 and v1 migration](#transition-transaction-v2-and-v1-migration).
-- Active automatic retries are bounded by the recovery-control policy below. Ambiguous evidence becomes explicit `retained_ambiguous` or `operator_required`; source evidence is never collected merely because it is old.
+- **Open:** owner lease duration, clock-skew allowance, takeover revision encoding, and compatibility for existing v1 records that have only `not_after`/`owner_epoch`.
+- **Open:** bounded retention and an operator disposition for permanently ambiguous records. Until defined, retained ambiguity is safer than collection.
 
 ## Tier mutation intent
 
@@ -165,15 +163,14 @@ The current background loop runs every 60 seconds, scans at most 1,000 records p
 
 New intents use a 15-minute expiry. A peer-only terminal tombstone is retained until that expiry plus five minutes of clock-skew allowance and until no coordinator record remains. Expiry bounds replay protection; it is not config commit/abort evidence.
 
-The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. Prepare, Commit, and Abort use all-node fanout rather than quorum: independent peer calls use a work-conserving concurrency limit of four, a 30-second per-peer deadline, and a 30-second fanout-wide deadline; Prepare is additionally capped by the intent expiry. The coordinator collects every completed outcome. A timed-out or otherwise ambiguous started Prepare is included in compensating Abort because cancellation does not prove the peer failed to persist its fence; peers not started before the fanout deadline make Prepare fail but do not require Abort. The coordinator then conditionally writes tier config, durably commits the coordinator intent, releases its exclusive guards, requires every prepared peer to commit, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.
+The coordinator creates its durable record and peer `Prepare` blocks new reference creation, drains exact tier-operation leases, and proves that edit/remove/clear will not strand authoritative references. The coordinator then conditionally writes tier config, commits peers, publishes the runtime candidate, and clears the block. Per-mutation sharded mutexes serialize local phases only; persisted intent plus tier-config ETag is authoritative.
 
 ### Recovery decisions
 
 | Observed durable state/input | Unique current owner | Current recovery decision | Destructive/config admission |
 |---|---|---|---|
 | `Prepared`; current tier-config digest equals candidate | Coordinator recovery; each peer recovery owns only its matching peer record/block | CAS to `Committed`, replay peer Commit and publish | Exact mutation identity and candidate digest; never infer from expiry |
-| `Prepared`; intent is unexpired and current config still proves the old ETag/config | The live coordinator remains owner | Retain `Prepared` and the runtime block | Recovery must not abort work that may still be in lock-free peer Prepare or reference proof |
-| `Prepared`; intent is expired and current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone |
+| `Prepared`; current config still proves the old ETag/config | Coordinator recovery | Fan out canonical Abort, then CAS `Aborted` | Abort only the matching intent; a delayed matching Prepare converges to the tombstone |
 | `Prepared`; config is a third generation, unreadable, or peer outcome is ambiguous | Coordinator record remains owner of the block | Retain `Prepared` and runtime block | No commit, abort, cleanup, or unblock |
 | `Committed` | Coordinator recovery, with peers owning convergence of their local records | Replay peer Commit/runtime publication; clean exact converged records | Config ETag/digest and peer identity must match |
 | `Aborted` | Coordinator recovery; peer recovery retains the local tombstone | Replay/confirm Abort and clear matching block; retain peer tombstone until expiry plus clock skew and no coordinator | Never roll back config based on timeout alone |
@@ -186,8 +183,8 @@ Intent transitions retry an ETag race at most three times before returning a ret
 ### Approved target and open design
 
 - Keep create-only, ETag CAS, exact identity comparison, canonical Abort tombstones, and lost-response readback.
-- The phase split is fixed: ETag snapshot, lock-free backend validation, ETag revalidation, all-node Prepare, reference proof, ETag/digest/intent revalidation, config CAS, all-node Commit, and local publication. Backend validation uses `(old_config_etag, candidate_digest)` as its stable config generation; the durable mutation identity adds `mutation_id`. Runtime driver revisions are not persistence identities and Add does not require one before publication.
-- Lease drain, peer Prepare, and reference proof run outside both exclusive guards after the coordinator Prepared record and local fence are durable. The commit path reacquires namespace WRITE then `admin_updates` and repeats the full generation/identity proof. Expiry is checked as an additional rejection boundary and never replaces config-generation proof.
+- Any shorter configuration-lock window must leave a durable `Prepared` fence installed on every required peer before releasing the broad exclusion scope and must prove recovery restores that fence before admitting reference creators.
+- **Open:** the exact split between backend validation, peer fanout, reference scan, config CAS, and publication; expiry must never replace config-generation proof.
 - **Open:** a dedicated operator reconcile/status surface and bounded retention for irreconcilable coordinator/peer records.
 
 ## Manual transition job, task, result, and checkpoint
@@ -314,149 +311,7 @@ Single and child manifest preparation is bounded by 200,000 journals and a 32 Mi
 - New destructive prefix paths use only v6 plus either one byte-compatible manifest or one parent-bound sequence of byte-compatible child manifests. No new v1-v5 sole-owner records may be created.
 - Preserve the two-phase authorization barrier: all prepared records, durable barrier, all dispatched records, durable `DispatchAuthorized`, local mutation, journals committed, durable `Completed`, then remote DELETE.
 - Do not downgrade every v6-aware recovery worker while v6 records remain. v5-and-older readers reject and retain v6 records; older nodes may continue producing fallback free-versions until the fleet is homogeneous.
-- Quarantined v1/v2 records, incomplete manifests, and repeatedly failing exact deletes use the bounded retry, explicit operator state, and single-record control surface approved below. Capacity and recovery throughput must not be “fixed” by weakening ownership proof or age-deleting source evidence.
-
-## Bounded recovery control and operator disposition
-
-This section is an **approved target that is not implemented yet**. It closes the ownership, retry, and operator-disposition design required before backlog recovery work can change destructive behavior. It does not authorize an implementation to enable a v2 writer, take over a v1 transaction, or remove a legacy journal until the fleet and storage gates below exist.
-
-### Recovery-control record
-
-Retry scheduling is persisted separately from the source transaction or journal so legacy bytes remain readable and their cleanup ownership does not move. The control record uses schema `rustfs-ilm-recovery-control-v1`, a checksum envelope, a 16 KiB encoded-size limit, strict unknown-field rejection, and this canonical key:
-
-```text
-ilm/recovery-controls/<protocol>/<aa>/<bb>/<source-operation-digest>.json
-```
-
-Export envelopes use `ilm/recovery-exports/<protocol>/<aa>/<bb>/<export-id>.json`; disposition receipts use `ilm/recovery-dispositions/<protocol>/<aa>/<bb>/<disposition-id>.json`. `export-id` is the SHA-256 of the control ID plus exact observed source content/copy-set digests; `disposition-id` is the SHA-256 of the export ID plus the bounded action. Repeating an identical export or disposition reuses and strongly validates the same canonical object; different bytes at that ID are a conflict. Both use strict checksum envelopes and create-only installation. A control or disposition is limited to 16 KiB. An export is limited to the source protocol's own maximum encoded record size plus 64 KiB for its copy manifest and envelope; it stores the source bytes once rather than duplicating identical replica bytes.
-
-Admission is fail-closed and cluster-wide. One control may have at most one export creation and one disposition application in flight. The immutable candidate is fully encoded before quota admission. The cluster-scoped recovery-admission WRITE lock is always outermost; a caller already holding any control, source, disposition, bucket, physical, migration, or decommission guard must release it and restart in the order above. While admission is held, a complete artifact inventory must prove that the projected totals, including the candidate, are at most 10,000 export envelopes, 10,000 disposition receipts, 1 GiB of encoded export data, and 256 MiB of encoded control/disposition data. The create-only candidate installation and exact strong readback complete before the lock is released, so neither a single candidate nor a concurrent node can oversubscribe the pre-create snapshot. A crash before installation leaves no artifact; a lost create response is resolved by exact canonical readback while admission remains serialized or after reacquiring it in the same order. Replaying an existing canonical ID consumes no new count, byte, or rate token. New creations are limited to ten per authenticated actor per minute and 100 cluster-wide per minute, with at most 32 export creations and eight disposition applications executing cluster-wide. Exceeding a count, byte, rate, or concurrency limit returns a retryable capacity result before source mutation; it never evicts evidence, interrupts an admitted operation, or blocks ordinary object I/O. The collector examines at most 100 terminal artifacts per minute and never acquires the admission lock while holding an artifact/source guard.
-
-`source-operation-digest` is SHA-256 over the length-delimited source protocol, canonical source path, and stable semantic operation identity: transaction UUID, journal identity, or manifest operation ID. For corrupt bytes whose semantic ID cannot be trusted, the canonical path plus a `corrupt` domain separator is the stable identity; a replacement at that path conflicts with the existing control instead of resetting its history. The control's immutable identity repeats the protocol, path, stable identity, and record class. Its mutable `observed_source_generation` contains the source schema, source ETag/content SHA-256, and sorted all-pool copy-set digest. The copy set records each authoritative pool/set identity, canonical path, ETag, byte length, and content SHA-256; an unreachable pool, missing ETag, divergent copy, or incomplete listing cannot produce an actionable generation. The remaining mutable generation contains:
-
-- `revision`, an ETag-CAS successor counter;
-- `classification`: `retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`;
-- `owner_id`, `owner_epoch`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos` while an attempt is owned;
-- monotonic `attempt_count`, `consecutive_failure_count`, `first_failure_at_unix_nanos`, `last_failure_at_unix_nanos`, and `next_attempt_at_unix_nanos`;
-- one bounded enum `last_error_code`; no free-form provider error, endpoint, credential material, request payload, or response body is persisted;
-- for an operator disposition only, the authenticated actor identifier, reason code, confirmation time, exported payload digest, and exact source/control ETags that were confirmed.
-
-The control record is a scheduler and audit fence, not a remote-object owner. It never substitutes for the source record's version, backend, manifest, source-identity, or absence proof. Before every source CAS, local cleanup, or remote request, the worker strongly rereads every authoritative source copy and the control, requires the current observed generation to match, and then applies the source protocol's own locks and proof. A missing, stale, corrupt, divergent, or unavailable control read cannot authorize work.
-
-Control creation is `If-None-Match: *`; every update is exact ETag `If-Match`; response loss requires exact strong readback. A legal source-state CAS does not create a new control. The stable control CAS advances `observed_source_generation` only from the exact predecessor to one source-protocol successor while preserving `first_failure_at_unix_nanos`, lifetime `attempt_count`, and first-seen lineage. If the source CAS succeeded before a crash, recovery accepts the new bytes only after validating that exact legal successor and then converges the old control generation by CAS. A multi-edge jump, semantic-identity change, replacement ETag/content, or unavailable predecessor proof is a conflict and cannot reset counters. Recovery conditionally removes a terminal control only after strongly proving the stable source operation resolved or absent, recording any active decommission terminal receipt, and confirming that no in-flight operator request still names its ETag. `durable_namespace.rs` must register every new namespace, path parser, decoder, size bound, successor relation, and terminal checkpoint before rollout.
-
-### Transition transaction v2 and v1 migration
-
-The successor envelope is `rustfs-transition-transaction-v2`. It preserves the v1 transaction ID, deployment ID, write ID, complete source identity, tier/backend identity, canonical remote object, remote-version state, operation state, and revision. It also records an immutable `origin_format` (`native_v2` or `migrated_v1`), the state-entry revision and predecessor state/revision, and, for migration, the exact source v1 state/revision. These fields make the state history needed for destructive authorization independently checkable instead of inferring it from the current state. The envelope and body reject unknown fields, checksum every field, use no default for a required v2 value, require the existing exact lowercase canonical path, and reject nil UUIDs, nonpositive timestamps, revision zero/overflow, lease inversion, impossible state/revision history, and illegal state/version combinations. It replaces the fixed ownership pair with:
-
-- immutable `creator_epoch` and `creator_not_after_unix_nanos`, copied exactly from v1 `owner_epoch` and `not_after_unix_nanos` during migration;
-- optional `created_at_unix_nanos`; migrated v1 records use `None` rather than inventing an age;
-- mutable `owner_role` (`creator` or `recovery`), `owner_id`, `owner_epoch`, `owner_generation`, `lease_acquired_at_unix_nanos`, and `lease_expires_at_unix_nanos`. `owner_id` is the authenticated stable fleet-node identity; `owner_epoch` is a fresh UUID for one process claim. A node without both identities cannot create, renew, take over, or act on v2.
-
-A native v2 create uses revision and owner generation 1, a fresh non-nil creator owner/epoch, `UploadStarted`, and an unknown remote version. The first rollout keeps the current seven-day creator ownership window. Creator leases are not renewable; a creator that cannot finish inside the safety window stops publishing and leaves the record for recovery. Recovery-owner leases are 15 minutes. The persisted clock-skew allowance is five minutes: an action may start only when its bounded deadline fits before `lease_expires_at - 5 minutes`, and another owner cannot take over until its local time is at least `lease_expires_at + 5 minutes`. A recovery attempt remains capped at five minutes and every remote call remains subject to its narrower client deadline. A recovery renewal is a same-owner, next-revision CAS that strictly extends expiry; a takeover changes `owner_role`, `owner_id`, and `owner_epoch`, increments `owner_generation` and `revision`, and uses the exact observed ETag. Takeover and state advancement are separate CAS operations; one revision cannot both acquire ownership and claim a recovery outcome.
-
-Before migration can be enabled, the fleet must first deploy a v1 creator fence that strongly rereads the exact transaction path, ETag, state, owner epoch, and source generation immediately before local metadata publication and refuses to publish after any migration/takeover change. The fleet then durably disables new v1 admission and proves that every captured creator/recovery process epoch has either acknowledged quiescence or terminated. A paused or unreachable epoch prevents migration. This drain barrier is distinct from format capability advertisement and remains in force until v2 writer admission is enabled.
-
-Only these checksum-valid v1 state/revision pairs are migration inputs: `UploadStarted@1`; `UploadOutcomeUnknown@2`; `AbortedNoRemote@2`; `Uploaded@2` or `Uploaded@3`; `LocalCommitStarted@3` or `LocalCommitStarted@4`; `Committed@4` or `Committed@5`; and `CleanupPending@3`, `CleanupPending@4`, or `CleanupPending@5`. Any other pair is `corrupt`, inspect-only, and cannot be migrated or authorize a probe, local cleanup, or remote DELETE. A native v2 record must prove a legal predecessor edge at its recorded state-entry revision; ownership-only revisions may increase the outer revision but cannot change the recorded state-entry history. Missing, contradictory, or skipped history is corrupt.
-
-An identity-preserving `v1 -> v2` conversion is one legal successor with `revision + 1` and an unchanged remote tuple. The state is unchanged except that every v1 `UploadStarted` maps conservatively to v2 `UploadOutcomeUnknown`. Historical v1 writers could issue PUT while still in `UploadStarted`; no age, fleet version, or current process observation proves that a retained record came from the later pre-PUT-fence writer. It is permitted only when:
-
-1. every node that can create, commit, recover, heal, or decommission the record advertises both `transition_transaction_v2` and `ilm_recovery_control_v1` for the captured fleet/topology generation, the durable v1-admission stop is active, and the process-epoch drain barrier above is complete;
-2. current time is at least the v1 `not_after_unix_nanos` plus five minutes of skew;
-3. the canonical path, checksum, full immutable identity, state, remote-version invariant, source record, and ETag all match the observed v1 generation;
-4. the migration CAS and strong readback install one fresh recovery lease before any state transition or side effect.
-
-The original creator must successfully CAS `UploadStarted -> UploadOutcomeUnknown` before issuing remote PUT, and must CAS the exact current owner generation to `LocalCommitStarted` before publishing local metadata. A v2 takeover therefore fences a delayed creator. An implementation that can issue PUT or publish after losing this CAS is not compatible with this protocol.
-
-After takeover, recovery applies this matrix:
-
-| State | Approved recovery after exact takeover | Required proof before side effect |
-|---|---|---|
-| native-v2 `UploadStarted` | CAS `AbortedNoRemote`, then conditionally remove the terminal transaction | Valid native-v2 history proves the creator was fenced before the mandatory pre-PUT `UploadOutcomeUnknown` CAS; migrated v1 never enters this row and no remote request is made |
-| `UploadOutcomeUnknown` | Probe under the exact backend lease. Missing becomes terminal cleanup; proven unversioned presence or one nonempty, non-nil exact version becomes `CleanupPending`; nil, ambiguous, or unsupported results become `retained_ambiguous` | Exact transaction/control generations, bounded live probe, current tier destination and lease |
-| `Uploaded` | If the exact transitioned tuple or its free-version owns the candidate, remove only the transaction. If the complete original source is still unchanged and no local commit/free-version exists, CAS `CleanupPending`. Otherwise retain | All-pool source/free-version read, full source tuple, bucket incarnation, tier generation, object locks, and post-probe revalidation |
-| `LocalCommitStarted` | Exact committed tuple/free-version means record-only cleanup. A fully unchanged original source with no partial committed tuple may move to `CleanupPending`. Missing, divergent, partial, or unavailable metadata is `retained_ambiguous` | Same all-pool proof, including data directory, modification time, size, ETag, transition transaction ID, remote tuple, and destination identity |
-| `CleanupPending` | Resume the same exact idempotent candidate delete, or remove only the transaction when local ownership transfer is proven | Current owner lease, source/free-version proof, exact tier lease, physical locks, and before/after fence checks |
-| `Committed` | Conditionally remove only the exact terminal transaction when complete local ownership transfer is proven; otherwise classify the record as corrupt or retain it as ambiguous | All-pool strong read matches the complete logical `xl.meta` source identity, transaction ID, remote tuple, destination identity, and any required decommission terminal receipt; no remote DELETE |
-| `AbortedNoRemote` | Conditionally remove the exact terminal transaction | Valid native-v2 pre-PUT history or exact migrated v1 `AbortedNoRemote@2`, exact terminal generation, and any required decommission terminal receipt; no remote DELETE |
-
-Remote DELETE is never admitted directly from `UploadStarted`, `UploadOutcomeUnknown`, `Uploaded`, or `LocalCommitStarted`; it first requires a CAS-protected `CleanupPending` generation with known remote-version semantics. A lost source read, mixed all-pool result, expired lease, failed renewal, or source/control CAS conflict retains the evidence and performs no destructive action.
-
-New writers emit v2 only after the homogeneous fleet gate, durable v1-admission stop, and process-epoch drain barrier are complete. During a rolling upgrade, new readers accept v1 but all writers continue v1 and no v1 takeover/migration occurs. A v1 reader rejects and retains v2. Downgrade is blocked until v2 creation is disabled and all v2 transactions and recovery-control records are drained or exported; live v2 bytes are never rewritten to v1.
-
-### Retry and retention policy
-
-An attempt that loses a CAS or discovers a newer source generation reloads instead of recording a remote failure. A retryable transport timeout, backend 5xx/throttle, metadata quorum outage, or bounded remote-delete failure increments the persisted counters and schedules:
-
-```text
-min(60 seconds * 2^min(consecutive_failure_count - 1, 6), 1 hour)
-```
-
-A deterministic multiplier from 80 to 100 percent, derived from the source-generation digest and attempt count, is applied to that capped base. The jitter can only shorten the delay and therefore never exceeds the one-hour cap; restarts reproduce the same deadline without synchronizing a fleet. Success or a proven source-state advance resets `consecutive_failure_count` but never decreases `attempt_count`. `next_attempt_at_unix_nanos` is only a not-before scheduler hint; ownership and destructive authority still require the lease and source proofs.
-
-After 32 consecutive retryable failures or seven days since `first_failure_at_unix_nanos`, whichever occurs first, the control CAS moves to `operator_required` and automatic attempts stop. Unsupported probes and unknown remote-version semantics move directly to `retained_ambiguous`; corrupt source bytes use `corrupt`; incomplete destructive evidence uses `operator_required`. None is periodically hot-looped. An operator may explicitly request another bounded attempt after the underlying capability or configuration changes, but the request creates a new owner lease and preserves the lifetime attempt count.
-
-This policy bounds automatic work, not evidence lifetime. A source transaction, journal, or manifest is never deleted solely because it is old, numerous, or over a byte threshold. `operator_required` source evidence remains until its protocol reaches a proven terminal state or the legacy-journal disposition below is completed.
-
-Resolved control tombstones are retained for at least 30 days, immutable export envelopes for at least 90 days, and compact completed disposition receipts for at least 365 days. A collector may conditionally remove only a terminal artifact past its floor after proving that the bound source generation is absent where required, no nonterminal successor or active decommission references it, and the terminal audit checkpoint is durable. Capacity pressure blocks new export/disposition work rather than evicting unexpired or nonterminal evidence. Uncertainty retains the artifact; collection never authorizes source or remote deletion.
-
-### Legacy journal and manifest disposition
-
-Journal v1 has neither backend identity nor remote-version authority; v2 has backend identity but still lacks remote-version semantics. Their automatic classification is `retained_ambiguous`, and neither recovery nor an operator action may instantiate a backend or issue remote PUT, GET, probe, or DELETE from those bytes. The approved single-record actions are:
-
-- **inspect**: strictly decode a server-reconstructed canonical journal identity, perform an all-pool strong read, and return a redacted copy-set/content digest, version, quarantine reason, control classification, age information when known, topology readiness, and decommission coverage; it changes nothing and does not return raw object/version fields by default;
-- **export**: after a fresh exact inspect, create-only persist an immutable `rustfs-ilm-recovery-export-v1` envelope containing the raw source bytes and sorted copy manifest, then strongly read it back. The response downloads that envelope rather than rereading the live journal, uses no-store/attachment semantics, and never adds credentials or backend configuration;
-- **abandon after export**: v1 or v2 only; create a `Prepared` `rustfs-ilm-recovery-disposition-v1` receipt bound to the immutable export and every source copy, conditionally remove only those exact local journal generations, prove every bound copy absent with no replacement, and advance the receipt through `Applying` to `Completed`. This accepts a possible remote storage leak and never asserts that cleanup occurred.
-
-Export and abandon require a fresh all-member capability/topology proof including each member's current process epoch; inspect may remain available in a mixed fleet but returns not-ready for mutation. `abandon after export` uses POST and requires `confirm: true`, `action: abandon_remote_cleanup`, `acknowledge_remote_cleanup_abandoned: true`, the export operation ID/digest, source content and copy-set digests, every source ETag, control ETag, and a bounded operator reason code. It is refused while an active decommission or migration receipt covers either record, while physical copy discovery is incomplete, or when the implementation cannot target every discovered copy with its own `If-Match` condition.
-
-The disposition receipt has immutable action/export/control identities and an immutable sorted copy manifest. Its ETag-CAS generation contains state `Prepared`, `Applying`, or `Completed` and a monotonic sorted `confirmed_absent` set naming only entries from that manifest. Before `Prepared -> Applying`, a fresh all-pool read must find every bound copy at its exact ETag/content digest and repeat the fleet, lock, migration, and decommission checks. The manifest can never be widened, reordered, or replaced.
-
-During `Applying`, recovery treats each manifest entry independently while retaining the original all-pool boundary. An entry already in `confirmed_absent` must still be strongly absent with no successor or replacement generation. For an unconfirmed entry, an exact ETag/content match may be conditionally deleted and then added to `confirmed_absent` only after strong absence readback. If a crash or lost response left that exact path absent before the progress CAS, recovery may add it only after the same strong absence, stable source/control generation, topology, process-epoch, migration, and decommission proofs establish that no replacement exists. A different ETag/content, an unbound copy, an unreadable member, or loss of any proof is a conflict and preserves the current progress. Thus a crash after deleting copy A but before recording its progress can converge and continue with copy B without requiring deleted copy A to reappear.
-
-Immediately before each local metadata deletion, the server repeats the applicable all-pool/fleet/lock checks and conditionally targets only the still-unconfirmed exact ETag. Completion requires every immutable manifest entry in `confirmed_absent`, a fresh all-pool proof that all remain absent without replacement, unchanged fleet/process epochs, and no active decommission or migration coverage. A lost final response is success only when strong readback proves the canonical receipt `Completed`. Recovery may repeat only this canonical operation and never creates a tier client or issues a backend request.
-
-Malformed or unsupported bytes whose outer v1/v2 identity cannot be proven are inspect-only and cannot use abandon. Versions v3-v6 never use `abandon after export`. Their known candidate or manifest ownership must converge through the normal exact protocol. An operator may inspect/export and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or remote-version validation. `Preparing`/`Aborting` manifests may use their existing whole-set rollback; `DispatchAuthorized`/`Completed`, a missing member, a nonempty operation namespace, or any uncertain binding cannot be manually removed.
-
-### Admin and metrics contract
-
-The approved surface is single-record and uses a protocol-specific expected tuple; it does not reuse the legacy metadata-reconcile digest or create a bucket/prefix job:
-
-```text
-GET  /rustfs/admin/v3/ilm/recovery/records?protocol=<protocol>&classification=<classification>&limit=<n>&marker=<opaque>
-GET  /rustfs/admin/v3/ilm/recovery/records/<control-id>
-POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
-GET  /rustfs/admin/v3/ilm/recovery/exports/<export-id>
-```
-
-List and redacted inspect require `admin:ListTier`. Raw export creation/download, retry, or abandon require `admin:SetTier` because legacy bytes can reveal bucket, object, tier, and remote-version information. The default page is 100 records and the hard maximum is 1,000. A truncated page without a continuation marker is an error, and counts from an incomplete scan are labeled incomplete rather than reported as zero or complete.
-
-Inspect returns a 15-minute opaque observation receipt bound to the authenticated actor, canonical record identity, source/copy ETags and digests, topology/fleet generation, every member process epoch, action class, issue/expiry time, and nonce. It is observation evidence, not mutation authority. POST requires that receipt and `confirm: true` for terminal actions, and binds the control ETag/revision/classification, requested action, and export digest when applicable. The server repeats all live proofs; a restarted member, membership change, or process-epoch mismatch invalidates the receipt. Client fields are concurrency guards, not authority. Export download reads the immutable envelope, uses TLS plus `Cache-Control: no-store` and attachment disposition, and never logs the raw payload.
-
-Metrics use bounded labels only:
-
-- `rustfs_ilm_recovery_records{protocol,classification,schema}` and `rustfs_ilm_recovery_oldest_age_seconds{protocol,classification,schema}`;
-- `rustfs_ilm_recovery_attempts_total{protocol,outcome,error_code}`;
-- `rustfs_ilm_recovery_operator_actions_total{protocol,action,outcome}`;
-- scan completeness, corrupt-record, and orphan-control counters.
-
-Every label value is a closed enum. `schema` exposes recognized schema identifiers only; an unrecognized raw value maps to `unknown`, while a recognized future schema disabled by the current fleet maps to `unsupported`. `protocol`, `classification`, `outcome`, `error_code`, and `action` likewise map unknown input to one bounded fallback and never expose decoded or operator-provided text.
-
-Object names, bucket names, tier names, transaction IDs, control IDs, endpoints, ETags, error text, and credentials are never metric labels. Admin output may identify the selected record but redacts credentials and raw backend configuration. Audit/log events use the repository ILM event fields, stable reason codes, authenticated actor, source/control generations, action, and outcome; they do not persist or log provider response bodies.
-
-One canonical source generation counts once regardless of its physical copy count or how many recovery passes observed it. Attempt counters increment once per coordinator attempt, not per replica, CAS retry, or page revisit. Aggregate totals and oldest age are authoritative only after a complete all-pool scan; partial coverage reports `incomplete` and never publishes a false zero. A legacy record's first-seen time comes from its durable control record rather than an inferred object modification time.
-
-### Required protocol fixtures
-
-Implementation acceptance requires deterministic crash/restart and mixed-version fixtures, not timing-only tests. At minimum they cover:
-
-- a historical v1 `UploadStarted@1` whose PUT may have reached the provider, proving migration yields `UploadOutcomeUnknown` and never `AbortedNoRemote` or a direct delete;
-- every accepted v1 state/revision pair above plus checksum-valid impossible pairs such as `CleanupPending@1`, proving impossible history is inspect-only and makes zero backend calls;
-- an in-flight v1 creator interleaved with admission stop, process-epoch drain, migration, takeover, and local publication, proving no stale creator can publish after takeover;
-- `Committed` with complete, missing, partial, divergent, and unavailable all-pool `xl.meta` ownership proof, proving only the complete exact tuple permits record cleanup;
-- an old reader retaining v2, a mixed fleet blocking v2 writer and migration, and downgrade refusing until v2/control records are drained or exported;
-- operator abandon across a crash after one per-copy delete, a lost delete response, a lost progress CAS, a replacement ETag, incomplete topology, and active decommission, proving progress is monotonic, replacements survive, unsafe cases retain evidence, and every case issues zero backend PUT, GET, probe, or DELETE calls;
-- canonical export replay, a crash before candidate installation, a lost create response, concurrent admission at the remaining-byte boundary, and actor/cluster count, byte, rate, and concurrency exhaustion, proving duplicate IDs consume no new quota, projected totals never oversubscribe, and admission failure mutates no source evidence.
+- **Open:** bounded age/count policy and operator disposition for quarantined v1/v2, incomplete manifests, and repeatedly failing exact deletes. Capacity rejection and recovery throughput must not be “fixed” by weakening ownership proof.
 
 ## `xl.meta` free-version boundary
 
@@ -562,7 +417,7 @@ A bucket/prefix/fleet batch reconcile is still an **open design**. It requires a
 
 Decommission cannot treat durable ILM objects as ordinary configuration blobs. `validate_durable_ilm_record` validates namespace, size, schema/checksum, identity, and a protocol-specific checkpoint, and most protocol branches recompute the canonical path. Its transition-transaction branch currently inherits the weaker final-component parser: mismatched shard directories, extra components, and uppercase hex can pass when the final UUID and record contents agree. Exact transition-path validation is therefore an approved target, not a current decommission guarantee. Checkpoint successors enforce journal/manifest legal states, chunk-parent revision/sequence/count/binding progression, transition identity and revision progression, monotonic manual-job progress, scope ownership, and immutable task/result payloads.
 
-The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current exact ETag conditional-delete primitive. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
+The decommission coordinator copies and validates a durable record on a target, persists a receipt for that exact source path/identity/checkpoint, and records the expected receipt set on the source. While a matching decommission operation is active, protocol writers advance receipts as records change and terminal cleanup records a terminal checkpoint before deleting a covered record. Without an active decommission operation, the receipt helper creates no terminal receipt and ordinary protocol recovery proceeds with that protocol's current delete primitive: v6 journal/manifest/parent cleanup uses the exact ETag, while transition-transaction cleanup remains unconditional as documented above. Completion verifies every expected receipt and target checkpoint before the source pool can be removed.
 
 A terminal receipt is proof that an exact target copy reached a terminal checkpoint. It may authorize conditional removal of the matching source record when every active target copy is covered; it never authorizes remote DELETE. A terminal receipt on one target cannot hide a later nonterminal receipt on another target.
 
@@ -572,7 +427,7 @@ Receipts have no enum state. Their legal evolution is `absent -> checkpoint -> m
 
 | Observed state | Unique current owner | Current recovery decision | Destructive admission |
 |---|---|---|---|
-| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. Eligible v6 journal/manifest and transition-transaction records are removed only by their exact observed ETag | Receipt state grants no remote-delete authority |
+| No active decommission run | Underlying protocol owner | Protocol recovery proceeds normally and no receipt is created. An eligible v6 journal/manifest record is directly removed by exact ETag; transition-transaction cleanup follows its documented current unconditional path | Receipt state grants no remote-delete authority |
 | Source and target exact identity/checkpoint agree | Decommission coordinator for the run token | Create or CAS-advance the run-scoped receipt | Successor must be monotonic and topology-bound where required |
 | Receipt already covers the same successor | Decommission coordinator for the run token | Treat as idempotent | Exact identity/checkpoint only |
 | Conflicting receipt, checksum/schema/path error, divergent target record, missing ETag, or non-successor checkpoint | No decommission actor acquires cleanup authority | Fail decommission and retain source | Never overwrite or guess |
@@ -596,7 +451,7 @@ Receipt and expected-manifest create/CAS conflicts retry at most three times. Ex
 
 ### Approved target failure matrix
 
-The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. Transition transaction v1 now has create-only installation, exact ETag successor CAS, and conditional terminal deletion, but it still lacks mandatory lost-response exact-successor readback and a renewable durable recovery lease. The manual job's initial write remains non-create-only.
+The matrix below is the normative approved target, not a blanket description of current implementation. Current exceptions are authoritative only where each protocol section above labels them explicitly. In particular, transition-transaction initial and successor writes and deletes are currently unconditional, while the transition transaction's initial write and the manual job's initial write have neither create-only installation nor mandatory lost-response strong-readback convergence.
 
 | Event | Approved result |
 |---|---|
@@ -607,20 +462,19 @@ The matrix below is the normative approved target, not a blanket description of
 | Crash after remote DELETE but before journal/free-version cleanup | Retry the same exact idempotent DELETE under the same fences, then conditionally clean local evidence |
 | Cancellation | Stop issuing new work, persist monotonic cancellation where the protocol has it, and leave ambiguous durable records for recovery. Cancellation is never rollback proof after authorization |
 | Rolling upgrade | Gate writers on the minimum capability required by the format. Known older journal/RPC versions follow their explicit compatibility rule; unknown formats are retained |
-| Downgrade | Drain v6 journals and any enabled transition-v2/control protocol before removing their capable workers. Do not write a new format until its downgrade reader behavior and writer gate are specified |
+| Downgrade | Drain v6 journals before removing all v6-aware workers. Do not write a new format until its downgrade reader behavior and writer gate are specified |
 | Corrupt or unknown input | Record a diagnosable failure, retain bytes, and block destructive action/completion |
 
-Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have an implemented persisted-format negotiation for rolling downgrade. The approved transition-v2/control gate above is not current behavior. Until the applicable gate is implemented, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent.
+Transition transaction v1, manual job/task/result v1, and receipt v2 do not currently have a complete persisted-format negotiation for rolling downgrade. Until one is designed, caller/operator orchestration must not enable writers whose records required recovery nodes cannot decode. The manual async endpoint does not enforce that fleet gate and a direct request proceeds to job creation. This caller-side fail-closed rule is stricter than treating an unknown record as absent.
 
 ### Current format compatibility decisions
 
 | Family/version | Current reader and writer behavior | Upgrade, downgrade, and ignore rule |
 |---|---|---|
-| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. V1 remains the only writer format until the approved v2 fleet gate is implemented; a v2 reader never rewrites an active v1 record |
-| Transition transaction v2 and recovery-control/export/disposition v1 | Approved target only; no current reader or writer emits these formats | Roll out read support before the homogeneous writer gate; old readers reject and retain. Disable creation and prove all active records drained before downgrade; never rewrite v2 to v1 |
+| Transition transaction v1 | Writers emit v1; the payload decoder rejects another schema, bad checksum, unknown state, or inconsistent transaction/remote identity. The current record-path parser accepts any shard/extra-component layout and uppercase hex when the final 32-hex UUID parses and matches the payload | There is no intentional ignore path, but exact lowercase canonical-path rejection remains an approved fix. A future schema needs a fleet writer gate and an old-reader retention test before rollout; downgrade behavior is open |
 | Tier mutation intent v1; peer RPC v3/v4 | Durable readers/writers require intent v1. New peers accept signed/canonical v3 and v4 RPC; old v3 peers return an exact authenticated unsupported response to v4 | Pause and drain edit/remove/clear across the mixed interval; do not automatically retry v4 as v3. Unknown durable intent is retained and blocks recovery |
 | Manual job/scope/task/result v1 | Writers emit the v1 family. Manual-job runtime recovery accepts an uppercase UUID path when both shard strings match its uppercase prefix, then loads the lowercase canonical job by UUID; the decommission validator recomputes the canonical path and rejects that alias. Other decoder/path/checksum failures stop reconciliation. Runtime capabilities advertise `enqueue_only` and `async`, but the async run handler does not consult a fleet capability gate and a direct request creates a job | Runtime recovery still needs exact lowercase canonical-path validation to prevent alias-driven duplicate work. Caller/operator orchestration must verify every required node and fail closed when capability is unknown or unsupported. An automatic server-side fleet gate and persisted downgrade negotiation remain open; unknown records are never ignored as completed work |
-| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Never translate empty version ID to known-disabled or authorize remote DELETE. Retain unless the approved exact inspect/export/abandon protocol conditionally removes only the local journal generation |
+| Journal v1/v2 | Readers decode but quarantine because remote-version authority is missing; compatibility writers can preserve these forms | Retain indefinitely unless a separately approved, authoritative repair protocol resolves them; never translate empty version ID to known-disabled |
 | Journal v3/v4 | Readers recover supported committed records according to exact or explicit version-state semantics; current compatible writes use v4 for known state | Unknown/inconsistent state is retained. These legacy paths are not evidence that a new sole-owner operation may omit v5/v6 source proof |
 | Journal v5 | Readers use stable source/all-pool proof; decoded v5 can be checkpointed, while new online sole-owner transactions are not emitted as v5 | Retain and recover conservatively during upgrade. Do not manufacture v5 from older records or use it to bypass v6 manifest authorization |
 | Journal v6, dispatch manifest v1, and chunk parent v1 | v6-aware writers/readers require immutable manifest membership and topology. Complete sets at or below 200,000 retain the legacy root manifest bytes; larger sets install a strict parent at that root and operation-scoped v1 child payloads. Pre-chunking v6 readers reject the parent schema and child paths, while v5-and-older readers reject and retain v6 journals | Gate writers on the current fleet capability and retain the root parent for the entire active chunk sequence. Drain v6 before removing all v6-aware workers; do not downgrade by rewriting a live v6 operation |
@@ -630,10 +484,10 @@ Transition transaction v1, manual job/task/result v1, and receipt v2 do not curr
 
 | Protocol | Current operator/telemetry surface | Current retention | Required follow-up |
 |---|---|---|---|
-| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Implement the approved v2 lease/takeover, recovery-control status, bounded retry, and backlog metrics |
+| Transition transaction | Expired unknown-upload inspect/delete/finalize routes; `lifecycle_transition_transaction_recovery` diagnostics | Terminal records are deleted; ambiguous and unsafe states may remain indefinitely | Backlog age/count/state metrics, bounded policy, and durable takeover status |
 | Tier mutation intent | Admin mutation response plus recovery diagnostics; no dedicated reconcile API | Peer aborted tombstone through expiry plus skew; ambiguous coordinator/peer records retained | Status/reconcile view for mutation, peer convergence, config generation, and blocked tiers |
 | Manual job | POST run response, GET status, DELETE cancel; runtime capabilities advertise both modes | Job/task/result history is indefinite. Terminalizers only best-effort delete the exact scope; startup skips a terminal job with a leftover scope, which remains until a later admission claimant lazily replaces it | Age/count/bytes limit and a terminal-history/scope GC protocol that preserves recovery evidence |
-| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Implement the approved single-record inspect/export/disposition, bounded retry controls, and logical backlog metrics |
+| Tier-delete journal/manifest | `lifecycle_tier_delete_journal` events, quarantined counter, remote-delete failure/breaker/inflight metrics | Terminal records converge; quarantined/ambiguous records are unbounded by age | Safe operator inspection/disposition, backlog age/count by version/state, bounded recovery without evidence loss |
 | Decommission receipt | Decommission state/events including `receipt_cleanup_failed` | Completion triggers only best-effort receipt/manifest cleanup. Delete failures reported as `receipt_cleanup_failed`, as well as abandoned runs, can leave run-scoped records behind | Run-scoped retention and resume-safe cleanup policy |
 
 Retention is a protocol transition, not raw deletion. Any collector must name its unique owner, minimum age/count/bytes bound, exact terminal or quarantine predicate, readback behavior, decommission interaction, and audit/metric output. It may not collect a record solely because it is old.

docs/operations/tier-ilm-debugging.md

@@ -154,60 +154,6 @@ Historical transition transactions in `upload_outcome_unknown` state can use an
 
 `finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; the endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
 
-## Inspect and disposition retained recovery records
-
-This section describes an **approved target that is not implemented yet**. Current servers do not expose the routes below and continue to quarantine tier-delete journal v1/v2 records. Do not remove internal metadata objects by hand: that loses ETag, all-pool, decommission, export, and audit guarantees.
-
-The approved read-only inventory is bounded and paginated:
-
-```text
-GET /rustfs/admin/v3/ilm/recovery/records?protocol=<protocol>&classification=<classification>&limit=<n>&marker=<opaque>
-GET /rustfs/admin/v3/ilm/recovery/records/<control-id>
-```
-
-List and redacted inspect require `admin:ListTier`. The server reconstructs the canonical source identity, strongly reads every authoritative copy, and reports one logical record with its schema, classification (`retrying`, `retained_ambiguous`, `corrupt`, `operator_required`, `abandoned`, or `terminal`), stable reason code, copy/content digests, retry deadline/counters, fleet readiness, scan completeness, and decommission coverage. It does not return raw legacy bytes, object/version names, endpoints, credentials, or provider error text in the default JSON. Incomplete pool coverage, divergent copies, a missing ETag, corruption, or a truncated page without a continuation marker is fail-closed and cannot produce an actionable receipt.
-
-Inspect returns a 15-minute opaque observation receipt. It binds the authenticated actor, canonical record, every source copy/ETag/digest, topology/fleet generation, requested action class, issue/expiry time, and nonce. The receipt prevents a stale request from widening its target; it is not cleanup authority.
-
-For a strictly decoded v1/v2 tier-delete journal, the approved evidence-preserving flow is:
-
-1. Inspect the exact record and independently decide whether retaining the local cleanup obligation is still useful.
-2. With `admin:SetTier`, create an immutable server-side export from the current observation receipt. The export contains the exact raw journal bytes and copy manifest, is installed create-only at the canonical digest-derived export ID, strongly read back, and downloaded through a no-store attachment response:
-
-   ```text
-   POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
-   { "action": "export", "observation_receipt": "<opaque>" }
-
-   GET /rustfs/admin/v3/ilm/recovery/exports/<export-id>
-   ```
-
-3. Only after preserving that export, submit a fresh exact disposition with `admin:SetTier`:
-
-   ```json
-   POST /rustfs/admin/v3/ilm/recovery/records/<control-id>
-   {
-     "action": "abandon_remote_cleanup",
-     "confirm": true,
-     "acknowledge_remote_cleanup_abandoned": true,
-     "observation_receipt": "<opaque>",
-     "export_id": "<export-id>",
-     "export_sha256": "<sha256>",
-     "reason_code": "<bounded-operator-reason>"
-   }
-   ```
-
-The last action removes only the exact local v1/v2 journal generations by per-copy `If-Match` after a durable `Prepared` disposition receipt and fresh all-member capability proof. The receipt advances `Prepared -> Applying -> Completed` and records a monotonic per-copy `confirmed_absent` set. If the server deletes copy A and crashes before recording progress, recovery may confirm A absent under the unchanged source/control, topology, process-epoch, migration, and decommission proofs, persist that progress, and continue with still-exact copy B. A replacement ETag is always a conflict; recovery never widens the immutable copy manifest.
-
-The action never creates a tier client, probes a backend, or issues remote PUT/GET/DELETE. Its meaning is deliberately narrow: the operator accepts that remote storage may leak and abandons RustFS cleanup after preserving evidence. A changed copy, active decommission, missing member, topology/process restart, incomplete read, or uncertain replacement proof retains the evidence. Success requires every bound copy be durably confirmed absent, a fresh all-member/decommission proof, and the disposition receipt durably `Completed`; response loss resumes only the same canonical operation ID.
-
-Canonical replay of an identical export/disposition consumes no new quota. New operations require a complete artifact inventory and are refused before source mutation when the projected retained total, including the fully encoded candidate, would exceed 10,000 exports, 10,000 disposition receipts, 1 GiB of encoded export data, or 256 MiB of encoded control/disposition data. The quota decision, create-only installation, and exact readback share one cluster-scoped admission WRITE lock. That lock is always acquired before control/source/disposition and physical metadata locks and is released before disposition `Applying` or any source deletion; callers never acquire it while holding those inner guards. A crash before installation consumes no capacity, and lost installation response is resolved by canonical readback under the same serialized order, so concurrent nodes cannot oversubscribe a stale snapshot. Admission is also limited to ten new creations per actor per minute, 100 cluster-wide per minute, 32 concurrent exports, and eight concurrent dispositions. Capacity pressure never evicts recovery evidence or blocks ordinary object I/O; the collector examines at most 100 terminal artifacts per minute.
-
-Malformed/unsupported records and journal v3-v6 cannot use abandon. Known-version and v6 manifest ownership must converge through their normal exact recovery protocol. Operators may inspect, export, and request a bounded retry, but cannot bypass source/free-version proof, manifest membership, topology, or version semantics.
-
-Automatic retry state survives restart. Retryable transport/quorum failures use a 60-second exponential base capped at one hour and a deterministic 80-to-100-percent multiplier, so jitter never increases the capped delay. After 32 consecutive failures or seven days from the first persisted failure, automatic work stops at `operator_required`. Unsupported or ambiguous evidence goes directly to `retained_ambiguous`/`operator_required`; age alone never deletes it. Resolved controls, immutable exports, and completed disposition receipts have minimum 30-day, 90-day, and 365-day retention respectively, and are collected only after exact source absence, decommission, successor, and audit checks.
-
-The full schema, lease, mixed-version, retry, privacy, and metric requirements are in [../architecture/ilm-tiering-persistence-contracts.md](../architecture/ilm-tiering-persistence-contracts.md#bounded-recovery-control-and-operator-disposition).
-
 ## Reconcile legacy transition-version metadata
 
 This section describes an **approved target that is not implemented yet**. The current server has no admin route that backfills a missing `transitioned-version-state` in `xl.meta`. Do not use the transaction reconcile route above for this purpose: that route owns an upload transaction candidate and may delete it, while legacy metadata reconciliation is non-destructive and may update only the exact local metadata version.

docs/testing/README.md

@@ -12,10 +12,10 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
 |---|---|---|---|
 | Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p <crate>`); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) |
 | ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) |
-| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) |
+| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`, `e2e-distributed`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); 4-node 4-disk map: [distributed-e2e.md](distributed-e2e.md) |
 | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` | PR, report-only (second half of the `End-to-End Tests` job) |
 | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only |
-| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree |
+| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/offline-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos with the `e2e-distributed` nightly lane |
 | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus |
 | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p <crate>` | On demand; never a gate |
 
@@ -61,6 +61,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the
 | `e2e-full` | Merge-queue / main-push single-node e2e lane |
 | `e2e-repl-nightly` | Nightly slow / cross-process replication lane |
 | `e2e-nightly` | Nightly serial multi-process cluster fault lane |
+| `e2e-distributed` | Nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
 | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` |
 
 Membership of each e2e profile is pinned by a digest in `.config/e2e-<profile>-selection.txt` and checked by `scripts/check_test_wiring.py --check-profile <profile>` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only):

docs/testing/ci-gates.md

@@ -74,6 +74,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` |
 | `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
+| `e2e-distributed.yml` (nightly) | `distributed` | 4-node 4-disk e2e gate including direct/rolling upgrade of historical objects and IAM AK/SK; JUnit, membership listing, server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` |
 | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
 | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
 | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |

docs/testing/distributed-e2e.md

@@ -0,0 +1,70 @@
+# Distributed 4-node 4-disk e2e
+
+**Use this when:** adding or diagnosing GitHub Actions coverage for a 4-node cluster, or deciding whether a behaviour belongs in `e2e-distributed` versus the single-node `e2e-full` lane, the nightly cluster-fault lane, or the hardware functional chain.
+**Source of truth:** `crates/e2e_test/src/distributed/`, `[profile.e2e-distributed]` in `.config/nextest.toml`, `.github/workflows/e2e-distributed.yml`.
+
+## Topology
+
+The in-tree harness runs every node on `127.0.0.1` with a distinct port. That matches `RustFSTestClusterEnvironment` in `crates/e2e_test/src/common.rs`:
+
+| Layout | Constructor | Use |
+|---|---|---|
+| 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos |
+| 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release |
+| 2 single-node pools × 4 drives | `ClusterTopology::per_node_pools(4, [[0],[1]])` | Harness-only: `append_single_node_pool` unit tests. Live multi-pool expand/restart currently dies with `pool metadata recovery required`; this lane does not change that production gate |
+
+A pool striped across several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped pools remain the hardware functional-chain / backlog #1313 / #1314 lane.
+
+Decommission and rebalance POST on the 4×4 single-pool layout is refused by the current product (`single pool deployments do not support decommission`, NotImplemented, or opaque 500 InternalError when the inner pool-meta fence is wrapped). 502/503 are not treated as a product refusal. Those cases still assert object bytes and SHA-256; when the API starts they wait for completion and assert post-move integrity. They do not treat a refusal as a successful move. This lane does not change production pool-meta bootstrap, write-fence, or decommission policy; it only observes the current server behavior.
+
+## What this lane covers
+
+`cargo nextest run --profile e2e-distributed -p e2e_test` selects `distributed::*`:
+
+- S3 put / get / head / list / copy / rename / delete / presign / empty object
+- Object Lock COMPLIANCE, GOVERNANCE (with bypass), legal hold
+- Versioning, version GET, delete marker
+- Bucket replication between two 4-node clusters; hard quota
+- Health / admin info / storageinfo / audit target list
+- Pool restart, decommission/rebalance *attempts*, checksum integrity, S3 during those attempts on 4×4. Live multi-pool expand/restart is a production pool-meta bootstrap limitation and is not patched here; `append_single_node_pool` is covered by harness unit tests
+- Site replication object convergence
+- High-concurrency PUT/GET; concurrent PUT during decommission
+- Node kill/restart, full process restart, drive offline (4×4). Volume-proxy blackhole stays in `cluster_volume_fault_proxy_pass_smoke` (2×2); a 4-node volume proxy cannot format because RPC audience is the listen port
+- Multipart, cross-node listing, list-buckets agreement
+- Concurrent GET while a peer node is killed
+- Direct and rolling upgrade from the pinned previous release: historical objects, versioned history, and IAM user AK/SK still work afterwards
+
+## Existing Actions gaps this lane does not replace
+
+Those suites stay in place; this lane fills the in-tree 4×4 hole they leave.
+
+| Existing lane | Gap |
+|---|---|
+| `rustfs-*-test.yml` functional chain | Clones private `rustfs/auto-testing`, runs on three shared VMs (`vm000`–`vm002`), `continue-on-error: true`, not a merge signal, not 4 nodes. Hardware `rustfs-upgrade-test.yml` stays there |
+| `e2e-upgrade.yml` | Single-node SSE/multipart/delete-marker contracts plus mixed-version listing; does not pin IAM user AK/SK on a 4-node cluster |
+| `e2e-smoke` / `e2e-full` | Almost all cases are single-node |
+| `e2e-nightly` | 4-node cluster faults and heal, not S3/lock/versioning/quota/decommission matrix |
+| `e2e-repl-nightly` | Site and bucket replication on 1–3 *single-node* processes |
+| `e2e-s3tests.yml` `multi` | Weekly ceph/s3-tests against Docker 4-node; not lock/WORM, decommission, chaos, or checksum integrity |
+| `crates/e2e_test/src/chaos.rs` | Single-node disk faults only |
+
+Hardware power-loss, NIC pull, and real disk replacement still belong on the smoke-testing VMs. This lane simulates those with SIGKILL, directory rename, and `FaultProxy` blackhole.
+
+## Run
+
+```bash
+cargo build -p rustfs --bins
+# Upgrade cases require the pinned previous binary (CI downloads it).
+export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2
+cargo nextest run --profile e2e-distributed -p e2e_test
+```
+
+Without `RUSTFS_UPGRADE_SOURCE_BINARY` the two `distributed::upgrade_test::*` cases fail closed. Filter them out for a local run that is not checking upgrade:
+
+```bash
+cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distributed::upgrade_test::/)'
+```
+
+The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check.
+
+Membership is pinned by `.config/e2e-distributed-selection.txt`. Update it with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed <listing.json> linux` after adding or renaming a case.