Merge branch 'main' into test/distributed-e2e-hardening

test(odm): exercise overflow and invalid cursors reliably (#7236 )
test(odm): provide the source region in access fixture (#7235 )
2026-09-05 19:55:37 +00:00 · 2026-09-06 03:32:13 +08:00 · 2026-09-06 03:09:39 +08:00 · 2026-09-06 02:34:56 +08:00 · 2026-09-06 02:24:04 +08:00 · 2026-09-05 23:56:44 +08:00
39 changed files with 4202 additions and 521 deletions
@@ -0,0 +1,2 @@
+sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
+sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
@@ -183,6 +183,13 @@ test-group = 'e2e-reliability'
 filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
 test-group = 'e2e-inline-boundaries'

+# 4-node 4-drive distributed Actions suite: each case starts four rustfs
+# processes and up to sixteen data directories. Serialize across nextest's
+# process boundary so several 4x4 clusters never overlap.
+[[profile.default.overrides]]
+filter = 'package(e2e_test) & test(/^distributed::/)'
+test-group = 'e2e-cluster-nightly'
+
 # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
 # does not cross nextest process boundaries, so keep every Vault-backed test in
 # one group.
@@ -526,6 +533,27 @@ path = "junit.xml"
 filter = 'package(e2e_test)'
 test-group = 'e2e-cluster-nightly'

+# ---------------------------------------------------------------------------
+# e2e-distributed profile — 4-node 4-disk Actions suite
+# ---------------------------------------------------------------------------
+# Storage-sensitive PR / nightly / dispatch lane owned by
+# .github/workflows/e2e-distributed.yml.
+# Each case starts four rustfs processes (and for site replication, two
+# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
+# Serialized via e2e-cluster-nightly with no retries.
+[profile.e2e-distributed]
+default-filter = 'package(e2e_test) & test(/^distributed::/)'
+fail-fast = false
+# Decommission / rebalance cases poll for up to 180s with little stdout.
+slow-timeout = { period = "120s", terminate-after = 6 }
+
+[profile.e2e-distributed.junit]
+path = "junit.xml"
+
+[[profile.e2e-distributed.overrides]]
+filter = 'package(e2e_test)'
+test-group = 'e2e-cluster-nightly'
+
 # ---------------------------------------------------------------------------
 # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
 # ---------------------------------------------------------------------------
@@ -586,6 +614,10 @@ path = "junit.xml"
 #     cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
 #     excluded here because backlog#2213 promotes core heal rebuild coverage to
 #     this merge/main lane while retaining nightly coverage.
+#   * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
+#     replication, quota, observability, expand/decommission/rebalance, site
+#     replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
+#     .github/workflows/e2e-distributed.yml.
 #   * on_demand_migration::interop_test — the ODM-20 provider interoperability
 #     cases, which are meaningless without a source: they run in the dedicated
 #     [profile.e2e-odm-interop] lane below, where the workflow points them at a
@@ -607,6 +639,7 @@ default-filter = """
    package(e2e_test)
    & !test(/^protocols::/)
    & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
+    & !test(/^distributed::/)
    & !test(/^replication_extension_test::/)
    & !test(/^replication_target_matrix_test::/)
    & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
@@ -4,6 +4,11 @@
  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  {
+    "workflow": ".github/workflows/e2e-distributed.yml",
+    "max_age_hours": 36,
+    "never_ran_grace_until": "2026-09-18T00:00:00Z"
+  },
  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
@@ -0,0 +1,199 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 4-node 4-disk distributed e2e lane.
+#
+# Each selected test starts a real localhost cluster via
+# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
+# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
+# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
+# nightly runs, and manual dispatches all execute the same fail-closed suite.
+# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
+
+name: e2e-distributed
+
+on:
+  pull_request:
+    paths:
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - ".config/nextest.toml"
+      - ".github/workflows/e2e-distributed.yml"
+      - "crates/audit/**"
+      - "crates/common/**"
+      - "crates/config/**"
+      - "crates/e2e_test/**"
+      - "crates/ecstore/**"
+      - "crates/filemeta/**"
+      - "crates/heal/**"
+      - "crates/iam/**"
+      - "crates/lock/**"
+      - "crates/madmin/**"
+      - "crates/notify/**"
+      - "crates/replication/**"
+      - "crates/s3-client/**"
+      - "crates/s3-ops/**"
+      - "crates/s3-types/**"
+      - "crates/scanner/**"
+      - "crates/storage-api/**"
+      - "crates/utils/**"
+      - "rustfs/**"
+  workflow_dispatch:
+    inputs:
+      filter:
+        description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
+        required: false
+        default: ""
+  schedule:
+    # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
+    - cron: "53 5 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'schedule' }}
+
+jobs:
+  distributed:
+    name: Distributed 4-node 4-disk e2e
+    runs-on: sm-standard-4
+    timeout-minutes: 180
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
+      NO_PROXY: 127.0.0.1,localhost
+      HTTP_PROXY: ""
+      HTTPS_PROXY: ""
+      # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
+      UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
+      UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
+      UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+
+      - name: Setup Rust environment
+        uses: ./.github/actions/setup
+        with:
+          rust-version: stable
+          cache-shared-key: ci-e2e-distributed
+          cache-save-if: ${{ github.ref == 'refs/heads/main' }}
+          install-build-packaging-tools: 'false'
+
+      - name: Prepare isolated filesystems for pool movement
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          mkdir -p "${mount_base}"
+          roots=()
+          for pool in 0 1 2 3; do
+            image="${mount_base}/pool-${pool}.img"
+            mountpoint="${mount_base}/pool-${pool}"
+            truncate -s 1G "${image}"
+            mkfs.ext4 -q -F "${image}"
+            mkdir -p "${mountpoint}"
+            sudo mount -o loop,nosuid,nodev "${image}" "${mountpoint}"
+            sudo chmod 1777 "${mountpoint}"
+            roots+=("${mountpoint}")
+          done
+          printf -v joined_roots '%s:' "${roots[@]}"
+          echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[0]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[1]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[2]}"
+          findmnt --noheadings --output TARGET,SOURCE,FSTYPE --target "${roots[3]}"
+
+      - name: Download pinned previous release
+        env:
+          SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
+        run: |
+          set -euo pipefail
+          mkdir -p "$SOURCE_DIR"
+          archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
+          curl --fail --location --retry 3 --output "$archive" \
+            "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
+          echo "$UPGRADE_SOURCE_SHA256  $archive" | sha256sum --check --strict
+          unzip -q "$archive" -d "$SOURCE_DIR"
+          chmod +x "$SOURCE_DIR/rustfs"
+          test -x "$SOURCE_DIR/rustfs"
+          echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
+
+      - name: Build rustfs binary
+        run: |
+          cargo build -p rustfs --bins
+          : > target/debug/rustfs.features
+
+      - name: Verify distributed e2e membership
+        env:
+          NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+        run: |
+          cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
+          python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
+
+      - name: Run distributed 4-node e2e suite
+        env:
+          RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
+          FILTER: ${{ inputs.filter }}
+        run: |
+          set -euo pipefail
+          if [ -n "${FILTER}" ]; then
+            cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
+          else
+            cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
+          fi
+
+      - name: Upload distributed e2e diagnostics
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: e2e-distributed-${{ github.run_number }}
+          path: |
+            target/nextest/e2e-distributed/junit.xml
+            ${{ runner.temp }}/rustfs-e2e-distributed-list.json
+            ${{ runner.temp }}/rustfs-e2e-distributed-logs/
+          retention-days: 7
+          if-no-files-found: warn
+
+      - name: Unmount isolated pool filesystems
+        if: always()
+        run: |
+          set -euo pipefail
+          mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
+          for pool in 0 1 2 3; do
+            mountpoint="${mount_base}/pool-${pool}"
+            if mountpoint --quiet "${mountpoint}"; then
+              sudo umount "${mountpoint}"
+            fi
+          done
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [distributed]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -22,6 +22,7 @@ on:
      - "Continuous Integration"
      - "coverage"
      - "e2e-nightly"
+      - "e2e-distributed"
      - "e2e-s3tests"
      - "Fuzz"
      - "mint"
@@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
 | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
 | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
 | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
+| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
 | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
 | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |

@@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker.
 | KMS suite | `e2e-full` job, merge queue + main | **Active** |
 | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
 | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
+| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
 | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
 | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
 | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
@@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test
 cargo nextest run --profile e2e-full -p e2e_test
 # Cluster fault nightly lane
 cargo nextest run --profile e2e-nightly -p e2e_test
+# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
+# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
+cargo nextest run --profile e2e-distributed -p e2e_test
 # Replication nightly lane; awscurl is required for STS paths
 cargo nextest run --profile e2e-repl-nightly -p e2e_test
 # Fixed-port protocol nightly lane
@@ -1700,6 +1700,69 @@ impl RustFSTestClusterEnvironment {
        Ok(())
    }

+    /// Append a new single-node erasure pool to a stopped multi-pool cluster.
+    ///
+    /// Used to simulate pool expansion on localhost: every pool already owns
+    /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
+    /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
+    /// allocated a fresh port and empty drive directories; callers must
+    /// [`Self::start`] afterwards so every process picks up the extended
+    /// volumes argument. Existing data directories are left untouched.
+    pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
+        if self.nodes.iter().any(|node| node.process.is_some()) {
+            return Err("stop the cluster before appending a pool".into());
+        }
+        if self.topology.drives_per_node < 2 {
+            return Err(
+                "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
+                    .into(),
+            );
+        }
+
+        let mut pools = self.topology.normalized_pools();
+        for (pool_idx, nodes) in pools.iter().enumerate() {
+            if nodes.len() != 1 {
+                return Err(format!(
+                    "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
+                    nodes.len()
+                )
+                .into());
+            }
+        }
+
+        let new_idx = self.nodes.len();
+        let port = RustFSTestEnvironment::find_available_port().await?;
+        let address = format!("127.0.0.1:{port}");
+        let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
+            .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
+            .collect();
+        for dir in &data_dirs {
+            fs::create_dir_all(dir).await?;
+        }
+
+        self.nodes.push(ClusterNode {
+            url: format!("http://{address}"),
+            address,
+            data_dir: data_dirs[0].clone(),
+            data_dirs,
+            pool_idx: pools.len(),
+            process: None,
+        });
+        pools.push(vec![new_idx]);
+        self.topology.node_count = self.nodes.len();
+        self.topology.pools = pools;
+        self.node_extra_env.push(Vec::new());
+        self.node_capture_log_paths.push(None);
+        self.volume_proxy_addresses.push(None);
+
+        if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
+            self.extra_env
+                .push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
+        }
+
+        Ok(new_idx)
+    }
+
    /// Gracefully stop one cluster node and wait for its process to exit.
    ///
    /// This is intentionally separate from [`Self::stop_node`]: the latter is
@@ -0,0 +1,222 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
+    wait_for_ready, wait_until,
+};
+use crate::chaos::{census_object_version_on_disk, signed_admin_post};
+use crate::common::{build_test_s3_config, init_logging};
+use crate::fault_proxy::FaultMode;
+use aws_sdk_s3::Client;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{Barrier, mpsc};
+use tokio::time::timeout;
+
+#[tokio::test]
+async fn kill_and_restart_node_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("killnode");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x11u8; 128 * 1024];
+    put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
+
+    dist.cluster.stop_node(3)?;
+    retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn full_cluster_restart_preserves_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("pwr");
+    dist.create_bucket(&bucket).await?;
+    let body = vec![0x44u8; 64 * 1024];
+    put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
+
+    dist.cluster.stop();
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
+    let bucket = unique_bucket("baddrive");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
+
+    let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
+    let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(
+        baseline.is_complete(),
+        "replacement target did not hold a complete baseline shard: {baseline:?}"
+    );
+    assert!(
+        !baseline.expected_part_numbers.is_empty(),
+        "replacement witness must use physical part shards: {baseline:?}"
+    );
+
+    dist.cluster.stop_node(0)?;
+    let format_path = replaced_drive.join(".rustfs.sys/format.json");
+    let format = std::fs::read(&format_path)?;
+    let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
+    std::fs::rename(&replaced_drive, &retired_drive)?;
+    std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
+    std::fs::write(&format_path, format)?;
+    let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+    assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
+
+    dist.cluster.start_node(0).await?;
+    wait_for_ready(&dist.cluster).await?;
+    let heal_body =
+        r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
+    let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
+    signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
+    wait_until(
+        Duration::from_secs(90),
+        || async {
+            let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
+            Ok(healed.matches_manifest(&baseline))
+        },
+        "fresh replacement contains the original complete shard manifest",
+    )
+    .await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("getkill");
+    dist.create_bucket(&bucket).await?;
+    let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
+    put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
+
+    let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
+    let worker_count = 12;
+    let release = Arc::new(Barrier::new(worker_count + 1));
+    let (started_tx, mut started_rx) = mpsc::unbounded_channel();
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let client = live[idx % live.len()].clone();
+        let bucket = bucket.clone();
+        let body = body.clone();
+        let release = release.clone();
+        let started_tx = started_tx.clone();
+        handles.push(tokio::spawn(async move {
+            let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
+            if response.content_length() != Some(body.len() as i64) {
+                return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
+                    format!("worker {idx} received a wrong content length").into(),
+                );
+            }
+            started_tx.send(idx)?;
+            release.wait().await;
+            let actual = response.body.collect().await?.into_bytes();
+            if actual.as_ref() != body.as_slice() {
+                return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
+            }
+            Ok(())
+        }));
+    }
+    drop(started_tx);
+    for _ in 0..worker_count {
+        timeout(Duration::from_secs(30), started_rx.recv())
+            .await?
+            .ok_or("a streaming GET exited before reaching the kill barrier")?;
+    }
+
+    dist.cluster.stop_node(3)?;
+    release.wait().await;
+    for handle in handles {
+        handle.await??;
+    }
+
+    dist.cluster.start_node(3).await?;
+    wait_for_ready(&dist.cluster).await?;
+    assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
+    let proxied_url = format!("http://{}", proxy.local_addr());
+    let proxied_client = Client::from_conf(build_test_s3_config(
+        &proxied_url,
+        &dist.cluster.access_key,
+        &dist.cluster.secret_key,
+        None,
+        "distributed-network-chaos",
+    ));
+
+    let result: TestResult = async {
+        let bucket = unique_bucket("netfault");
+        dist.create_bucket(&bucket).await?;
+        let baseline = payload_for("network/baseline.bin", 1024 * 1024);
+        put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
+        assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
+
+        proxy.set_mode(FaultMode::Blackhole);
+        assert_eq!(proxy.mode(), FaultMode::Blackhole);
+        if let Ok(Ok(_)) = timeout(
+            Duration::from_secs(5),
+            proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
+        )
+        .await
+        {
+            return Err("blackholed node endpoint unexpectedly completed a GET".into());
+        }
+
+        let during = payload_for("network/during.bin", 1024 * 1024);
+        timeout(Duration::from_secs(30), async {
+            put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
+            assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
+            assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
+        })
+        .await??;
+
+        proxy.set_mode(FaultMode::Pass);
+        retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
+        Ok(())
+    }
+    .await;
+
+    proxy.set_mode(FaultMode::Pass);
+    proxy.shutdown().await;
+    result
+}
@@ -0,0 +1,98 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
+use crate::common::init_logging;
+use std::collections::BTreeSet;
+use std::sync::Arc;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("conc");
+    dist.create_bucket(&bucket).await?;
+    let clients = Arc::new(dist.clients()?);
+    let worker_count = 24;
+    let rounds = 4;
+    let barrier = Arc::new(Barrier::new(worker_count));
+
+    let mut handles = Vec::new();
+    for idx in 0..worker_count {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let writer = &clients[idx % clients.len()];
+            let reader = &clients[(idx + 1) % clients.len()];
+            let copier = &clients[(idx + 2) % clients.len()];
+            let mut retained = Vec::with_capacity(rounds);
+            for round in 0..rounds {
+                let key = format!("source/worker-{idx:02}-round-{round}.bin");
+                let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
+                let body = payload_for(&key, 64 * 1024);
+                put_object(writer, &bucket, &key, body.clone()).await?;
+
+                let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
+                if head.content_length() != Some(body.len() as i64) {
+                    return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
+                }
+                assert_object_bytes(reader, &bucket, &key, &body).await?;
+
+                copier
+                    .copy_object()
+                    .bucket(&bucket)
+                    .key(&copy_key)
+                    .copy_source(format!("{bucket}/{key}"))
+                    .send()
+                    .await?;
+                assert_object_bytes(writer, &bucket, &copy_key, &body).await?;
+
+                writer.delete_object().bucket(&bucket).key(&key).send().await?;
+                let missing = reader
+                    .head_object()
+                    .bucket(&bucket)
+                    .key(&key)
+                    .send()
+                    .await
+                    .expect_err("deleted source key must not remain visible");
+                if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
+                    return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
+                }
+                retained.push((copy_key, body));
+            }
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
+        }));
+    }
+
+    let mut inventory = Vec::new();
+    for handle in handles {
+        inventory.extend(handle.await??);
+    }
+
+    let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
+    for (node_idx, client) in clients.iter().enumerate() {
+        let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
+        let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+        assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
+        for (key, body) in &inventory {
+            assert_object_bytes(client, &bucket, key, body)
+                .await
+                .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
+        }
+    }
+    Ok(())
+}
@@ -0,0 +1,74 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
+    unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
+};
+use crate::common::init_logging;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Barrier;
+
+#[tokio::test]
+async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("concdecom");
+    dist.create_bucket(&bucket).await?;
+    let baseline_client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+
+    let clients = Arc::new(dist.clients()?);
+    let barrier = Arc::new(Barrier::new(17));
+    let mut handles = Vec::new();
+    for idx in 0..16 {
+        let clients = clients.clone();
+        let barrier = barrier.clone();
+        let bucket = bucket.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let client = &clients[idx % clients.len()];
+            let key = format!("live/{idx:02}.bin");
+            let body = payload_for(&key, 8 * 1024);
+            retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
+            Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
+        }));
+    }
+
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    barrier.wait().await;
+
+    let mut live_objects = Vec::new();
+    for handle in handles {
+        live_objects.push(handle.await??);
+    }
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let checker = dist.client(2)?;
+    assert_inventory(&checker, &bucket, &inventory).await?;
+    for (key, body) in live_objects {
+        retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,156 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
+    sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("integrity");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    enable_versioning(&client, &bucket).await?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
+
+    let versioned_key = "history/versioned.bin";
+    let version_one = b"historical bytes before data movement".to_vec();
+    let version_two = b"current bytes before data movement".to_vec();
+    let version_one_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_one.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("historical PUT omitted version ID")?
+        .to_string();
+    let version_two_id = client
+        .put_object()
+        .bucket(&bucket)
+        .key(versioned_key)
+        .body(ByteStream::from(version_two.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("current PUT omitted version ID")?
+        .to_string();
+
+    let multipart_key = "multipart/moved.bin";
+    let first_part = vec![0x31; 5 * 1024 * 1024];
+    let second_part = vec![0x72; 1024 * 1024];
+    let upload = client
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
+    let uploaded_one = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(first_part.clone()))
+        .send()
+        .await?;
+    let uploaded_two = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .part_number(2)
+        .body(ByteStream::from(second_part.clone()))
+        .send()
+        .await?;
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .upload_id(upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after_client = dist.client(2)?;
+    assert_inventory(&after_client, &bucket, &inventory).await?;
+    for (key, expected_hash) in before {
+        let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
+        let body = got.body.collect().await?.into_bytes();
+        assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
+    }
+    for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
+        let got = after_client
+            .get_object()
+            .bucket(&bucket)
+            .key(versioned_key)
+            .version_id(version_id)
+            .send()
+            .await?
+            .body
+            .collect()
+            .await?
+            .into_bytes();
+        assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
+    }
+    let mut expected_multipart = first_part;
+    expected_multipart.extend_from_slice(&second_part);
+    let got_multipart = after_client
+        .get_object()
+        .bucket(&bucket)
+        .key(multipart_key)
+        .send()
+        .await?
+        .body
+        .collect()
+        .await?
+        .into_bytes();
+    assert_eq!(
+        sha256_hex(got_multipart.as_ref()),
+        sha256_hex(&expected_multipart),
+        "multipart checksum changed after decommission"
+    );
+    Ok(())
+}
@@ -0,0 +1,81 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
+    put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
+    wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("expand");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
+    assert_inventory(&client, &bucket, &inventory).await?;
+
+    for expected_nodes in 2..=4 {
+        let new_node = dist.append_pool_and_restart().await?;
+        assert_eq!(new_node + 1, expected_nodes);
+        assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
+    }
+    assert_eq!(dist.cluster.nodes.len(), 4);
+
+    // Prove that the expanded pool map is durable, and clear any recovery
+    // latch raised while the newly-added pool replicas converged.
+    dist.restart_current_binary_gracefully().await?;
+
+    let after_expand = dist.client(0)?;
+    assert_inventory(&after_expand, &bucket, &inventory).await?;
+    let peer = dist.client(3)?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&peer, &bucket, &inventory).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("decom");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    let pools_before = list_pools_json(&dist.cluster).await?;
+    let pool_count = pools_before
+        .as_array()
+        .map(Vec::len)
+        .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
+        .ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
+    assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+
+    let after = dist.client(2)?;
+    assert_inventory(&after, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,149 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
+};
+use crate::common::init_logging;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("extra");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+
+    let key = "multipart.bin";
+    let part1 = vec![0x41u8; 5 * 1024 * 1024];
+    let part2 = vec![0x42u8; 5 * 1024 * 1024];
+    let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
+    let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
+
+    let uploaded1 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(1)
+        .body(ByteStream::from(part1.clone()))
+        .send()
+        .await?;
+    let uploaded2 = client
+        .upload_part()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .part_number(2)
+        .body(ByteStream::from(part2.clone()))
+        .send()
+        .await?;
+
+    client
+        .complete_multipart_upload()
+        .bucket(&bucket)
+        .key(key)
+        .upload_id(&upload_id)
+        .multipart_upload(
+            CompletedMultipartUpload::builder()
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(1)
+                        .e_tag(uploaded1.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .parts(
+                    CompletedPart::builder()
+                        .part_number(2)
+                        .e_tag(uploaded2.e_tag().unwrap_or_default())
+                        .build(),
+                )
+                .build(),
+        )
+        .send()
+        .await?;
+
+    let mut expected = part1;
+    expected.extend_from_slice(&part2);
+    for node_idx in 0..dist.cluster.nodes.len() {
+        assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
+    }
+
+    put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
+    put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
+    let mut seen = Vec::new();
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let listed = dist
+            .client(node_idx)?
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("list/")
+            .send()
+            .await?;
+        let keys: Vec<String> = listed
+            .contents()
+            .iter()
+            .filter_map(|object| object.key().map(str::to_string))
+            .collect();
+        seen.push(keys);
+    }
+    for keys in &seen[1..] {
+        assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
+    }
+
+    let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
+    assert_eq!(got, b"a");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("listed");
+    dist.create_bucket(&bucket).await?;
+    put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
+
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let client = dist.client(node_idx)?;
+        let name = bucket.clone();
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = client.clone();
+                let name = name.clone();
+                async move {
+                    let listed = client.list_buckets().send().await?;
+                    Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
+                }
+            },
+            &format!("node {node_idx} lists {bucket}"),
+        )
+        .await?;
+        wait_until(
+            Duration::from_secs(20),
+            || {
+                let client = dist.client(node_idx).expect("client");
+                let name = bucket.clone();
+                async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
+            },
+            &format!("node {node_idx} reads seed.bin"),
+        )
+        .await?;
+    }
+    Ok(())
+}
@@ -0,0 +1,35 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node 4-drive distributed e2e coverage.
+//!
+//! Selected by `[profile.e2e-distributed]` and run from
+//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
+//! each case starts four real `rustfs` processes.
+
+mod chaos_test;
+mod concurrency_stability_test;
+mod concurrent_data_movement_test;
+mod data_integrity_movement_test;
+mod expand_decommission_rebalance_test;
+mod extra_test;
+mod harness;
+mod object_lock_test;
+mod observability_test;
+mod replication_quota_test;
+mod s3_basic_test;
+mod s3_during_data_movement_test;
+mod site_replication_test;
+mod upgrade_test;
+mod versioning_test;
@@ -0,0 +1,219 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
+use crate::common::init_logging;
+use crate::object_lock::common::{
+    delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::error::SdkError;
+use aws_sdk_s3::operation::delete_object::DeleteObjectError;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{
+    DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
+    ObjectLockRule,
+};
+use chrono::{Duration as ChronoDuration, Utc};
+
+fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
+    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+    if code == Some("AccessDenied") {
+        Ok(())
+    } else {
+        Err(format!("{context}: expected AccessDenied, got {error:?}").into())
+    }
+}
+
+async fn expect_versioned_delete_denied(
+    client: &Client,
+    bucket: &str,
+    key: &str,
+    version_id: &str,
+    bypass: bool,
+    context: &str,
+) -> TestResult {
+    match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
+        Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
+        Err(error) => delete_denied(error.as_ref(), context),
+    }
+}
+
+#[tokio::test]
+async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let client = dist.client(0)?;
+    let peer = dist.client(2)?;
+    let bucket = unique_bucket("objlock");
+
+    client
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+
+    let retain_until = Utc::now() + ChronoDuration::days(1);
+
+    let compliance_key = "compliance.bin";
+    let compliance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        compliance_key,
+        b"locked-compliance",
+        ObjectLockRetentionMode::Compliance,
+        retain_until,
+    )
+    .await?;
+
+    // Unversioned DELETE is allowed: it only creates a delete marker. WORM
+    // applies to a specific version id.
+    let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
+    assert_eq!(
+        marker.delete_marker(),
+        Some(true),
+        "unversioned DELETE on a locked object must create a delete marker"
+    );
+
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
+        .await?;
+    expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
+
+    let governance_key = "governance.bin";
+    let governance_version = put_object_with_retention(
+        &client,
+        &bucket,
+        governance_key,
+        b"locked-governance",
+        ObjectLockRetentionMode::Governance,
+        retain_until,
+    )
+    .await?;
+
+    expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
+        .await?;
+    delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
+    let deleted_governance = peer
+        .head_object()
+        .bucket(&bucket)
+        .key(governance_key)
+        .version_id(&governance_version)
+        .send()
+        .await
+        .expect_err("GOVERNANCE bypass must remove the retained version");
+    assert_eq!(
+        deleted_governance.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
+    );
+
+    let hold_key = "legal-hold.bin";
+    let hold_version =
+        put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
+    expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let bucket = unique_bucket("default-lock");
+
+    writer
+        .create_bucket()
+        .bucket(&bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+    put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
+
+    let key = "default-governance.bin";
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"default retention payload"))
+        .send()
+        .await?;
+    let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
+
+    let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
+    let default_retention = config
+        .object_lock_configuration()
+        .and_then(|configuration| configuration.rule())
+        .and_then(|rule| rule.default_retention())
+        .ok_or("GetObjectLockConfiguration omitted default retention")?;
+    assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    assert_eq!(default_retention.days(), Some(1));
+
+    let retention = reader
+        .get_object_retention()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(version_id)
+        .send()
+        .await?;
+    let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
+    assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
+    let retain_until = retention
+        .retain_until_date()
+        .ok_or("default retention omitted retain-until date")?;
+    assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
+
+    let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
+    assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
+    expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
+        .await?;
+
+    let plain_bucket = unique_bucket("no-lock");
+    dist.create_bucket(&plain_bucket).await?;
+    let configuration = ObjectLockConfiguration::builder()
+        .object_lock_enabled(ObjectLockEnabled::Enabled)
+        .rule(
+            ObjectLockRule::builder()
+                .default_retention(
+                    DefaultRetention::builder()
+                        .mode(ObjectLockRetentionMode::Governance)
+                        .days(1)
+                        .build(),
+                )
+                .build(),
+        )
+        .build();
+    let error = writer
+        .put_object_lock_configuration()
+        .bucket(&plain_bucket)
+        .object_lock_configuration(configuration)
+        .send()
+        .await
+        .expect_err("an unversioned bucket must reject Object Lock enablement");
+    let service_error = error
+        .as_service_error()
+        .ok_or("non-lock bucket rejection was not an S3 service error")?;
+    assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
+    assert_eq!(
+        service_error.message(),
+        Some("Object Lock configuration cannot be enabled on existing buckets"),
+        "unexpected error: {error:?}"
+    );
+
+    Ok(())
+}
@@ -0,0 +1,236 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
+use crate::common::{admin_request, init_logging, local_http_client};
+use aws_sdk_s3::operation::RequestId;
+use aws_sdk_s3::primitives::ByteStream;
+use bytes::Bytes;
+use http::Method;
+use http_body_util::{BodyExt, Empty};
+use hyper::body::Incoming;
+use hyper::service::service_fn;
+use hyper::{Request, Response};
+use hyper_util::rt::TokioIo;
+use local_ip_address::local_ip;
+use rustfs_madmin::metrics::RealtimeMetrics;
+use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
+use serde_json::Value;
+use std::convert::Infallible;
+use std::time::Duration;
+use tokio::net::TcpListener;
+use tokio::sync::mpsc;
+use tokio::task::JoinHandle;
+use tokio::time::{Instant, timeout};
+
+async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
+    let listener = TcpListener::bind("0.0.0.0:0").await?;
+    let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
+    let (tx, rx) = mpsc::unbounded_channel();
+    let handle = tokio::spawn(async move {
+        loop {
+            let Ok((stream, _)) = listener.accept().await else {
+                return;
+            };
+            let tx = tx.clone();
+            tokio::spawn(async move {
+                let service = service_fn(move |request: Request<Incoming>| {
+                    let tx = tx.clone();
+                    async move {
+                        let method = request.method().clone();
+                        if let Ok(body) = request.into_body().collect().await
+                            && method == Method::POST
+                            && let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
+                        {
+                            if let Some(records) = payload["Records"].as_array() {
+                                for entry in records {
+                                    let _ = tx.send(entry.clone());
+                                }
+                            } else {
+                                let _ = tx.send(payload);
+                            }
+                        }
+                        Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
+                    }
+                });
+                let _ = hyper::server::conn::http1::Builder::new()
+                    .serve_connection(TokioIo::new(stream), service)
+                    .await;
+            });
+        }
+    });
+    Ok((endpoint, rx, handle))
+}
+
+async fn wait_for_audit_entry(
+    rx: &mut mpsc::UnboundedReceiver<Value>,
+    bucket: &str,
+    key: &str,
+    request_id: &str,
+) -> TestResult<Value> {
+    let deadline = Instant::now() + Duration::from_secs(30);
+    let mut seen = Vec::new();
+    loop {
+        let remaining = deadline.saturating_duration_since(Instant::now());
+        if remaining.is_zero() {
+            return Err(format!(
+                "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                seen.len()
+            )
+            .into());
+        }
+        let entry = match timeout(remaining, rx.recv()).await {
+            Ok(Some(entry)) => entry,
+            Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
+            Err(_) => {
+                return Err(format!(
+                    "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
+                    seen.len()
+                )
+                .into());
+            }
+        };
+        if entry["api"]["name"].as_str() == Some("s3:PutObject")
+            && entry["api"]["bucket"].as_str() == Some(bucket)
+            && entry["api"]["object"].as_str() == Some(key)
+            && entry["requestID"].as_str() == Some(request_id)
+        {
+            return Ok(entry);
+        }
+        if seen.len() < 8 {
+            seen.push(format!(
+                "api={:?} bucket={:?} object={:?} requestID={:?}",
+                entry["api"]["name"].as_str(),
+                entry["api"]["bucket"].as_str(),
+                entry["api"]["object"].as_str(),
+                entry["requestID"].as_str()
+            ));
+        }
+    }
+}
+
+#[tokio::test]
+async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
+    init_logging();
+    let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
+    let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
+    let audit_env = [
+        ("RUSTFS_AUDIT_ENABLE", "true"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
+        ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
+        (ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
+    ];
+    let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
+    for node_idx in 0..dist.cluster.nodes.len() {
+        let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
+        tokio::fs::create_dir_all(&queue_dir).await?;
+        dist.cluster
+            .set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
+    }
+    dist.cluster.start().await?;
+    wait_for_ready(&dist.cluster).await?;
+
+    let http = local_http_client();
+    for node in &dist.cluster.nodes {
+        for probe in ["ready", "live"] {
+            let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
+            assert!(
+                response.status().is_success(),
+                "node {} {probe} probe failed: {}",
+                node.address,
+                response.status()
+            );
+        }
+    }
+
+    let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
+    let info: Value = serde_json::from_str(&info_body)?;
+    let servers = info["info"]["servers"]
+        .as_array()
+        .ok_or_else(|| format!("admin info omitted servers: {info}"))?;
+    assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
+
+    let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
+    let storage: Value = serde_json::from_str(&storage_body)?;
+    let disks = storage["info"]["disks"]
+        .as_array()
+        .ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
+    assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
+    assert!(
+        disks.iter().all(|disk| {
+            disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
+                && disk["runtimeState"]
+                    .as_str()
+                    .is_some_and(|state| state.eq_ignore_ascii_case("online"))
+        }),
+        "storageinfo reported a drive that was not healthy and online: {storage}"
+    );
+
+    for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
+        let (status, metrics_body) = admin_request(
+            &node.url,
+            Method::GET,
+            "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
+            None,
+            &dist.cluster.access_key,
+            &dist.cluster.secret_key,
+        )
+        .await?;
+        assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
+        let sample: RealtimeMetrics = serde_json::from_str(
+            metrics_body
+                .lines()
+                .next()
+                .ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
+        )?;
+        assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
+        assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
+        assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
+    }
+
+    let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
+    let targets: Value = serde_json::from_str(&targets_body)?;
+    let configured = targets["audit_endpoints"]
+        .as_array()
+        .ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
+        .iter()
+        .any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
+    assert!(configured, "configured audit webhook was missing: {targets}");
+
+    let bucket = unique_bucket("audit");
+    dist.create_bucket(&bucket).await?;
+    let key = "correlated/audit-object.bin";
+    let put = dist
+        .client(2)?
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"distributed audit payload"))
+        .send()
+        .await?;
+    let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
+    let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
+    assert_eq!(
+        audit["api"]["status_code"].as_i64(),
+        Some(200),
+        "audit entry did not report success: {audit}"
+    );
+    assert!(
+        !audit.to_string().contains(&dist.cluster.secret_key),
+        "audit entry leaked the root secret key"
+    );
+
+    collector.abort();
+    Ok(())
+}
@@ -0,0 +1,191 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
+    set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
+};
+use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::primitives::ByteStream;
+use http::Method;
+use std::time::Duration;
+
+async fn wait_for_replication_status(
+    client: &aws_sdk_s3::Client,
+    bucket: &str,
+    key: &str,
+    expected: &[&str],
+    timeout: Duration,
+) -> TestResult {
+    wait_until(
+        timeout,
+        || async {
+            let head = client.head_object().bucket(bucket).key(key).send().await?;
+            Ok(head
+                .replication_status()
+                .is_some_and(|status| expected.contains(&status.as_str())))
+        },
+        &format!("replication status for {bucket}/{key} in {expected:?}"),
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
+    init_logging();
+    let (source, mut target) = DistCluster::start_replication_pair().await?;
+    let source_bucket = unique_bucket("replsrc");
+    let target_bucket = unique_bucket("repldst");
+    source.create_bucket(&source_bucket).await?;
+    target.create_bucket(&target_bucket).await?;
+
+    let source_client = source.client(0)?;
+    let target_client = target.client(0)?;
+    enable_versioning(&source_client, &source_bucket).await?;
+    enable_versioning(&target_client, &target_bucket).await?;
+
+    let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
+    put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
+
+    let key = "replicated/metadata-and-tags.bin";
+    let body = b"distributed-bucket-replication".to_vec();
+    source_client
+        .put_object()
+        .bucket(&source_bucket)
+        .key(key)
+        .metadata("origin", "four-node-source")
+        .tagging("suite=distributed&shape=metadata")
+        .body(ByteStream::from(body.clone()))
+        .send()
+        .await?;
+    wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
+
+    let peer_read = target.client(3)?;
+    wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
+    let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
+    assert_eq!(
+        replica_head
+            .metadata()
+            .and_then(|metadata| metadata.get("origin"))
+            .map(String::as_str),
+        Some("four-node-source")
+    );
+    assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
+    let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
+    let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
+    assert_eq!(tags.get("suite"), Some(&"distributed"));
+    assert_eq!(tags.get("shape"), Some(&"metadata"));
+
+    target.cluster.stop();
+    let outage_key = "replicated/queued-during-target-outage.bin";
+    let outage_body = b"retry-after-target-restart".to_vec();
+    put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
+    wait_for_replication_status(
+        &source_client,
+        &source_bucket,
+        outage_key,
+        &["PENDING", "FAILED"],
+        Duration::from_secs(30),
+    )
+    .await?;
+
+    target.cluster.start().await?;
+    wait_for_ready(&target.cluster).await?;
+    wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
+    wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
+    let bucket = unique_bucket("quota");
+    dist.create_bucket(&bucket).await?;
+    set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
+
+    let client = dist.client(1)?;
+    retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, body) = super::harness::cluster_admin(
+                &dist.cluster,
+                Method::GET,
+                &format!("/rustfs/admin/v3/quota-stats/{bucket}"),
+                None,
+            )
+            .await?;
+            if !status.is_success() {
+                return Ok(false);
+            }
+            let stats: serde_json::Value =
+                serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
+            let usage = stats
+                .get("current_usage")
+                .and_then(serde_json::Value::as_u64)
+                .ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
+            Ok(usage >= 1024)
+        },
+        "quota stats observe small object",
+    )
+    .await?;
+
+    let oversized_key = "too-big.bin";
+    let error = client
+        .put_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .body(vec![0u8; 16 * 1024].into())
+        .send()
+        .await
+        .expect_err("hard quota must reject the oversized PUT");
+    let service_error = error
+        .as_service_error()
+        .ok_or("quota rejection was not an S3 service error")?;
+    assert_eq!(
+        error.raw_response().map(|response| response.status().as_u16()),
+        Some(400),
+        "quota rejection must be HTTP 400: {error:?}"
+    );
+    assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
+    assert!(
+        service_error
+            .message()
+            .is_some_and(|message| message.starts_with("Bucket quota exceeded")),
+        "PUT must fail specifically at quota admission: {error:?}"
+    );
+
+    let missing = client
+        .head_object()
+        .bucket(&bucket)
+        .key(oversized_key)
+        .send()
+        .await
+        .expect_err("an object rejected by quota must not become visible");
+    assert_eq!(
+        missing.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "quota-rejected object returned an unexpected HEAD result: {missing:?}"
+    );
+
+    let listed = client.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
+        "quota-rejected key leaked into ListObjectsV2"
+    );
+    Ok(())
+}
@@ -0,0 +1,258 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
+use crate::common::{init_logging, local_http_client};
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::presigning::PresigningConfig;
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
+use std::time::Duration;
+
+#[tokio::test]
+async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3basic");
+    dist.create_bucket(&bucket).await?;
+
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    let key = "dir/object.bin";
+    let body = vec![0xA5u8; 256 * 1024];
+    put_object(&writer, &bucket, key, body.clone()).await?;
+
+    let head = reader.head_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(head.content_length(), Some(body.len() as i64));
+    assert_object_bytes(&reader, &bucket, key, &body).await?;
+
+    let ranged = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .range("bytes=0-15")
+        .send()
+        .await?;
+    let ranged_body = ranged.body.collect().await?.into_bytes();
+    assert_eq!(ranged_body.as_ref(), &body[..16]);
+
+    let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
+    let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
+    assert_eq!(keys, vec![key]);
+
+    let copy_key = "dir/object-copy.bin";
+    reader
+        .copy_object()
+        .bucket(&bucket)
+        .key(copy_key)
+        .copy_source(format!("{bucket}/{key}"))
+        .metadata_directive(MetadataDirective::Copy)
+        .send()
+        .await?;
+    assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
+
+    let moved_key = "dir/object-moved.bin";
+    writer
+        .copy_object()
+        .bucket(&bucket)
+        .key(moved_key)
+        .copy_source(format!("{bucket}/{copy_key}"))
+        .send()
+        .await?;
+    writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
+    match writer.head_object().bucket(&bucket).key(copy_key).send().await {
+        Ok(_) => return Err("copied source still present after rename delete".into()),
+        Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
+        Err(error) => return Err(error.into()),
+    }
+    assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
+
+    let presigned = writer
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
+        .await?;
+    let response = local_http_client().get(presigned.uri().to_string()).send().await?;
+    assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
+    let presigned_body = response.bytes().await?;
+    assert_eq!(presigned_body.as_ref(), body.as_slice());
+
+    let empty_key = "empty";
+    put_object(&writer, &bucket, empty_key, Vec::new()).await?;
+    let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
+    assert!(empty.is_empty());
+
+    let deleted = writer
+        .delete_objects()
+        .bucket(&bucket)
+        .delete(
+            Delete::builder()
+                .objects(ObjectIdentifier::builder().key(key).build()?)
+                .objects(ObjectIdentifier::builder().key(moved_key).build()?)
+                .objects(ObjectIdentifier::builder().key(empty_key).build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
+    assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
+
+    let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("s3matrix");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+
+    let special_key = "unicode/测试 space+percent%25.txt";
+    let special_body = b"metadata and tagging survive distributed routing".to_vec();
+    let put = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .metadata("test-meta", "distributed")
+        .tagging("purpose=compatibility&scope=four-by-four")
+        .body(ByteStream::from(special_body.clone()))
+        .send()
+        .await?;
+    let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
+
+    let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
+    assert_eq!(
+        head.metadata()
+            .and_then(|metadata| metadata.get("test-meta"))
+            .map(String::as_str),
+        Some("distributed")
+    );
+    assert_eq!(head.e_tag(), Some(etag.as_str()));
+    let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
+    let actual_tags: std::collections::BTreeMap<_, _> = tags
+        .tag_set()
+        .iter()
+        .map(|tag| (tag.key().to_string(), tag.value().to_string()))
+        .collect();
+    assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
+    assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
+
+    let conditional = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .if_match(&etag)
+        .send()
+        .await?;
+    assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
+    let invalid_range = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(special_key)
+        .range("bytes=999999-1000000")
+        .send()
+        .await
+        .expect_err("an unsatisfiable range must fail");
+    assert_eq!(
+        invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
+        Some("InvalidRange"),
+        "unexpected invalid-range error: {invalid_range:?}"
+    );
+
+    let upload_key = "multipart/aborted.bin";
+    let upload = writer
+        .create_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await?;
+    let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
+    writer
+        .upload_part()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .part_number(1)
+        .body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
+        .send()
+        .await?;
+    let pending = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
+    writer
+        .abort_multipart_upload()
+        .bucket(&bucket)
+        .key(upload_key)
+        .upload_id(upload_id)
+        .send()
+        .await?;
+    let after_abort = reader
+        .list_multipart_uploads()
+        .bucket(&bucket)
+        .prefix("multipart/")
+        .send()
+        .await?;
+    assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
+    let aborted_head = reader
+        .head_object()
+        .bucket(&bucket)
+        .key(upload_key)
+        .send()
+        .await
+        .expect_err("aborted multipart upload must not create an object");
+    assert_eq!(
+        aborted_head.raw_response().map(|response| response.status().as_u16()),
+        Some(404),
+        "aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
+    );
+
+    for index in 0..113 {
+        let key = format!("page/{index:04}.txt");
+        put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
+    }
+    let mut token = None;
+    let mut paged_keys = Vec::new();
+    loop {
+        let page = reader
+            .list_objects_v2()
+            .bucket(&bucket)
+            .prefix("page/")
+            .max_keys(37)
+            .set_continuation_token(token.take())
+            .send()
+            .await?;
+        paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
+        if page.is_truncated() != Some(true) {
+            break;
+        }
+        token = Some(
+            page.next_continuation_token()
+                .ok_or("truncated ListObjectsV2 page omitted next continuation token")?
+                .to_string(),
+        );
+    }
+    assert_eq!(paged_keys.len(), 113);
+    let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
+    assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
+    Ok(())
+}
@@ -0,0 +1,94 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
+    decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
+    retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
+    wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
+};
+use crate::common::init_logging;
+use std::time::Duration;
+
+#[tokio::test]
+async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
+    let bucket = unique_bucket("s3move");
+    dist.create_bucket(&bucket).await?;
+    let client = dist.client(0)?;
+    let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
+    dist.expand_to_four_pools().await?;
+
+    start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
+    wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
+    let live = dist.client(2)?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-decommission.bin",
+        b"written-while-decommissioning",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let listed = live.list_objects_v2().bucket(&bucket).send().await?;
+    assert!(
+        listed
+            .contents()
+            .iter()
+            .any(|object| object.key() == Some("during-decommission.bin")),
+        "list during decommission missed the newly written key"
+    );
+    let status = decommission_status_json(&dist.cluster).await?;
+    if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
+        return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
+    }
+
+    wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
+    assert_inventory(&live, &bucket, &inventory).await?;
+
+    let rebalance_id = start_rebalance(&dist.cluster).await?;
+    wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
+    retrying_put(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing".to_vec(),
+        Duration::from_secs(30),
+    )
+    .await?;
+    retrying_get_equals(
+        &live,
+        &bucket,
+        "during-rebalance.bin",
+        b"written-while-rebalancing",
+        Duration::from_secs(30),
+    )
+    .await?;
+    let status = rebalance_status_json(&dist.cluster).await?;
+    if !rebalance_running_with_progress(&status, &rebalance_id)? {
+        return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
+    }
+    wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
+    assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
+    Ok(())
+}
@@ -0,0 +1,128 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{
+    DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
+    wait_until,
+};
+use crate::common::{init_logging, signed_request};
+use http::{Method, StatusCode};
+use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
+use std::time::Duration;
+
+async fn site_replication_add(
+    cluster: &crate::common::RustFSTestClusterEnvironment,
+    sites: &[PeerSite],
+) -> TestResult<ReplicateAddStatus> {
+    let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
+    let response = signed_request(
+        Method::PUT,
+        &url,
+        &cluster.access_key,
+        &cluster.secret_key,
+        Some(serde_json::to_vec(sites)?),
+        Some("application/json"),
+    )
+    .await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("site replication add failed: {status} {body}").into());
+    }
+    Ok(serde_json::from_slice(&response.bytes().await?)?)
+}
+
+async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
+    let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let info = site_replication_info(cluster).await?;
+            Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
+        },
+        "site replication enabled with two synchronized sites",
+    )
+    .await
+}
+
+#[tokio::test]
+async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
+    init_logging();
+    let (site_a, site_b) = DistCluster::start_replication_pair().await?;
+    let bucket = unique_bucket("siterepl");
+    site_a.create_bucket(&bucket).await?;
+    site_b.create_bucket(&bucket).await?;
+
+    let client_a = site_a.client(0)?;
+    let client_b = site_b.client(0)?;
+    enable_versioning(&client_a, &bucket).await?;
+    enable_versioning(&client_b, &bucket).await?;
+
+    let sites = vec![
+        PeerSite {
+            name: "site-a".to_string(),
+            endpoint: site_a.cluster.nodes[0].url.clone(),
+            access_key: site_a.cluster.access_key.clone(),
+            secret_key: site_a.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+        PeerSite {
+            name: "site-b".to_string(),
+            endpoint: site_b.cluster.nodes[0].url.clone(),
+            access_key: site_b.cluster.access_key.clone(),
+            secret_key: site_b.cluster.secret_key.clone(),
+            ..Default::default()
+        },
+    ];
+    let add_status = site_replication_add(&site_a.cluster, &sites).await?;
+    assert!(
+        add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
+        "site replication add reported failure: {add_status:?}"
+    );
+    wait_for_site_replication_enabled(&site_a.cluster).await?;
+    wait_for_site_replication_enabled(&site_b.cluster).await?;
+
+    let info_a = site_replication_info(&site_a.cluster).await?;
+    let remote = info_a
+        .sites
+        .iter()
+        .find(|site| site.name == "site-b")
+        .ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
+    assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
+    let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
+    assert!(
+        deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
+        "site peers must have two distinct non-empty deployment IDs: {info_a:?}"
+    );
+    assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
+    assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
+
+    let key = "site-object.bin";
+    let body = b"four-node-site-replication".to_vec();
+    put_object(&client_a, &bucket, key, body.clone()).await?;
+    wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
+
+    let peer_b = site_b.client(3)?;
+    wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
+
+    let reverse_key = "reverse/site-object.bin";
+    let reverse_body = b"site-b-to-site-a".to_vec();
+    put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
+    wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
+    Ok(())
+}
@@ -0,0 +1,345 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! 4-node upgrade coverage for historical objects and IAM AK/SK.
+//!
+//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
+//! mixed-version listing). This module pins the distributed contract the
+//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
+//! written on the previous release still read back, and IAM user credentials
+//! created before the upgrade still authenticate.
+//!
+//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
+//! release. The `e2e-distributed` workflow downloads that binary; a local run
+//! without it fails closed rather than skipping.
+
+use super::harness::{
+    DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
+    unique_bucket, wait_until,
+};
+use crate::common::{
+    AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
+};
+use aws_sdk_s3::Client;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+use uuid::Uuid;
+
+const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const IAM_SECRET: &str = "UpgradeTestSecretKey1";
+const WRONG_SECRET: &str = "WrongSecretKey000000";
+const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct UpgradeSeed {
+    history_bucket: String,
+    history_key: &'static str,
+    history_body: Vec<u8>,
+    versioned_bucket: String,
+    versioned_key: &'static str,
+    version1: String,
+    version1_body: Vec<u8>,
+    version2: String,
+    version2_body: Vec<u8>,
+    iam_bucket: String,
+    iam_key: &'static str,
+    iam_body: Vec<u8>,
+    iam_user: String,
+    iam_secret: &'static str,
+}
+
+fn source_binary() -> TestResult<PathBuf> {
+    let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
+        format!(
+            "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
+        )
+    })?;
+    if !path.is_file() {
+        return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
+    }
+    Ok(path)
+}
+
+fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
+    let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
+        return Ok(());
+    };
+    std::fs::create_dir_all(&log_dir)?;
+    for node_idx in 0..cluster.cluster.nodes.len() {
+        let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
+        cluster
+            .cluster
+            .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
+    }
+    Ok(())
+}
+
+fn iam_rw_policy(bucket: &str) -> String {
+    serde_json::json!({
+        "Version": "2012-10-17",
+        "Statement": [{
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+                format!("arn:aws:s3:::{bucket}"),
+                format!("arn:aws:s3:::{bucket}/*")
+            ]
+        }]
+    })
+    .to_string()
+}
+
+async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
+    let url = &dist.cluster.nodes[0].url;
+    let access = &dist.cluster.access_key;
+    let admin_secret = &dist.cluster.secret_key;
+    admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
+    admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
+    admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
+    Ok(())
+}
+
+async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let body = body.clone();
+            async move {
+                put_object(&client, &bucket, &key, body).await?;
+                Ok(true)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
+    wait_until(
+        CREDENTIAL_TIMEOUT,
+        || {
+            let client = client.clone();
+            let bucket = bucket.to_string();
+            let key = key.to_string();
+            let expected = expected.to_vec();
+            async move {
+                let got = get_object_bytes(&client, &bucket, &key).await?;
+                Ok(got == expected)
+            }
+        },
+        label,
+    )
+    .await
+}
+
+async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
+    let history_bucket = unique_bucket("upg-hist");
+    let versioned_bucket = unique_bucket("upg-ver");
+    let iam_bucket = unique_bucket("upg-iam");
+    dist.create_bucket(&history_bucket).await?;
+    dist.create_bucket(&versioned_bucket).await?;
+    dist.create_bucket(&iam_bucket).await?;
+
+    let root = dist.client(0)?;
+    enable_versioning(&root, &versioned_bucket).await?;
+
+    let history_key = "plain-history.bin";
+    let history_body = b"written by the previous 4-node release".to_vec();
+    put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
+
+    let versioned_key = "versioned-history.txt";
+    let version1_body = b"version-one-before-upgrade".to_vec();
+    let version1 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("first versioned PUT omitted version ID")?
+        .to_string();
+    let version2_body = b"version-two-before-upgrade".to_vec();
+    let version2 = root
+        .put_object()
+        .bucket(&versioned_bucket)
+        .key(versioned_key)
+        .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
+        .send()
+        .await?
+        .version_id()
+        .ok_or("second versioned PUT omitted version ID")?
+        .to_string();
+
+    let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
+    create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
+
+    let iam_key = "iam-history.bin";
+    let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
+    let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
+    wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
+
+    Ok(UpgradeSeed {
+        history_bucket,
+        history_key,
+        history_body,
+        versioned_bucket,
+        versioned_key,
+        version1,
+        version1_body,
+        version2,
+        version2_body,
+        iam_bucket,
+        iam_key,
+        iam_body,
+        iam_user,
+        iam_secret: IAM_SECRET,
+    })
+}
+
+async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
+    let root_a = dist.client(0)?;
+    let root_b = dist.client(3)?;
+    wait_for_bytes(
+        &root_b,
+        &seed.history_bucket,
+        seed.history_key,
+        &seed.history_body,
+        &format!("{context}: root GET historical object"),
+    )
+    .await?;
+    assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
+
+    let v1 = root_b
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version1)
+        .send()
+        .await?;
+    let v1_body = v1.body.collect().await?.into_bytes();
+    if v1_body.as_ref() != seed.version1_body.as_slice() {
+        return Err(format!("{context}: version 1 bytes changed after upgrade").into());
+    }
+    let v2 = root_a
+        .get_object()
+        .bucket(&seed.versioned_bucket)
+        .key(seed.versioned_key)
+        .version_id(&seed.version2)
+        .send()
+        .await?;
+    let v2_body = v2.body.collect().await?.into_bytes();
+    if v2_body.as_ref() != seed.version2_body.as_slice() {
+        return Err(format!("{context}: version 2 bytes changed after upgrade").into());
+    }
+
+    let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
+    if !users.contains(&seed.iam_user) {
+        return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
+    }
+
+    let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
+    let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
+    wait_for_bytes(
+        &iam_on_upgraded,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 0"),
+    )
+    .await?;
+    wait_for_bytes(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        seed.iam_key,
+        &seed.iam_body,
+        &format!("{context}: IAM GET historical object on node 3"),
+    )
+    .await?;
+
+    let post_key = format!("after-upgrade-{context}.txt");
+    let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
+    wait_for_put(
+        &iam_on_peer,
+        &seed.iam_bucket,
+        &post_key,
+        post_body.clone(),
+        &format!("{context}: IAM PUT after upgrade"),
+    )
+    .await?;
+    assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
+
+    let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
+    match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
+        Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
+        Err(error) => {
+            let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+            let rejected = code == Some("SignatureDoesNotMatch")
+                || code == Some("InvalidAccessKeyId")
+                || code == Some("AccessDenied")
+                || code == Some("InvalidArgument")
+                || error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
+            if !rejected {
+                return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
+            }
+        }
+    }
+
+    let post_root_key = format!("root-after-{context}.bin");
+    let post_root_body = format!("{context}: root write after upgrade").into_bytes();
+    put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
+    assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "direct-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+    dist.restart_with_current_binary().await?;
+    assert_history_and_iam(&dist, &seed, "direct").await?;
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
+    init_logging();
+    let previous = source_binary()?;
+    let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
+    capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
+    dist.start_from_binary(&previous).await?;
+
+    let seed = seed_history_and_iam(&dist).await?;
+
+    dist.replace_node_with_current_binary(0).await?;
+    assert_history_and_iam(&dist, &seed, "one-current-node").await?;
+
+    for node_idx in [1, 2] {
+        dist.replace_node_with_current_binary(node_idx).await?;
+    }
+    assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
+
+    dist.replace_node_with_current_binary(3).await?;
+    assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
+    Ok(())
+}
@@ -0,0 +1,188 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
+use crate::common::init_logging;
+use aws_sdk_s3::error::ProvideErrorMetadata;
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+
+#[tokio::test]
+async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("version");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "versioned.txt";
+    let v1_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v1".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v1 PUT omitted version ID")?
+        .to_string();
+    let v2_id = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"v2".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("v2 PUT omitted version ID")?
+        .to_string();
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_versions: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
+    assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
+        "v1 was not the historical version: {versions:?}"
+    );
+    assert!(
+        matching_versions
+            .iter()
+            .any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
+        "v2 was not the latest version: {versions:?}"
+    );
+
+    let latest = get_object_bytes(&reader, &bucket, key).await?;
+    assert_eq!(latest, b"v2");
+
+    let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let older_body = older.body.collect().await?.into_bytes();
+    assert_eq!(older_body.as_ref(), b"v1");
+
+    let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
+    assert_eq!(deleted.delete_marker(), Some(true));
+    let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
+    let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching_markers: Vec<_> = after_delete
+        .delete_markers()
+        .iter()
+        .filter(|marker| marker.key() == Some(key))
+        .collect();
+    assert_eq!(
+        matching_markers.len(),
+        1,
+        "delete marker missing or duplicated after current-version delete: {after_delete:?}"
+    );
+    assert!(
+        matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
+        "DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
+    );
+
+    let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
+    match latest_after_delete {
+        Ok(_) => return Err("current version should be a delete marker".into()),
+        Err(error)
+            if error
+                .as_service_error()
+                .and_then(ProvideErrorMetadata::code)
+                .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
+        Err(error) => return Err(error.into()),
+    }
+
+    let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
+    let restored_body = restored.body.collect().await?.into_bytes();
+    assert_eq!(restored_body.as_ref(), b"v1");
+
+    writer
+        .delete_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(marker_id)
+        .send()
+        .await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
+    Ok(())
+}
+
+#[tokio::test]
+async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
+    init_logging();
+    let dist = DistCluster::start(DistLayout::FourByFour).await?;
+    let bucket = unique_bucket("suspend");
+    dist.create_bucket(&bucket).await?;
+    let writer = dist.client(0)?;
+    let reader = dist.client(3)?;
+    enable_versioning(&writer, &bucket).await?;
+
+    let key = "suspended.txt";
+    let original = writer
+        .put_object()
+        .bucket(&bucket)
+        .key(key)
+        .body(b"enabled-history".to_vec().into())
+        .send()
+        .await?
+        .version_id()
+        .ok_or("enabled PUT omitted version ID")?
+        .to_string();
+    writer
+        .put_bucket_versioning()
+        .bucket(&bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Suspended)
+                .build(),
+        )
+        .send()
+        .await?;
+
+    put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
+    put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
+    assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
+
+    let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
+    let matching: Vec<_> = versions
+        .versions()
+        .iter()
+        .filter(|version| version.key() == Some(key))
+        .collect();
+    assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
+    let null_version_count = matching
+        .iter()
+        .filter(|version| {
+            matches!(
+                version.version_id(),
+                None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
+            )
+        })
+        .count();
+    assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
+
+    let historical = reader
+        .get_object()
+        .bucket(&bucket)
+        .key(key)
+        .version_id(&original)
+        .send()
+        .await?;
+    assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
+    Ok(())
+}
@@ -378,6 +378,11 @@ mod bucket_stats_regression_test;
 #[cfg(test)]
 mod distributed_startup_regression_test;

+// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
+// quota, observability, expand/decommission/rebalance, site replication, chaos).
+#[cfg(test)]
+mod distributed;
+
 // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
 #[cfg(test)]
 mod tier_transition_regression_test;
@@ -20,9 +20,10 @@
 //! journal (`count_requests`) carries the assertion in every one of them.

 use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
-use crate::fake_s3_target::Operation;
+use crate::fake_s3_target::{FaultAction, Operation};
 use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
 use bytes::Bytes;
+use futures::{StreamExt, TryStreamExt};
 use std::time::Duration;

 type TestResult = Result<(), BoxError>;
@@ -145,14 +146,38 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
    .await?;

    let body = payload(128 * 1024);
+    let blocker = "queue/blocker.bin";
+    env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
+    // The one-chunk range completes immediately; its full background pull
+    // occupies the only slot while the remaining requests fill the queue.
+    env.source.inject_for_key(
+        Operation::GetObject,
+        blocker,
+        FaultAction::SlowSendBody {
+            chunk_bytes: 1024,
+            delay: Duration::from_millis(100),
+        },
+        2,
+    );
+    let response = env
+        .raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
+        .await?;
+    assert_eq!(response.status, 206);
+    assert_eq!(response.body, body.slice(0..1024));
+    env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
+
    let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
    let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
    env.seed_source(SOURCE_BUCKET, &seeds);

-    let responses: Vec<RawResponse> = futures::future::try_join_all(
+    // Bound source connections below the fixture's limit while still
+    // submitting all 100 requests to the eight-slot background queue.
+    let responses: Vec<RawResponse> = futures::stream::iter(
        keys.iter()
            .map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
    )
+    .buffered(16)
+    .try_collect()
    .await?;
    for (key, response) in keys.iter().zip(&responses) {
        assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
@@ -168,6 +193,15 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
        .wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
        .await?;
    assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
+    let queue_full = usize::try_from(queue_full)?;
+    assert!(queue_full <= REQUESTS);
+    env.wait_for_status_counter(
+        bucket,
+        "/counters/pulled_objects_total/background",
+        u64::try_from(REQUESTS + 1 - queue_full)?,
+        SETTLE,
+    )
+    .await?;

    let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
    assert!(
@@ -175,9 +209,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
        "every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
    );
    let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
-    assert!(
-        dropped > 0,
-        "the overflowed keys are the ones with no backfill GET, but every key got one"
-    );
+    assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
    Ok(())
 }
@@ -265,16 +265,13 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
    let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
    assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");

-    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
-    let rejected = env
-        .raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
-        .await?;
-    assert_eq!(
-        rejected.status,
-        400,
-        "a bumped token version is a client error: {}",
-        String::from_utf8_lossy(&rejected.body)
-    );
+    let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
+    assert_ne!(tampered, token, "the test must change the token version");
+    let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
+    let rejected = env.raw_list_objects_v2(bucket, &query).await?;
+    let error_body = String::from_utf8_lossy(&rejected.body);
+    assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
+    assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
    Ok(())
 }

@@ -32,7 +32,7 @@ pub mod bucket {
    pub mod bucket_target_sys {
        pub use crate::bucket::bucket_target_sys::{
            AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
-            SsecPassthroughCapability, TargetClient, UnreadableTargetsPolicy, append_version_id_query,
+            SsecPassthroughCapability, TargetClient, append_version_id_query,
        };
    }

@@ -369,26 +369,6 @@ struct SsecPassthroughRecord {
    recorded_at: Instant,
 }

-/// What a target write does when the bucket's persisted target set exists but
-/// cannot be decoded.
-///
-/// `docs/architecture/remote-credential-sealing-adr.md` forbids rewriting a
-/// configuration that could not be fully read, because re-serializing a
-/// partial in-memory view is the one mechanism by which a configured target
-/// really disappears. That rule guards against an *unintentional* overwrite,
-/// so an operator who names the hazard keeps a repair path
-/// (rustfs/backlog#2309); everything that does not name it stays refused.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
-pub enum UnreadableTargetsPolicy {
-    /// Refuse the write with [`BucketTargetError::BucketRemoteTargetsUnreadable`].
-    #[default]
-    FailClosed,
-    /// Discard the unreadable set; the target being written becomes the whole
-    /// configuration. Reachable only from an admin request that asked for it
-    /// explicitly, and audited by the caller.
-    Replace,
-}
-
 #[derive(Debug, Default)]
 pub struct BucketTargetSys {
    pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
@@ -811,45 +791,20 @@ impl BucketTargetSys {
        bucket: &str,
        target: &BucketTarget,
        update: bool,
-        unreadable_policy: UnreadableTargetsPolicy,
    ) -> Result<BucketTargets, BucketTargetError> {
        self.validate_target(bucket, target).await?;

-        let mut bucket_targets = self.targets_base_for_write(bucket, unreadable_policy).await?;
+        let mut bucket_targets = match self.list_bucket_targets(bucket).await {
+            Ok(targets) => targets,
+            Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
+            Err(err) => return Err(err),
+        };

        Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;

        Ok(bucket_targets)
    }

-    /// The persisted target set a write merges into.
-    ///
-    /// An absent configuration starts from the empty set. An unreadable one is
-    /// refused, because re-serializing a partial view of a set this node could
-    /// not decode is how a configured target disappears for good — unless the
-    /// caller carries the operator's explicit
-    /// [`UnreadableTargetsPolicy::Replace`] opt-in, which discards it
-    /// deliberately (rustfs/backlog#2309).
-    async fn targets_base_for_write(
-        &self,
-        bucket: &str,
-        unreadable_policy: UnreadableTargetsPolicy,
-    ) -> Result<BucketTargets, BucketTargetError> {
-        match self.list_bucket_targets(bucket).await {
-            Ok(targets) => Ok(targets),
-            Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
-            // The opt-in discards only a set this node genuinely cannot read.
-            // A readable set still merges through the arm above, so the policy
-            // can never drop a target that was visible here.
-            Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
-                if unreadable_policy == UnreadableTargetsPolicy::Replace =>
-            {
-                Ok(BucketTargets::default())
-            }
-            Err(err) => Err(err),
-        }
-    }
-
    pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
        if !target.target_type.is_valid() {
            return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
@@ -4358,75 +4313,4 @@ mod tests {
        let window = LastMinuteLatency::new();
        assert_eq!(window.get_total().avg, Duration::from_secs(0));
    }
-
-    fn repair_target(bucket: &str, id: &str) -> BucketTarget {
-        BucketTarget {
-            source_bucket: bucket.to_string(),
-            endpoint: "remote.example.com".to_string(),
-            target_bucket: "remote".to_string(),
-            arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"),
-            target_type: BucketTargetType::ReplicationService,
-            region: "us-east-1".to_string(),
-            ..Default::default()
-        }
-    }
-
-    /// rustfs/backlog#2309: after rustfs/rustfs#7172 an undecodable
-    /// `bucket-targets.json` left the bucket with no API repair path at all.
-    /// The refusal is the default and stays the default; the operator's
-    /// explicit opt-in is the only thing that discards the set, and it starts
-    /// the replacement from empty rather than from a partial view of bytes
-    /// this node never decoded.
-    #[tokio::test]
-    async fn an_unreadable_target_set_is_replaced_only_with_the_explicit_opt_in() {
-        let sys = BucketTargetSys::default();
-        let bucket = "targets-repair-opt-in";
-        sys.mark_targets_unreadable(bucket).await;
-
-        assert!(
-            matches!(
-                sys.targets_base_for_write(bucket, UnreadableTargetsPolicy::FailClosed).await,
-                Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
-            ),
-            "without the opt-in an unreadable target set must still refuse the write"
-        );
-        assert_eq!(
-            UnreadableTargetsPolicy::default(),
-            UnreadableTargetsPolicy::FailClosed,
-            "a caller that says nothing must get the refusal"
-        );
-
-        let base = sys
-            .targets_base_for_write(bucket, UnreadableTargetsPolicy::Replace)
-            .await
-            .expect("the explicit opt-in must let an operator replace an unreadable set");
-        assert!(
-            base.is_empty(),
-            "the replacement must start from an empty set, never from a partial decode"
-        );
-    }
-
-    /// The opt-in is not a wipe switch. On a set this node can read, both
-    /// policies take the same merge path, so a stray `replace-unreadable=true`
-    /// cannot drop a visible target — which is what makes the flag safe to
-    /// repeat in an operator's repair script.
-    #[tokio::test]
-    async fn the_opt_in_never_discards_a_readable_target_set() {
-        let sys = BucketTargetSys::default();
-        let bucket = "targets-repair-readable";
-        let existing = repair_target(bucket, "keep");
-        sys.targets_map
-            .write()
-            .await
-            .insert(bucket.to_string(), vec![existing.clone()]);
-
-        for policy in [UnreadableTargetsPolicy::FailClosed, UnreadableTargetsPolicy::Replace] {
-            let base = sys
-                .targets_base_for_write(bucket, policy)
-                .await
-                .expect("a readable target set must be readable under either policy");
-            assert_eq!(base.targets.len(), 1, "{policy:?} must keep the persisted target");
-            assert_eq!(base.targets[0].arn, existing.arn, "{policy:?} must not rewrite the persisted target");
-        }
-    }
 }
@@ -1611,34 +1611,6 @@ mod test {
        assert!(bm.bucket_target_config.is_none());
    }

-    /// rustfs/backlog#2309: the MinIO-origin `.metadata.bin` this repository
-    /// already carries as a compatibility fixture stores
-    /// `BucketTargetsConfigJSON` as a bare JSON array, which `BucketTargets`
-    /// (a `{"targets":[…]}` struct with no array fallback) cannot decode. The
-    /// bytes below are the exact payload the fixture in
-    /// `metadata_test.rs::TEST_BUCKET_METADATA_HEX` decodes to, so if RustFS
-    /// ever grows the array-shaped compatibility parse, this test is where the
-    /// upgrade break is pinned and where the decision has to be recorded.
-    #[test]
-    fn minio_array_shaped_bucket_targets_are_unreadable() {
-        let minio_array = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#.to_vec();
-        let mut bm = BucketMetadata::new("minio-array-targets");
-        bm.bucket_targets_config_json = minio_array.clone();
-
-        bm.parse_all_configs()
-            .expect("a MinIO-shaped targets blob must not fail the whole metadata load");
-
-        assert!(
-            bm.bucket_targets_unreadable(),
-            "an array-shaped MinIO targets blob is unreadable, not an empty target set"
-        );
-        assert!(bm.bucket_target_config.is_none());
-        assert_eq!(
-            bm.bucket_targets_config_json, minio_array,
-            "the raw MinIO bytes must survive so the configuration stays recoverable"
-        );
-    }
-
    /// The invariant every branch of `parse_all_configs` shares: a stored but
    /// undecodable payload keeps its raw bytes and leaves the typed field
    /// `None`, so no branch fabricates a value. What a reader may then do with
@@ -1,12 +1,12 @@
 # Bucket metadata diagnostics and recovery

-`GET /rustfs/admin/v3/export-bucket-metadata` exports every configuration it can read. A configuration that is stored but unreadable is never exported and never replaced by a fabricated default; instead the bucket gains an entry `<bucket>/rustfs-unreadable-configs.json` of the shape `{"bucket": …, "unreadable": [{"config": …, "error": …}]}` naming each configuration that could not be read and why, and the export continues, so one bucket's undecodable payload cannot cost an operator the whole-cluster backup (rustfs/backlog#2309). The marker name is outside the configuration namespace importers dispatch on, so importing the archive back leaves the affected bucket's stored bytes untouched. A failure of the server's own serialization or archive writing still fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets.
+`GET /rustfs/admin/v3/export-bucket-metadata` keeps its strict behavior: an unreadable configuration fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets.

-To collect a shareable support artifact that identifies the failures without carrying parser detail, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has:
+To inspect readable configurations while identifying failures, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has:

 - Filename `bucket-meta-diagnostic.zip` and header `x-rustfs-bucket-metadata-export: diagnostic`.
 - Readable entries under `_diagnostic/<bucket>/<config>`; target credentials remain redacted.
- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details, and therefore carries no `rustfs-unreadable-configs.json` marker: the manifest reports the same failures with less detail, which is what makes a diagnostic archive safe to hand out.
+- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details.

 `complete` reports whether all supported configuration reads succeeded. A diagnostic archive is never a restorable backup, including when `complete` is true. Import rejects the manifest or reserved directory before any bucket creation or configuration write. The reserved directory is not a valid bucket name, so older importers cannot restore diagnostic entries as ordinary bucket configurations.

@@ -14,11 +14,9 @@ To collect a shareable support artifact that identifies the failures without car

 RustFS currently accepts the documented `{"targets": [...]}` object format. It cannot decrypt MinIO KMS-encrypted target metadata. Unreadable target payloads remain failures instead of being interpreted as an empty target set; diagnostic export and replacement import do not add MinIO KMS decryption support.

-1. Inspect the diagnostic manifest, or the `rustfs-unreadable-configs.json` marker in an ordinary export, to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery.
+1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery.
 2. Prepare a ZIP containing `<bucket>/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets.
 3. Submit the ZIP to the existing authenticated `PUT /rustfs/admin/v3/import-bucket-metadata` endpoint with `ImportBucketMetadataAction` permission. Import validates the replacement and persists it against the bucket incarnation; it does not need to parse the old target payload successfully.
-4. Verify target listing and the intended replication configuration. Retry the ordinary metadata export to confirm the bucket no longer carries an unreadable marker.
-
-Alternatively, `PUT /rustfs/admin/v3/set-remote-target?replace-unreadable=true` discards an undecodable target set as part of setting a replacement target. The flag is the operator's explicit acknowledgement that the stored set is being thrown away; without it the request is refused rather than rewriting an unreadable set from a partial view.
+4. Verify target listing and the intended replication configuration. Retry the ordinary strict metadata export to confirm the unreadable configuration no longer blocks it.

 Do not submit the diagnostic archive itself to the import endpoint. Copy only reviewed replacement entries into an ordinary import archive.
@@ -12,10 +12,10 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
 |---|---|---|---|
 | Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p <crate>`); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) |
 | ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) |
-| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) |
+| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`, `e2e-distributed`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); 4-node 4-disk map: [distributed-e2e.md](distributed-e2e.md) |
 | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` | PR, report-only (second half of the `End-to-End Tests` job) |
 | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only |
-| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree |
+| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` |
 | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus |
 | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p <crate>` | On demand; never a gate |

@@ -63,6 +63,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the
 | `e2e-full` | Merge-queue / main-push single-node e2e lane |
 | `e2e-repl-nightly` | Nightly slow / cross-process replication lane |
 | `e2e-nightly` | Nightly serial multi-process cluster fault lane |
+| `e2e-distributed` | Storage-sensitive PR and nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane |
 | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` |

 Membership of each e2e profile is pinned by a digest in `.config/e2e-<profile>-selection.txt` and checked by `scripts/check_test_wiring.py --check-profile <profile>` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only):
@@ -74,6 +74,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
 | `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` |
 | `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
+| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` |
 | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
 | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
 | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow |
@@ -0,0 +1,75 @@
+# Distributed 4-node 4-disk e2e
+
+**Use this when:** adding or diagnosing GitHub Actions coverage for a 4-node cluster, or deciding whether a behaviour belongs in `e2e-distributed` versus the single-node `e2e-full` lane, the nightly cluster-fault lane, or the hardware functional chain.
+**Source of truth:** `crates/e2e_test/src/distributed/`, `[profile.e2e-distributed]` in `.config/nextest.toml`, `.github/workflows/e2e-distributed.yml`.
+
+## Topology
+
+The in-tree harness runs every node on `127.0.0.1` with a distinct port. That matches `RustFSTestClusterEnvironment` in `crates/e2e_test/src/common.rs`:
+
+| Layout | Constructor | Use |
+|---|---|---|
+| 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos |
+| 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release |
+| 1 single-node pool × 4 drives, then `append_single_node_pool` three times | expansion seed | Pool expand, then decommission / rebalance / integrity |
+
+A multi-pool layout in which any pool spans several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped expansion pools remain the hardware functional-chain / backlog #1313 / #1314 lane.
+
+Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran.
+
+The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated ext4 loopback filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata.
+
+The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing.
+
+## What this lane covers
+
+`cargo nextest run --profile e2e-distributed -p e2e_test` selects `distributed::*`:
+
+- S3 put / get / head / list / copy / rename / delete / presign, range and conditional reads, special keys, metadata, tags, pagination, empty objects, multipart complete and abort
+- Object Lock COMPLIANCE, GOVERNANCE and bypass, legal hold, bucket default retention, and non-lock bucket rejection
+- Versioning, exact historical reads, delete-marker removal, and suspended null-version overwrite semantics
+- Bucket replication between two 4-node clusters, including metadata/tags and target-outage retry; hard quota admission and absence of rejected keys
+- Ready/live probes on every node, exact 4-server/16-disk inventory, realtime metrics on every node, and correlated audit-webhook delivery
+- Pool expand, decommission, rebalance, checksum integrity, versioned and multipart data, and S3 during active movement
+- Bidirectional site-replication convergence plus enabled/synchronized peer state on both sites
+- A 24-worker mixed PUT/HEAD/GET/COPY/DELETE workload; concurrent PUT during active decommission
+- Node kill/restart, full process restart, node-facing TCP blackhole/recovery, in-flight streaming GETs across a peer kill, and fresh-drive replacement verified by physical `xl.meta`/part-shard census
+- Multipart, cross-node listing, list-buckets agreement
+- Direct and rolling upgrade from the pinned previous release: historical objects, versioned history, and IAM user AK/SK still work afterwards
+
+## Existing Actions gaps this lane does not replace
+
+Those suites stay in place; this lane fills the in-tree 4×4 hole they leave.
+
+| Existing lane | Gap |
+|---|---|
+| `rustfs-*-test.yml` functional chain | Clones private `rustfs/auto-testing`, runs on three shared VMs (`vm000`–`vm002`), `continue-on-error: true`, not a merge signal, not 4 nodes. Hardware `rustfs-upgrade-test.yml` stays there |
+| `e2e-upgrade.yml` | Single-node SSE/multipart/delete-marker contracts plus mixed-version listing; does not pin IAM user AK/SK on a 4-node cluster |
+| `e2e-smoke` / `e2e-full` | Most selected cases are single-node; distributed modules are intentionally owned by this serialized lane |
+| `e2e-nightly` | 4-node cluster faults and heal, not S3/lock/versioning/quota/decommission matrix |
+| `e2e-repl-nightly` | Site and bucket replication on 1–3 *single-node* processes |
+| `e2e-s3tests.yml` `multi` | Weekly ceph/s3-tests against Docker 4-node; not lock/WORM, decommission, chaos, or checksum integrity |
+| `crates/e2e_test/src/chaos.rs` | Single-node disk faults only |
+
+Hardware power-loss, physical NIC pull, authenticated inter-node partition, firmware/media errors, and replacement-server provisioning still belong on the hardware validation VMs. This lane provides deterministic process kill, fresh local-volume replacement, and node-facing TCP blackhole analogues; it does not claim physical fault certification.
+
+## Run
+
+```bash
+cargo build -p rustfs --bins
+# Expansion/decommission/rebalance cases require four paths on distinct filesystems.
+export RUSTFS_E2E_POOL_ROOTS=/mnt/rustfs-pool-0:/mnt/rustfs-pool-1:/mnt/rustfs-pool-2:/mnt/rustfs-pool-3
+# Upgrade cases require the pinned previous binary (CI downloads it).
+export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2
+cargo nextest run --profile e2e-distributed -p e2e_test
+```
+
+Without `RUSTFS_UPGRADE_SOURCE_BINARY` the two `distributed::upgrade_test::*` cases fail closed. Without four distinct `RUSTFS_E2E_POOL_ROOTS`, the expansion and data-movement cases fail closed. Filter upgrades out for a local run that is not checking upgrade:
+
+```bash
+cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distributed::upgrade_test::/)'
+```
+
+The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check.
+
+Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linux and Darwin entries with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed <listing.json> <platform>` after adding or renaming a case.
@@ -67,17 +67,6 @@ use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
 const DIAGNOSTIC_EXPORT_PREFIX: &str = "_diagnostic";
 const DIAGNOSTIC_EXPORT_MANIFEST: &str = "_diagnostic-manifest.json";

-/// Archive entry naming the configurations a bucket stores but this build
-/// could not read (rustfs/backlog#2309).
-///
-/// The name deliberately sits outside the configuration-file namespace the
-/// importers switch on — `ImportBucketMetadata` matches known configuration
-/// names and ignores everything else — so no importer can mistake the marker
-/// for a configuration. Ordinary exports carry it; diagnostic exports report
-/// the same failures through [`DIAGNOSTIC_EXPORT_MANIFEST`] instead, which
-/// deliberately withholds the parser detail this marker records.
-const EXPORT_UNREADABLE_MANIFEST: &str = "rustfs-unreadable-configs.json";
-
 const LOG_COMPONENT_ADMIN: &str = "admin";
 const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta";
 const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state";
@@ -87,82 +76,31 @@ fn export_internal_error(message: impl Into<String>) -> s3s::S3Error {
    s3_error!(InternalError, "{message}")
 }

-/// One configuration that is stored for a bucket but could not be exported.
-#[derive(serde::Serialize)]
-struct UnreadableExportEntry {
-    config: &'static str,
-    error: String,
-}
-
-#[derive(serde::Serialize)]
-struct UnreadableExportManifest<'a> {
-    bucket: &'a str,
-    unreadable: &'a [UnreadableExportEntry],
-}
-
-/// Why one of a bucket's configurations could not be exported.
-///
-/// The two variants are what an ordinary export dispatches on: a bucket whose
-/// stored bytes this build cannot turn into a configuration is named and
-/// skipped, while a failure of our own output machinery still fails the whole
-/// export closed.
-#[derive(Debug)]
-enum ExportConfigError {
-    /// The configuration is stored but this build cannot read it: a
-    /// MinIO-origin or otherwise undecodable blob, or a revision that moved
-    /// underneath the export. No retry of ours turns those bytes into a
-    /// configuration, so one such bucket must not abort a whole-cluster export
-    /// (rustfs/backlog#2309).
-    Unreadable(String),
-    /// This build failed to produce its own output for a configuration it had
-    /// already decoded. Nothing about the stored bytes is in doubt, so the
-    /// export fails closed rather than reporting healthy metadata as
-    /// unreadable.
-    Internal(s3s::S3Error),
-}
-
-impl ExportConfigError {
-    fn unreadable(message: impl Into<String>) -> Self {
-        Self::Unreadable(message.into())
-    }
-
-    fn internal(message: impl Into<String>) -> Self {
-        Self::Internal(export_internal_error(message))
-    }
-}
-
-fn checked_raw_xml<T, E, F>(validated: &T, raw: Vec<u8>, parse: F) -> Result<Vec<u8>, ExportConfigError>
+fn checked_raw_xml<T, E, F>(validated: &T, raw: Vec<u8>, parse: F) -> S3Result<Vec<u8>>
 where
    T: PartialEq,
    E: std::fmt::Display,
    F: FnOnce(&[u8]) -> Result<T, E>,
 {
-    let selected = parse(&raw).map_err(|e| {
-        ExportConfigError::unreadable(format!("persisted bucket metadata changed to invalid XML during export: {e}"))
-    })?;
+    let selected = parse(&raw)
+        .map_err(|e| export_internal_error(format!("persisted bucket metadata changed to invalid XML during export: {e}")))?;
    if selected != *validated {
-        return Err(ExportConfigError::unreadable("bucket metadata changed during export"));
+        return Err(export_internal_error("bucket metadata changed during export"));
    }
    Ok(raw)
 }

-fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec<u8>) -> Result<Vec<u8>, ExportConfigError> {
+fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec<u8>) -> S3Result<Vec<u8>> {
    if raw.is_empty() {
        if *validated != VersioningConfiguration::default() {
-            return Err(ExportConfigError::unreadable("bucket metadata changed during export"));
+            return Err(export_internal_error("bucket metadata changed during export"));
        }
-        return serialize(validated).map_err(|e| ExportConfigError::internal(format!("serialize config failed: {e}")));
+        return serialize(validated).map_err(|e| export_internal_error(format!("serialize config failed: {e}")));
    }
    checked_raw_xml(validated, raw, deserialize::<VersioningConfiguration>)
 }

-/// Bytes to export for one of a bucket's configurations.
-///
-/// `Ok(None)` means the bucket has not configured it. An `Err` never becomes an
-/// exported configuration — a fabricated default here would reach an importer
-/// as a real one — and its variant tells the caller whether the failure belongs
-/// to the stored bytes or to this build's own output; see [`ExportConfigError`].
-async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u8>>, ExportConfigError> {
+async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result<Option<Vec<u8>>> {
    match conf {
        BUCKET_POLICY_CONFIG => {
            let config: BucketPolicy = match metadata_sys::get_bucket_policy(bucket).await {
@@ -171,11 +109,11 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}")));
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
                }
            };
-            let config_json = serde_json::to_vec(&config)
-                .map_err(|e| ExportConfigError::internal(format!("failed to serialize config: {e}")))?;
+            let config_json =
+                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?;
            Ok(Some(config_json))
        }
        BUCKET_NOTIFICATION_CONFIG => {
@@ -185,14 +123,14 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")));
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
                }
                Ok(None) => return Ok(None),
            };

            let raw_config = metadata_sys::get(bucket)
                .await
-                .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
                .notification_config_xml
                .clone();
            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<s3s::dto::NotificationConfiguration>)?;
@@ -206,12 +144,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}")));
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
                }
            };
            let raw_config = metadata_sys::get(bucket)
                .await
-                .map_err(|e| ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}")))?
+                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
                .lifecycle_config_xml
                .clone();
            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<BucketLifecycleConfiguration>)?;
@@ -225,12 +163,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}")));
+                    return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}"));
                }
            };
            let raw_config = metadata_sys::get(bucket)
                .await
-                .map_err(|e| ExportConfigError::unreadable(format!("failed to load bucket metadata: {e}")))?
+                .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))?
                .tagging_config_xml
                .clone();
            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<Tagging>)?;
@@ -244,11 +182,11 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")));
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
                }
            };
            let config_json =
-                serde_json::to_vec(&config).map_err(|e| ExportConfigError::internal(format!("serialize config failed: {e}")))?;
+                serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;

            Ok(Some(config_json))
        }
@@ -259,12 +197,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")));
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
                }
            };
            let raw_config = metadata_sys::get(bucket)
                .await
-                .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
                .object_lock_config_xml
                .clone();
            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ObjectLockConfiguration>)?;
@@ -278,12 +216,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")));
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
                }
            };
            let raw_config = metadata_sys::get(bucket)
                .await
-                .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
                .encryption_config_xml
                .clone();
            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ServerSideEncryptionConfiguration>)?;
@@ -297,12 +235,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")));
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
                }
            };
            let raw_config = metadata_sys::get(bucket)
                .await
-                .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
                .versioning_config_xml
                .clone();
            let config_xml = checked_versioning_xml(&config, raw_config)?;
@@ -316,12 +254,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")));
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
                }
            };
            let raw_config = metadata_sys::get(bucket)
                .await
-                .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))?
+                .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))?
                .replication_config_xml
                .clone();
            let config_xml = checked_raw_xml(&config, raw_config, deserialize::<ReplicationConfiguration>)?;
@@ -335,12 +273,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> Result<Option<Vec<u
                    if e == StorageError::ConfigNotFound {
                        return Ok(None);
                    }
-                    return Err(ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")));
+                    return Err(s3_error!(InternalError, "get bucket metadata failed: {e}"));
                }
            };

            let config_json = serde_json::to_vec(&config.redacted_credentials())
-                .map_err(|e| ExportConfigError::internal(format!("serialize config failed: {e}")))?;
+                .map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?;

            Ok(Some(config_json))
        }
@@ -439,52 +377,19 @@ impl Operation for ExportBucketMetadata {
        ];

        for bucket in buckets {
-            let mut unreadable: Vec<UnreadableExportEntry> = Vec::new();
            for &conf in confs.iter() {
                let conf_path = path_join_buf(&[bucket.name.as_str(), conf]);
                let config = match exported_bucket_config(&bucket.name, conf).await {
                    Ok(Some(config)) => config,
                    Ok(None) => continue,
-                    Err(error) => {
-                        if query.diagnostic {
-                            // A diagnostic archive names every configuration it
-                            // could not export under one fixed code, carrying
-                            // neither the payload nor the parser detail, so it
-                            // stays shareable (rustfs/rustfs#7225).
-                            errors.push(serde_json::json!({
-                                "bucket": bucket.name,
-                                "config": conf,
-                                "code": "configuration_unavailable",
-                            }));
-                            continue;
-                        }
-                        match error {
-                            // One bucket's undecodable blob must not abort the
-                            // whole-cluster export: record which configuration
-                            // could not be read and keep going, so an operator
-                            // migrating away still gets every readable
-                            // configuration (rustfs/backlog#2309).
-                            ExportConfigError::Unreadable(error) => {
-                                warn!(
-                                    event = EVENT_ADMIN_BUCKET_META_STATE,
-                                    component = LOG_COMPONENT_ADMIN,
-                                    subsystem = LOG_SUBSYSTEM_BUCKET_META,
-                                    action = "export_bucket_metadata",
-                                    result = "config_unreadable",
-                                    bucket = %bucket.name,
-                                    config_name = %conf,
-                                    error = %error,
-                                    "admin bucket meta state"
-                                );
-                                unreadable.push(UnreadableExportEntry { config: conf, error });
-                                continue;
-                            }
-                            // Our own encoder failed on a configuration this
-                            // build had already decoded. The stored bytes are
-                            // not in question, so fail the export instead of
-                            // reporting readable metadata as unreadable.
-                            ExportConfigError::Internal(error) => return Err(error),
-                        }
+                    Err(error) if !query.diagnostic => return Err(error),
+                    Err(_) => {
+                        errors.push(serde_json::json!({
+                            "bucket": bucket.name,
+                            "config": conf,
+                            "code": "configuration_unavailable",
+                        }));
+                        continue;
                    }
                };
                let conf_path = if query.diagnostic {
@@ -499,23 +404,6 @@ impl Operation for ExportBucketMetadata {
                    .write_all(&config)
                    .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
            }
-
-            // Only reachable outside diagnostic mode, which reports the same
-            // failures through the archive-wide manifest instead.
-            if !unreadable.is_empty() {
-                let manifest = serde_json::to_vec(&UnreadableExportManifest {
-                    bucket: bucket.name.as_str(),
-                    unreadable: &unreadable,
-                })
-                .map_err(|e| export_internal_error(format!("failed to serialize unreadable manifest: {e}")))?;
-                let manifest_path = path_join_buf(&[bucket.name.as_str(), EXPORT_UNREADABLE_MANIFEST]);
-                zip_writer
-                    .start_file(manifest_path, SimpleFileOptions::default())
-                    .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?;
-                zip_writer
-                    .write_all(&manifest)
-                    .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?;
-            }
        }

        if query.diagnostic {
@@ -1476,10 +1364,6 @@ mod backup_zip_compatibility_tests {
    const ROOT_ACCESS_KEY: &str = "BUCKETMETABACKUPROOT";
    const ROOT_SECRET_KEY: &str = "bucketMetaBackupRootSecret123";
    const BUCKET: &str = "backup-compatibility";
-    const UNREADABLE_BUCKET: &str = "minio-origin-targets";
-    /// The exact `BucketTargetsConfigJSON` payload carried by the MinIO
-    /// `.metadata.bin` fixture in `crates/ecstore/src/bucket/metadata_test.rs`.
-    const MINIO_ARRAY_TARGETS: &[u8] = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#;
    const NOTIFICATION_XML: &[u8] = b"<NotificationConfiguration>\n</NotificationConfiguration>";
    const LIFECYCLE_XML: &[u8] = b"<LifecycleConfiguration>\n<Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule>\n</LifecycleConfiguration>";
    const SSE_XML: &[u8] = b"<ServerSideEncryptionConfiguration>\n<Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule>\n</ServerSideEncryptionConfiguration>";
@@ -1611,63 +1495,14 @@ mod backup_zip_compatibility_tests {
                .expect("publish unreadable targets fixture");
            assert!(metadata_sys::get_bucket_targets_config(UNREADABLE).await.is_err());

-            // rustfs/backlog#2309: an ordinary export no longer fails closed on
-            // a configuration that is stored but unreadable. It names that one
-            // configuration in the bucket's own marker entry and keeps every
-            // readable configuration of every bucket, so one MinIO-origin blob
-            // cannot cost an operator the whole-cluster backup.
-            let ordinary = ExportBucketMetadata {}
+            let strict_error = ExportBucketMetadata {}
                .call(
                    admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
                    Params::new(),
                )
                .await
-                .expect("one unreadable configuration must not abort the ordinary export");
-            assert_eq!(ordinary.output.0, StatusCode::OK);
-            assert!(!ordinary.headers.contains_key("x-rustfs-bucket-metadata-export"));
-            let ordinary_bytes = ordinary.output.1.collect().await.expect("read ordinary archive").to_bytes();
-            let mut ordinary_archive = ZipArchive::new(Cursor::new(&ordinary_bytes)).expect("open ordinary archive");
-            assert!(
-                ordinary_archive
-                    .by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}"))
-                    .is_ok(),
-                "a healthy bucket must still export while another bucket is unreadable"
-            );
-            assert!(
-                ordinary_archive
-                    .by_name(&format!("{HEALTHY}/{EXPORT_UNREADABLE_MANIFEST}"))
-                    .is_err(),
-                "a bucket whose configurations all read must carry no unreadable marker"
-            );
-            assert!(
-                ordinary_archive
-                    .by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}"))
-                    .is_err(),
-                "an unreadable targets blob must never be exported as a configuration"
-            );
-            let mut ordinary_marker = Vec::new();
-            ordinary_archive
-                .by_name(&format!("{UNREADABLE}/{EXPORT_UNREADABLE_MANIFEST}"))
-                .expect("the ordinary export must name the configuration it could not read")
-                .read_to_end(&mut ordinary_marker)
-                .expect("read unreadable marker");
-            assert!(
-                !ordinary_marker
-                    .windows(SECRET.len())
-                    .any(|window| window == SECRET.as_bytes())
-            );
-            let ordinary_marker: serde_json::Value = serde_json::from_slice(&ordinary_marker).expect("the marker must be JSON");
-            assert_eq!(ordinary_marker["bucket"], UNREADABLE);
-            assert_eq!(
-                ordinary_marker["unreadable"].as_array().map(Vec::len),
-                Some(1),
-                "only the configuration that could not be read may be marked: {ordinary_marker}"
-            );
-            assert_eq!(ordinary_marker["unreadable"][0]["config"], BUCKET_TARGETS_FILE);
-            assert!(
-                ordinary_marker["unreadable"][0]["error"].is_string(),
-                "the marker must carry the reason an operator needs to repair the bucket"
-            );
+                .expect_err("a complete export must fail closed on unreadable targets");
+            assert_eq!(*strict_error.code(), s3s::S3ErrorCode::InternalError);

            let response = ExportBucketMetadata {}
                .call(
@@ -1841,12 +1676,6 @@ mod backup_zip_compatibility_tests {
        assert!(archive.by_name(DIAGNOSTIC_EXPORT_MANIFEST).is_err());
        assert!(archive.by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")).is_ok());
        assert!(archive.by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")).is_ok());
-        assert!(
-            archive
-                .by_name(&format!("{UNREADABLE}/{EXPORT_UNREADABLE_MANIFEST}"))
-                .is_err(),
-            "the marker must disappear once the configuration reads again"
-        );
    }

    #[tokio::test]
@@ -1989,89 +1818,6 @@ mod backup_zip_compatibility_tests {
        }
        let _: ReplicationConfiguration =
            deserialize(&restored.replication_config_xml).expect("old parser must read the newly exported archive payload");
-
-        // rustfs/backlog#2309: a MinIO-origin `.metadata.bin` stores its
-        // targets as a bare JSON array, which `BucketTargets` cannot decode.
-        // Since rustfs/rustfs#7172 that reads as "stored but unreadable" —
-        // which must mark one bucket's one configuration, not abort the
-        // whole-cluster export an operator needs to migrate away.
-        env.make_bucket(UNREADABLE_BUCKET, false).await;
-        metadata_sys::update(UNREADABLE_BUCKET, BUCKET_TARGETS_FILE, MINIO_ARRAY_TARGETS.to_vec())
-            .await
-            .expect("persist the MinIO-shaped targets blob");
-        metadata_sys::get_bucket_targets_config(UNREADABLE_BUCKET)
-            .await
-            .expect_err("a MinIO array-shaped targets blob must read as unreadable, not as an empty set");
-
-        let cluster_export = ExportBucketMetadata {}
-            .call(
-                admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()),
-                Params::new(),
-            )
-            .await
-            .expect("one bucket's unreadable configuration must not abort the whole-cluster export");
-        assert_eq!(cluster_export.output.0, StatusCode::OK);
-        let cluster_archive = cluster_export
-            .output
-            .1
-            .collect()
-            .await
-            .expect("read cluster archive body")
-            .to_bytes()
-            .to_vec();
-        let mut archive = ZipArchive::new(Cursor::new(&cluster_archive)).expect("open cluster archive");
-
-        // Every readable configuration of every other bucket still exports.
-        for (config_file, payload) in persisted_xml_fixtures() {
-            let mut exported_payload = Vec::new();
-            archive
-                .by_name(&format!("{BUCKET}/{config_file}"))
-                .unwrap_or_else(|_| panic!("cluster export must still contain {config_file}"))
-                .read_to_end(&mut exported_payload)
-                .unwrap_or_else(|_| panic!("read exported {config_file}"));
-            assert_eq!(exported_payload, payload, "one bad bucket must not change another bucket's export");
-        }
-        assert!(
-            archive.by_name(&format!("{BUCKET}/{EXPORT_UNREADABLE_MANIFEST}")).is_err(),
-            "a bucket whose configurations all read must carry no unreadable marker"
-        );
-
-        // The unreadable configuration is named rather than fabricated: no
-        // targets entry is exported for it at all.
-        assert!(
-            archive
-                .by_name(&format!("{UNREADABLE_BUCKET}/{BUCKET_TARGETS_FILE}"))
-                .is_err(),
-            "an unreadable targets blob must never be exported as a configuration"
-        );
-        let mut marker = Vec::new();
-        archive
-            .by_name(&format!("{UNREADABLE_BUCKET}/{EXPORT_UNREADABLE_MANIFEST}"))
-            .expect("the export must name the configuration it could not read")
-            .read_to_end(&mut marker)
-            .expect("read unreadable marker");
-        let marker: serde_json::Value = serde_json::from_slice(&marker).expect("the marker must be JSON");
-        assert_eq!(marker["bucket"], UNREADABLE_BUCKET);
-        assert_eq!(
-            marker["unreadable"].as_array().map(Vec::len),
-            Some(1),
-            "only the configuration that could not be read may be marked: {marker}"
-        );
-        assert_eq!(marker["unreadable"][0]["config"], BUCKET_TARGETS_FILE);
-        drop(archive);
-
-        // The marker cannot be misread as a configuration on the way back in:
-        // the importer switches on configuration names and ignores everything
-        // else, so the bucket's stored bytes come through untouched and the
-        // operator still has to repair them explicitly.
-        import_archive(cluster_archive).await;
-        let after_round_trip = metadata_sys::get_config_from_disk(UNREADABLE_BUCKET)
-            .await
-            .expect("the marked bucket must still load after the round trip");
-        assert_eq!(
-            after_round_trip.bucket_targets_config_json, MINIO_ARRAY_TARGETS,
-            "importing the marker must not overwrite or fabricate the bucket's targets configuration"
-        );
    }
 }

@@ -29,7 +29,7 @@ use crate::admin::storage_api::bucket::replication::{REMOTE_TARGET_READ_ONLY_HIS
 use crate::admin::storage_api::bucket::target::{
    BucketTarget, BucketTargetType, Credentials as TargetCredentials, LatencyStat, duration_from_secs_or_nanos,
 };
-use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys, UnreadableTargetsPolicy};
+use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys};
 use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
 use crate::admin::storage_api::contract::list::ListOperations as _;
 use crate::admin::storage_api::error::StorageError;
@@ -57,20 +57,6 @@ use url::Host;

 const SUPPORTED_REMOTE_TARGET_API: &str = "s3v4";

-const LOG_COMPONENT_ADMIN: &str = "admin";
-const LOG_SUBSYSTEM_REPLICATION: &str = "replication";
-const EVENT_ADMIN_REMOTE_TARGET_STATE: &str = "admin_remote_target_state";
-
-/// `set-remote-target?replace-unreadable=true`: the operator's explicit
-/// acknowledgement that this bucket's persisted target set cannot be decoded
-/// and is to be discarded (rustfs/backlog#2309).
-///
-/// Without it the refusal from rustfs/rustfs#7172 stands, which is what keeps
-/// an unreadable set from being silently rewritten from a partial view. The
-/// flag is deliberately absent from `PutBucketReplication`: targets are
-/// repaired first, then the rule is set.
-const REPLACE_UNREADABLE_TARGETS_PARAM: &str = "replace-unreadable";
-
 /// Field groups a `set-remote-target?update=true` request may modify, mirroring
 /// MinIO's `TargetUpdateType` / `GetTargetUpdateOps` query contract: the update
 /// overlays only the requested groups onto the stored target, so a client can
@@ -555,7 +541,6 @@ impl Operation for SetRemoteTargetHandler {
        };

        let update = queries.get("update").is_some_and(|v| v == "true");
-        let replace_unreadable = queries.get(REPLACE_UNREADABLE_TARGETS_PARAM).is_some_and(|v| v == "true");

        warn!("set remote target, bucket: {}, update: {}", bucket, update);

@@ -723,37 +708,10 @@ impl Operation for SetRemoteTargetHandler {

        let arn = remote_target.arn.clone();

-        let unreadable_policy = if replace_unreadable {
-            UnreadableTargetsPolicy::Replace
-        } else {
-            UnreadableTargetsPolicy::FailClosed
-        };
-        let discarding_unreadable_targets = replace_unreadable
-            && matches!(
-                bucket_target_sys.list_bucket_targets(bucket).await,
-                Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
-            );
-
        let targets = bucket_target_sys
-            .set_target(bucket, &remote_target, update, unreadable_policy)
+            .set_target(bucket, &remote_target, update)
            .await
            .map_err(map_bucket_target_error)?;
-
-        // Audited only where the discard actually happened: the flag alone is
-        // not an event, on a readable set it changes nothing, and a refused
-        // write must not leave a record claiming the set was replaced.
-        if discarding_unreadable_targets {
-            warn!(
-                event = EVENT_ADMIN_REMOTE_TARGET_STATE,
-                component = LOG_COMPONENT_ADMIN,
-                subsystem = LOG_SUBSYSTEM_REPLICATION,
-                action = "set_remote_target",
-                result = "unreadable_targets_replaced",
-                bucket = %bucket,
-                arn = %remote_target.arn,
-                "admin remote target state"
-            );
-        }
        let json_targets = serde_json::to_vec(&targets).map_err(|e| {
            error!("Serialization error: {}", e);
            S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets".to_string())
@@ -212,7 +212,6 @@ pub(crate) mod bucket_target_sys {
    pub(crate) type S3ClientError = super::ecstore_bucket::bucket_target_sys::S3ClientError;
    pub(crate) type SsecPassthroughCapability = super::ecstore_bucket::bucket_target_sys::SsecPassthroughCapability;
    pub(crate) type TargetClient = super::ecstore_bucket::bucket_target_sys::TargetClient;
-    pub(crate) type UnreadableTargetsPolicy = super::ecstore_bucket::bucket_target_sys::UnreadableTargetsPolicy;
 }

 pub(crate) mod lifecycle {
@@ -133,6 +133,7 @@ impl NativeHttp {
        self.send_classified(request, error_code_header, false).await
    }

+    #[cfg(feature = "gcs")]
    pub(super) async fn send_object(
        &self,
        request: reqwest::Request,
@@ -210,6 +211,7 @@ pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) ->
 /// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
 /// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
 /// an MD5.
+#[cfg(any(test, feature = "gcs"))]
 pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
    let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
    (raw.len() == 16).then(|| faster_hex::hex_string(&raw))
@@ -4034,7 +4034,7 @@ mod tests {
            let enabled_before = sys.is_module_enabled();
            sys.set_module_enabled(true);
            let bucket = format!("odm-capture-failure-{}", uuid::Uuid::new_v4());
-            let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
+            let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","region":"us-east-1","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
            config.policy.list_through = true;
            sys.apply_for_incarnation(&bucket, uuid::Uuid::new_v4(), Some(&config)).await;
            let mut get = build_request(
@@ -50,13 +50,8 @@ cd "$(dirname "$0")/.."
 # s3_error! stays flat at 1616.
 # 1616 → 1613 on 2026-09-02: dependency refresh verified the current tree has
 # already shed three s3_error! invocation lines; retighten the line counter.
-# 1613 → 1589 on 2026-09-06: rustfs/backlog#2309 and rustfs/rustfs#7225 both
-# folded the ten per-config arms of ExportBucketMetadata into one helper, which
-# now reports an unreadable configuration as a plain string instead of raising
-# an S3 error per arm (24 invocation lines removed from
-# rustfs/src/admin/handlers/bucket_meta.rs; measured after merging the two).
 S3S_IMPORT_FILES_BASELINE=213
-S3_ERROR_LINES_BASELINE=1589
+S3_ERROR_LINES_BASELINE=1613
 # ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not
 # know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming*
 # client was extracted to crates/s3-client, where s3s usage is legitimate;
Author	SHA1	Message	Date
Zhengchao An	043842d571	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-06 03:32:13 +08:00
Zhengchao An	e1608fbd9c	test(odm): exercise overflow and invalid cursors reliably (#7236 )	2026-09-06 03:09:39 +08:00
Zhengchao An	eb1b17802c	test(odm): provide the source region in access fixture (#7235 )	2026-09-06 02:34:56 +08:00
Zhengchao An	112f70914d	fix(build): scope migration helpers to their features (#7234 ) * test(odm): keep listing header import test scoped * fix(build): gate GCS-only migration HTTP helpers	2026-09-06 02:24:04 +08:00
Zhengchao An	4da48f888e	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 23:56:44 +08:00
overtrue	d8580ec970	test(e2e): prove operations overlap data movement	2026-09-05 22:50:00 +08:00
Zhengchao An	034efa8dac	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 22:21:31 +08:00
Zhengchao An	1277c11f04	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 11:57:33 +08:00
RustFS	bdb069c0fd	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 09:29:28 +08:00
RustFS	b65ab747d3	Merge branch 'main' into test/distributed-e2e-hardening	2026-09-05 09:17:48 +08:00
loverustfs	8c48f184b5	test(e2e): add distributed 4x4 validation	2026-09-05 02:51:03 +08:00