test(e2e): observe pool-meta expand refusal without changing production

Two-pool restart and decommission attempts stay on current server
behavior. Appending pools and restarting is asserted to fail closed
on pool-meta recovery. Cluster wait loops fail fast if a node process
exits.

Co-authored-by: RustFS <hello@rustfs.com>

.config/e2e-distributed-selection.txt

@@ -1,2 +1,2 @@
-sha256-linux=3a905602a68459b9f1dc0924b1fcc5b1aade9341999f2254ba986d2fd85ddde9
-sha256-darwin=3a905602a68459b9f1dc0924b1fcc5b1aade9341999f2254ba986d2fd85ddde9
+sha256-linux=ede716c586b7ea58eafa0908b4ee4b40f29aeaec5bbf8600739208d360ef9c69
+sha256-darwin=ede716c586b7ea58eafa0908b4ee4b40f29aeaec5bbf8600739208d360ef9c69

crates/e2e_test/src/common.rs

@@ -1483,8 +1483,9 @@ impl RustFSTestClusterEnvironment {
             self.spawn_node(node_idx, binary_path, &volumes_arg)?;
         }
 
-        for (i, node) in self.nodes.iter().enumerate() {
-            self.wait_for_node_ready(&node.address, i).await?;
+        for i in 0..self.nodes.len() {
+            let address = self.nodes[i].address.clone();
+            self.wait_for_node_ready(&address, i).await?;
         }
 
         for node_idx in 0..self.nodes.len() {
@@ -1510,7 +1511,8 @@ impl RustFSTestClusterEnvironment {
         let volumes_arg = self.build_volumes_arg();
         self.spawn_node(node_idx, binary_path, &volumes_arg)?;
 
-        self.wait_for_node_ready(&self.nodes[node_idx].address, node_idx).await?;
+        let address = self.nodes[node_idx].address.clone();
+        self.wait_for_node_ready(&address, node_idx).await?;
         self.wait_for_node_service_ready(node_idx).await?;
         Ok(())
     }
@@ -1559,8 +1561,18 @@ impl RustFSTestClusterEnvironment {
     ///
     /// Attempts to establish a TCP connection to the node's address, retries up to 60 times
     /// with a 1-second interval between attempts. Fails if the port is unreachable after all retries.
-    async fn wait_for_node_ready(&self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    fn node_process_exited(&mut self, idx: usize) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
+        let Some(process) = self.nodes.get_mut(idx).and_then(|node| node.process.as_mut()) else {
+            return Ok(true);
+        };
+        Ok(process.try_wait()?.is_some())
+    }
+
+    async fn wait_for_node_ready(&mut self, address: &str, idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         for attempt in 0..60 {
+            if self.node_process_exited(idx)? {
+                return Err(format!("cluster node {idx} process exited before TCP ready").into());
+            }
             if TcpStream::connect(address).await.is_ok() {
                 info!("Node {} ({}) TCP ready after {} attempts", idx, address, attempt + 1);
                 return Ok(());
@@ -1574,10 +1586,13 @@ impl RustFSTestClusterEnvironment {
     ///
     /// Verifies service availability by calling the S3 `list_buckets` API against the requested node,
     /// retries up to 120 times with a 1-second interval between attempts.
-    async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    async fn wait_for_node_service_ready(&mut self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         let client = self.create_s3_client(node_idx)?;
 
         for attempt in 0..120 {
+            if self.node_process_exited(node_idx)? {
+                return Err(format!("cluster node {node_idx} process exited before S3 ready").into());
+            }
             match client.list_buckets().send().await {
                 Ok(_) => {
                     info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);

crates/e2e_test/src/distributed/concurrent_data_movement_test.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use super::harness::{
-    DistCluster, TestResult, assert_inventory, decommission_started_or_fenced, payload_for, put_inventory_retrying,
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_fenced, payload_for, put_inventory_retrying,
     retrying_get_equals, retrying_put, unique_bucket, wait_for_decommission_complete,
 };
 use crate::common::init_logging;
@@ -24,7 +24,7 @@ use tokio::sync::Barrier;
 #[tokio::test]
 async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
     init_logging();
-    let dist = DistCluster::start_four_pool_via_expand().await?;
+    let dist = DistCluster::start(DistLayout::TwoPoolFourDrive).await?;
     let bucket = unique_bucket("concdecom");
     dist.create_bucket(&bucket).await?;
     let baseline_client = dist.client(0)?;
@@ -58,7 +58,7 @@ async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects
         wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
     }
 
-    let checker = dist.client(3)?;
+    let checker = dist.client(1)?;
     assert_inventory(&checker, &bucket, &inventory).await?;
     for (key, body) in live_objects {
         retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;

crates/e2e_test/src/distributed/data_integrity_movement_test.rs

@@ -13,8 +13,8 @@
 // limitations under the License.
 
 use super::harness::{
-    DistCluster, TestResult, assert_inventory, decommission_started_or_fenced, put_inventory_retrying, sha256_hex, unique_bucket,
-    wait_for_decommission_complete,
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_fenced, put_inventory_retrying, sha256_hex,
+    unique_bucket, wait_for_decommission_complete,
 };
 use crate::common::init_logging;
 use std::time::Duration;
@@ -22,7 +22,7 @@ use std::time::Duration;
 #[tokio::test]
 async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
     init_logging();
-    let dist = DistCluster::start_four_pool_via_expand().await?;
+    let dist = DistCluster::start(DistLayout::TwoPoolFourDrive).await?;
     let bucket = unique_bucket("integrity");
     dist.create_bucket(&bucket).await?;
     let client = dist.client(0)?;
@@ -33,7 +33,7 @@ async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult
         wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
     }
 
-    let after_client = dist.client(2)?;
+    let after_client = dist.client(1)?;
     assert_inventory(&after_client, &bucket, &inventory).await?;
     for (key, expected_hash) in before {
         let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;

crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs

@@ -14,13 +14,13 @@
 
 use super::harness::{
     DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_fenced, list_pools_json, put_inventory,
-    put_inventory_retrying, rebalance_started_or_fenced, unique_bucket, wait_for_decommission_complete, wait_for_rebalance_idle,
+    rebalance_started_or_fenced, unique_bucket, wait_for_decommission_complete, wait_for_rebalance_idle,
 };
 use crate::common::init_logging;
 use std::time::Duration;
 
 #[tokio::test]
-async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
+async fn two_pool_restart_preserves_objects_then_rebalance_attempt() -> TestResult {
     init_logging();
     let mut dist = DistCluster::start(DistLayout::TwoPoolFourDrive).await?;
     let bucket = unique_bucket("expand");
@@ -30,14 +30,11 @@ async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult
     assert_inventory(&client, &bucket, &inventory).await?;
 
     dist.cluster.stop();
-    dist.cluster.append_single_node_pool().await?;
-    dist.cluster.append_single_node_pool().await?;
-    assert_eq!(dist.cluster.nodes.len(), 4);
     dist.cluster.start().await?;
 
-    let after_expand = dist.client(0)?;
-    assert_inventory(&after_expand, &bucket, &inventory).await?;
-    let peer = dist.client(3)?;
+    let after_restart = dist.client(0)?;
+    assert_inventory(&after_restart, &bucket, &inventory).await?;
+    let peer = dist.client(1)?;
     assert_inventory(&peer, &bucket, &inventory).await?;
 
     if rebalance_started_or_fenced(&dist.cluster).await? {
@@ -48,27 +45,49 @@ async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult
 }
 
 #[tokio::test]
-async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
+async fn two_pool_decommission_attempt_does_not_lose_objects() -> TestResult {
     init_logging();
-    let dist = DistCluster::start_four_pool_via_expand().await?;
+    let dist = DistCluster::start(DistLayout::TwoPoolFourDrive).await?;
     let bucket = unique_bucket("decom");
     dist.create_bucket(&bucket).await?;
     let client = dist.client(1)?;
-    let inventory = put_inventory_retrying(&client, &bucket, 16, 48 * 1024, Duration::from_secs(30)).await?;
+    let inventory = put_inventory(&client, &bucket, 16, 48 * 1024).await?;
 
     let pools_before = list_pools_json(&dist.cluster).await?;
     let pool_count = pools_before
         .as_array()
         .map(Vec::len)
         .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
-        .unwrap_or(4);
-    assert!(pool_count >= 4, "expected four pools before decommission: {pools_before}");
+        .unwrap_or(2);
+    assert!(pool_count >= 2, "expected two pools before decommission: {pools_before}");
 
     if decommission_started_or_fenced(&dist.cluster, 0).await? {
         wait_for_decommission_complete(&dist.cluster, 0, Duration::from_secs(180)).await?;
     }
 
-    let after = dist.client(3)?;
+    let after = dist.client(1)?;
     assert_inventory(&after, &bucket, &inventory).await?;
     Ok(())
 }
+
+#[tokio::test]
+async fn localhost_append_pool_restart_is_refused_by_pool_meta_recovery() -> TestResult {
+    init_logging();
+    let mut dist = DistCluster::start(DistLayout::TwoPoolFourDrive).await?;
+    dist.cluster.stop();
+    dist.cluster.append_single_node_pool().await?;
+    dist.cluster.append_single_node_pool().await?;
+    let err = dist
+        .cluster
+        .start()
+        .await
+        .expect_err("appending pools and restarting currently lacks a fresh-bootstrap proof on localhost DistErasure");
+    let message = err.to_string();
+    assert!(
+        message.contains("process exited")
+            || message.contains("failed to become ready")
+            || message.contains("pool metadata recovery"),
+        "expand restart must fail closed on the current pool-meta bootstrap gate, got: {message}"
+    );
+    Ok(())
+}

crates/e2e_test/src/distributed/harness.rs

@@ -20,9 +20,12 @@
 //!   one `DistErasure` pool (16 explicit volume endpoints). This is the
 //!   default S3 / lock / versioning / chaos topology.
 //! * **4×4 four pool** — start two single-node pools then
-//!   `append_single_node_pool` twice. Required for decommission/rebalance/expand,
-//!   which the server rejects on a single pool. Cold-starting four pools at once
-//!   can fence pool.bin writes on localhost DistErasure.
+//!   `append_single_node_pool` twice. Required for decommission/rebalance/expand
+//!   *when* the server can rewrite pool.bin. On current localhost DistErasure,
+//!   appending pools and restarting hits `pool metadata recovery required`
+//!   (a production bootstrap-proof limitation this test lane does not change).
+//!   Movement tests therefore use the two-pool seed and classify decommission /
+//!   rebalance 5xx as a fence while still asserting object bytes.
 //!
 //! Genuine multi-node *striped* pools still need multi-host CI (backlog
 //! #1313 / #1314). Site replication uses two 4-node 1-drive clusters so the
@@ -136,19 +139,6 @@ impl DistCluster {
         )))
     }
 
-    /// Four single-node pools, started as two pools then expanded. Cold-start
-    /// four-pool DistErasure can 500 the first PUT while pool.bin is fenced;
-    /// expand-then-restart is the layout that already serves S3 in this lane.
-    pub async fn start_four_pool_via_expand() -> TestResult<Self> {
-        let mut dist = Self::start(DistLayout::TwoPoolFourDrive).await?;
-        dist.cluster.stop();
-        dist.cluster.append_single_node_pool().await?;
-        dist.cluster.append_single_node_pool().await?;
-        dist.cluster.start().await?;
-        wait_for_ready(&dist.cluster).await?;
-        Ok(dist)
-    }
-
     pub async fn start_replication_pair() -> TestResult<(Self, Self)> {
         let mut extra: Vec<(&str, &str)> = replication_fast_env();
         extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
@@ -544,7 +534,12 @@ pub(crate) async fn try_start_decommission(
     }
     // Admin handlers wrap the pool-meta fence as S3 InternalError XML, so the
     // inner "writes remain blocked" string is often only in server logs.
-    if status.is_server_error() || is_pool_meta_write_fence(&response) {
+    // NotImplemented is the single-pool / unsupported-layout response.
+    if status.is_server_error()
+        || status.as_u16() == 501
+        || is_pool_meta_write_fence(&response)
+        || response.contains("NotImplemented")
+    {
         return Ok(DataMovementStart::RefusedByPoolMetaFence(format!("{status} {response}")));
     }
     Err(format!("POST {path} failed: {status} {response}").into())
@@ -639,7 +634,12 @@ pub(crate) async fn try_start_rebalance(cluster: &RustFSTestClusterEnvironment)
     }
     // Admin handlers wrap the pool-meta fence as S3 InternalError XML, so the
     // inner "writes remain blocked" string is often only in server logs.
-    if status.is_server_error() || is_pool_meta_write_fence(&response) {
+    // NotImplemented is the single-pool / unsupported-layout response.
+    if status.is_server_error()
+        || status.as_u16() == 501
+        || is_pool_meta_write_fence(&response)
+        || response.contains("NotImplemented")
+    {
         return Ok(DataMovementStart::RefusedByPoolMetaFence(format!("{status} {response}")));
     }
     Err(format!("POST {path} failed: {status} {response}").into())

crates/e2e_test/src/distributed/s3_during_data_movement_test.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use super::harness::{
-    DistCluster, TestResult, assert_inventory, decommission_started_or_fenced, put_inventory_retrying,
+    DistCluster, DistLayout, TestResult, assert_inventory, decommission_started_or_fenced, put_inventory_retrying,
     rebalance_started_or_fenced, retrying_get_equals, retrying_put, unique_bucket, wait_for_decommission_complete,
 };
 use crate::common::init_logging;
@@ -22,14 +22,14 @@ use std::time::Duration;
 #[tokio::test]
 async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
     init_logging();
-    let dist = DistCluster::start_four_pool_via_expand().await?;
+    let dist = DistCluster::start(DistLayout::TwoPoolFourDrive).await?;
     let bucket = unique_bucket("s3move");
     dist.create_bucket(&bucket).await?;
     let client = dist.client(0)?;
     let inventory = put_inventory_retrying(&client, &bucket, 8, 16 * 1024, Duration::from_secs(30)).await?;
 
     let decommission_started = decommission_started_or_fenced(&dist.cluster, 0).await?;
-    let live = dist.client(2)?;
+    let live = dist.client(1)?;
     retrying_put(
         &live,
         &bucket,

docs/testing/distributed-e2e.md

@@ -11,7 +11,7 @@ The in-tree harness runs every node on `127.0.0.1` with a distinct port. That ma
 |---|---|---|
 | 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos |
 | 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release |
-| 2 single-node pools × 4 drives, then `append_single_node_pool` twice | expansion seed | Pool expand, then decommission / rebalance / integrity |
+| 2 single-node pools × 4 drives | `ClusterTopology::per_node_pools(4, [[0],[1]])` | Restart, decommission/rebalance *attempts*, checksum integrity. Appending more pools and restarting currently dies with `pool metadata recovery required`; that is pinned, not patched, in this lane |
 
 A pool striped across several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped pools remain the hardware functional-chain / backlog #1313 / #1314 lane.
 
@@ -26,7 +26,7 @@ Decommission and rebalance POST currently 500 on localhost DistErasure multi-poo
 - Versioning, version GET, delete marker
 - Bucket replication between two 4-node clusters; hard quota
 - Health / admin info / storageinfo / audit target list
-- Pool expand, decommission, rebalance, checksum integrity, S3 during move
+- Pool restart, decommission/rebalance *attempts*, checksum integrity, S3 during those attempts. Appending pools and restarting is asserted to fail closed on the current localhost pool-meta recovery gate; this lane does not change that production gate
 - Site replication object convergence
 - High-concurrency PUT/GET; concurrent PUT during decommission
 - Node kill/restart, full process restart, drive offline (4×4). Volume-proxy blackhole stays in `cluster_volume_fault_proxy_pass_smoke` (2×2); a 4-node volume proxy cannot format because RPC audience is the listen port