fix(ecstore): keep multi-pool fresh bootstrap proof

Combining a Fresh format-load proof with None from a peer-formatted pool
was collapsing to no authority, so localhost multi-pool clusters never
wrote pool.bin. Treat None as no opinion. Also stabilize the 4-node
quota and volume-proxy e2e cases.

Co-authored-by: RustFS <hello@rustfs.com>

crates/e2e_test/src/distributed/chaos_test.rs

@@ -75,9 +75,11 @@ async fn offline_drive_then_replace_keeps_object_readable() -> TestResult {
 #[tokio::test]
 async fn volume_proxy_blackhole_then_restore_keeps_s3_available() -> TestResult {
     init_logging();
+    // 4 nodes × 1 drive (4-disk DistErasure). Proxying a 16-disk 4×4 set
+    // prevents first-disk format: proxied drives look like missing peers, so
+    // `should_init_erasure_disks` is false and the first disk waits out.
     let mut cluster =
-        crate::common::RustFSTestClusterEnvironment::with_topology(crate::common::ClusterTopology::single_pool_multidrive(4, 4))
+        crate::common::RustFSTestClusterEnvironment::with_topology(crate::common::ClusterTopology::single_pool(4)).await?;
-            .await?;
     let proxy = cluster.start_volume_proxy_for_node(1).await?;
     cluster.start().await?;
     cluster.create_test_bucket("chaos-net").await?;

crates/e2e_test/src/distributed/replication_quota_test.rs

@@ -13,11 +13,12 @@
 // limitations under the License.
 
 use super::harness::{
-    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, set_bucket_quota,
+    DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
-    set_remote_target, unique_bucket, wait_for_replicated_bytes,
+    set_remote_target, unique_bucket, wait_for_replicated_bytes, wait_until,
 };
 use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
 use aws_sdk_s3::error::ProvideErrorMetadata;
+use http::Method;
 use std::time::Duration;
 
 #[tokio::test]
@@ -56,43 +57,60 @@ async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult
     set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
 
     let client = dist.client(1)?;
-    put_object(&client, &bucket, "small.bin", vec![0u8; 1024]).await?;
+    retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
+    wait_until(
+        Duration::from_secs(30),
+        || async {
+            let (status, body) = super::harness::cluster_admin(
+                &dist.cluster,
+                Method::GET,
+                &format!("/rustfs/admin/v3/quota-stats/{bucket}"),
+                None,
+            )
+            .await?;
+            if !status.is_success() {
+                return Ok(false);
+            }
+            let stats: serde_json::Value = serde_json::from_str(&body).unwrap_or_default();
+            Ok(stats.get("current_usage").and_then(serde_json::Value::as_u64).unwrap_or(0) >= 1024)
+        },
+        "quota stats observe small object",
+    )
+    .await?;
 
-    let over_limit = client
+    let mut oversized_attempt = 0u32;
-        .put_object()
+    wait_until(
-        .bucket(&bucket)
+        Duration::from_secs(30),
-        .key("too-big.bin")
+        || {
-        .body(vec![0u8; 16 * 1024].into())
+            oversized_attempt += 1;
-        .send()
+            let key = format!("too-big-{oversized_attempt}.bin");
-        .await;
+            let client = client.clone();
-    match over_limit {
+            let bucket = bucket.clone();
-        Ok(_) => {
+            async move {
-            // Scanner-backed quota can lag a cycle; a second over-quota PUT must fail.
+                match client
-            let second = client
+                    .put_object()
-                .put_object()
+                    .bucket(&bucket)
-                .bucket(&bucket)
+                    .key(key)
-                .key("too-big-2.bin")
+                    .body(vec![0u8; 16 * 1024].into())
-                .body(vec![0u8; 16 * 1024].into())
+                    .send()
-                .send()
+                    .await
-                .await;
+                {
-            match second {
+                    Ok(_) => Ok(false),
-                Ok(_) => return Err("hard quota admitted two oversized PUTs on a 4x4 cluster".into()),
+                    Err(error) => {
-                Err(error) => {
+                        let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
-                    let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+                        if matches!(code, Some("QuotaExceeded" | "SlowDown" | "AccessDenied" | "InvalidRequest")) {
-                    assert!(
+                            Ok(true)
-                        matches!(code, Some("QuotaExceeded" | "SlowDown" | "AccessDenied" | "InvalidRequest")),
+                        } else if matches!(code, Some("ServiceUnavailable")) {
-                        "unexpected over-quota error: {error:?}"
+                            Ok(false)
-                    );
+                        } else {
+                            Err(format!("unexpected over-quota error: {error:?}").into())
+                        }
+                    }
                 }
             }
-        }
+        },
-        Err(error) => {
+        "hard quota rejects oversized PUT",
-            let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
+    )
-            assert!(
+    .await?;
-                matches!(code, Some("QuotaExceeded" | "SlowDown" | "AccessDenied" | "InvalidRequest")),
-                "unexpected over-quota error: {error:?}"
-            );
-        }
-    }
     Ok(())
 }

crates/ecstore/src/core/pools.rs

@@ -4407,8 +4407,19 @@ pub(crate) enum PoolMetaBootstrapAuthority {
 }
 
 impl PoolMetaBootstrapAuthority {
+    /// Merge per-pool format-load proofs into one cluster bootstrap proof.
+    ///
+    /// `None` means this pool did not prove bootstrap itself: it loaded an
+    /// already-written `format.json`, usually formatted by that pool's first-disk
+    /// peer. That must not cancel a `Fresh` or `LegacyAdoption` proof from another
+    /// pool. Two different proven authorities still collapse to `None`.
     pub(crate) fn combine_across_pools(self, other: Self) -> Self {
-        if self == other { self } else { Self::None }
+        match (self, other) {
+            (Self::None, proven) | (proven, Self::None) => proven,
+            (Self::Fresh, Self::Fresh) => Self::Fresh,
+            (Self::LegacyAdoption, Self::LegacyAdoption) => Self::LegacyAdoption,
+            (Self::Fresh, Self::LegacyAdoption) | (Self::LegacyAdoption, Self::Fresh) => Self::None,
+        }
     }
 
     fn is_proven(self) -> bool {
@@ -4416,6 +4427,23 @@ impl PoolMetaBootstrapAuthority {
     }
 }
 
+#[cfg(test)]
+mod bootstrap_authority_combine_tests {
+    use super::PoolMetaBootstrapAuthority::*;
+
+    #[test]
+    fn none_does_not_cancel_a_proven_pool() {
+        assert_eq!(Fresh.combine_across_pools(None), Fresh);
+        assert_eq!(None.combine_across_pools(Fresh), Fresh);
+        assert_eq!(LegacyAdoption.combine_across_pools(None), LegacyAdoption);
+        assert_eq!(None.combine_across_pools(LegacyAdoption), LegacyAdoption);
+        assert_eq!(Fresh.combine_across_pools(Fresh), Fresh);
+        assert_eq!(None.combine_across_pools(None), None);
+        assert_eq!(Fresh.combine_across_pools(LegacyAdoption), None);
+        assert_eq!(LegacyAdoption.combine_across_pools(Fresh), None);
+    }
+}
+
 impl PoolMetaWriteState {
     #[cfg(test)]
     pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self {

docs/testing/distributed-e2e.md

@@ -28,7 +28,7 @@ A pool striped across several localhost ports is not expressible (`RUSTFS_VOLUME
 - Pool expand, decommission, rebalance, checksum integrity, S3 during move
 - Site replication object convergence
 - High-concurrency PUT/GET; concurrent PUT during decommission
-- Node kill/restart, full process restart, drive offline, volume-proxy blackhole
+- Node kill/restart, full process restart, drive offline, volume-proxy blackhole (4-node 1-drive DistErasure; a 4×4 volume proxy cannot format)
 - Multipart and cross-node listing agreement
 
 ## Existing Actions gaps this lane does not replace