diff --git a/crates/e2e_test/src/distributed/chaos_test.rs b/crates/e2e_test/src/distributed/chaos_test.rs index 5c066b1e7..28099f1d8 100644 --- a/crates/e2e_test/src/distributed/chaos_test.rs +++ b/crates/e2e_test/src/distributed/chaos_test.rs @@ -75,9 +75,11 @@ async fn offline_drive_then_replace_keeps_object_readable() -> TestResult { #[tokio::test] async fn volume_proxy_blackhole_then_restore_keeps_s3_available() -> TestResult { init_logging(); + // 4 nodes × 1 drive (4-disk DistErasure). Proxying a 16-disk 4×4 set + // prevents first-disk format: proxied drives look like missing peers, so + // `should_init_erasure_disks` is false and the first disk waits out. let mut cluster = - crate::common::RustFSTestClusterEnvironment::with_topology(crate::common::ClusterTopology::single_pool_multidrive(4, 4)) - .await?; + crate::common::RustFSTestClusterEnvironment::with_topology(crate::common::ClusterTopology::single_pool(4)).await?; let proxy = cluster.start_volume_proxy_for_node(1).await?; cluster.start().await?; cluster.create_test_bucket("chaos-net").await?; diff --git a/crates/e2e_test/src/distributed/replication_quota_test.rs b/crates/e2e_test/src/distributed/replication_quota_test.rs index 7f5967ffd..9c8112ab9 100644 --- a/crates/e2e_test/src/distributed/replication_quota_test.rs +++ b/crates/e2e_test/src/distributed/replication_quota_test.rs @@ -13,11 +13,12 @@ // limitations under the License. use super::harness::{ - DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, set_bucket_quota, - set_remote_target, unique_bucket, wait_for_replicated_bytes, + DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota, + set_remote_target, unique_bucket, wait_for_replicated_bytes, wait_until, }; use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging}; use aws_sdk_s3::error::ProvideErrorMetadata; +use http::Method; use std::time::Duration; #[tokio::test] @@ -56,43 +57,60 @@ async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?; let client = dist.client(1)?; - put_object(&client, &bucket, "small.bin", vec![0u8; 1024]).await?; + retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?; + wait_until( + Duration::from_secs(30), + || async { + let (status, body) = super::harness::cluster_admin( + &dist.cluster, + Method::GET, + &format!("/rustfs/admin/v3/quota-stats/{bucket}"), + None, + ) + .await?; + if !status.is_success() { + return Ok(false); + } + let stats: serde_json::Value = serde_json::from_str(&body).unwrap_or_default(); + Ok(stats.get("current_usage").and_then(serde_json::Value::as_u64).unwrap_or(0) >= 1024) + }, + "quota stats observe small object", + ) + .await?; - let over_limit = client - .put_object() - .bucket(&bucket) - .key("too-big.bin") - .body(vec![0u8; 16 * 1024].into()) - .send() - .await; - match over_limit { - Ok(_) => { - // Scanner-backed quota can lag a cycle; a second over-quota PUT must fail. - let second = client - .put_object() - .bucket(&bucket) - .key("too-big-2.bin") - .body(vec![0u8; 16 * 1024].into()) - .send() - .await; - match second { - Ok(_) => return Err("hard quota admitted two oversized PUTs on a 4x4 cluster".into()), - Err(error) => { - let code = error.as_service_error().and_then(ProvideErrorMetadata::code); - assert!( - matches!(code, Some("QuotaExceeded" | "SlowDown" | "AccessDenied" | "InvalidRequest")), - "unexpected over-quota error: {error:?}" - ); + let mut oversized_attempt = 0u32; + wait_until( + Duration::from_secs(30), + || { + oversized_attempt += 1; + let key = format!("too-big-{oversized_attempt}.bin"); + let client = client.clone(); + let bucket = bucket.clone(); + async move { + match client + .put_object() + .bucket(&bucket) + .key(key) + .body(vec![0u8; 16 * 1024].into()) + .send() + .await + { + Ok(_) => Ok(false), + Err(error) => { + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + if matches!(code, Some("QuotaExceeded" | "SlowDown" | "AccessDenied" | "InvalidRequest")) { + Ok(true) + } else if matches!(code, Some("ServiceUnavailable")) { + Ok(false) + } else { + Err(format!("unexpected over-quota error: {error:?}").into()) + } + } } } - } - Err(error) => { - let code = error.as_service_error().and_then(ProvideErrorMetadata::code); - assert!( - matches!(code, Some("QuotaExceeded" | "SlowDown" | "AccessDenied" | "InvalidRequest")), - "unexpected over-quota error: {error:?}" - ); - } - } + }, + "hard quota rejects oversized PUT", + ) + .await?; Ok(()) } diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 4973f6831..9780d0630 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -4407,8 +4407,19 @@ pub(crate) enum PoolMetaBootstrapAuthority { } impl PoolMetaBootstrapAuthority { + /// Merge per-pool format-load proofs into one cluster bootstrap proof. + /// + /// `None` means this pool did not prove bootstrap itself: it loaded an + /// already-written `format.json`, usually formatted by that pool's first-disk + /// peer. That must not cancel a `Fresh` or `LegacyAdoption` proof from another + /// pool. Two different proven authorities still collapse to `None`. pub(crate) fn combine_across_pools(self, other: Self) -> Self { - if self == other { self } else { Self::None } + match (self, other) { + (Self::None, proven) | (proven, Self::None) => proven, + (Self::Fresh, Self::Fresh) => Self::Fresh, + (Self::LegacyAdoption, Self::LegacyAdoption) => Self::LegacyAdoption, + (Self::Fresh, Self::LegacyAdoption) | (Self::LegacyAdoption, Self::Fresh) => Self::None, + } } fn is_proven(self) -> bool { @@ -4416,6 +4427,23 @@ impl PoolMetaBootstrapAuthority { } } +#[cfg(test)] +mod bootstrap_authority_combine_tests { + use super::PoolMetaBootstrapAuthority::*; + + #[test] + fn none_does_not_cancel_a_proven_pool() { + assert_eq!(Fresh.combine_across_pools(None), Fresh); + assert_eq!(None.combine_across_pools(Fresh), Fresh); + assert_eq!(LegacyAdoption.combine_across_pools(None), LegacyAdoption); + assert_eq!(None.combine_across_pools(LegacyAdoption), LegacyAdoption); + assert_eq!(Fresh.combine_across_pools(Fresh), Fresh); + assert_eq!(None.combine_across_pools(None), None); + assert_eq!(Fresh.combine_across_pools(LegacyAdoption), None); + assert_eq!(LegacyAdoption.combine_across_pools(Fresh), None); + } +} + impl PoolMetaWriteState { #[cfg(test)] pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self { diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md index 5f4148418..43947cea4 100644 --- a/docs/testing/distributed-e2e.md +++ b/docs/testing/distributed-e2e.md @@ -28,7 +28,7 @@ A pool striped across several localhost ports is not expressible (`RUSTFS_VOLUME - Pool expand, decommission, rebalance, checksum integrity, S3 during move - Site replication object convergence - High-concurrency PUT/GET; concurrent PUT during decommission -- Node kill/restart, full process restart, drive offline, volume-proxy blackhole +- Node kill/restart, full process restart, drive offline, volume-proxy blackhole (4-node 1-drive DistErasure; a 4×4 volume proxy cannot format) - Multipart and cross-node listing agreement ## Existing Actions gaps this lane does not replace