Compare commits

..

1 Commits

Author SHA1 Message Date
Hauser Zhi ebe98daaaf fix(ecstore): admit degraded local startup at read quorum
Keep absent local volumes in configured offline slots and admit existing native formats with nonvoting blank replacements. Preserve legacy validation and elected migration authority before degraded reads. Add startup, migration, readiness and real S3 cold-start regressions.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-17 20:05:35 +08:00
7 changed files with 456 additions and 60 deletions
+16 -1
View File
@@ -170,6 +170,15 @@ impl DiskFaultHarness {
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
/// all volume directories and the same address.
pub async fn start_server(&mut self) -> ChaosResult<()> {
self.start_server_with_readiness(false).await
}
/// Start a process once its node read-readiness probe passes.
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
self.start_server_with_readiness(true).await
}
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
// The common helper always appends `env.temp_dir` as the final storage
// path: point it at the last disk for startup and pass the remaining
// disks explicitly, then restore it to the root so Drop cleans up
@@ -181,7 +190,13 @@ impl DiskFaultHarness {
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
let result = if read_quorum_only {
self.env
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
.await
} else {
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
};
self.env.temp_dir = self.root.to_string_lossy().to_string();
result
}
+35 -5
View File
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
cleanup_existing: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let binary_path = rustfs_binary_path();
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
.await
}
@@ -732,6 +732,7 @@ impl RustFSTestEnvironment {
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
cleanup_existing: bool,
read_quorum_only: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
if cleanup_existing {
self.cleanup_existing_processes().await?;
@@ -756,7 +757,7 @@ impl RustFSTestEnvironment {
self.process = Some(process);
// Wait for server to be ready
self.wait_for_server_ready().await?;
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
Ok(())
}
@@ -770,7 +771,7 @@ impl RustFSTestEnvironment {
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
.await
}
@@ -782,7 +783,7 @@ impl RustFSTestEnvironment {
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
.await
}
@@ -800,6 +801,18 @@ impl RustFSTestEnvironment {
self.start_rustfs_server_inner(extra_args, extra_env, true).await
}
/// Start a degraded read-serving process using the node readiness probe.
/// ListBuckets retains its namespace write-quorum requirement.
pub async fn start_rustfs_server_with_env_at_read_quorum(
&mut self,
extra_args: Vec<&str>,
extra_env: &[(&str, &str)],
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let binary_path = rustfs_binary_path();
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
.await
}
/// Start RustFS server without cleaning up other running RustFS processes.
///
/// This is useful for tests that need multiple independent RustFS instances
@@ -828,8 +841,16 @@ impl RustFSTestEnvironment {
/// Fails immediately if the spawned server process has already exited
/// (e.g. a port bind failure) instead of polling out the full timeout.
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.wait_for_server_ready_with_probe(false).await
}
async fn wait_for_server_ready_with_probe(
&mut self,
read_quorum_only: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
info!("Waiting for RustFS server to be ready on {}", self.address);
let client = self.create_s3_client();
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
for i in 0..60 {
if let Some(process) = self.process.as_mut()
@@ -838,7 +859,16 @@ impl RustFSTestEnvironment {
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
}
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
let ready = if read_quorum_only {
health_client
.get(format!("{}/minio/health/ready", self.url))
.send()
.await
.is_ok_and(|response| response.status().is_success())
} else {
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
};
if ready {
info!("✅ RustFS server is ready after {} attempts", i + 1);
return Ok(());
}
@@ -283,6 +283,7 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
#[tokio::test]
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
crate::common::init_logging();
assert_single_node_cold_start_with_missing_volume().await?;
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
let read_quorum = node_count - parity;
@@ -430,6 +431,138 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
Ok(())
}
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
for drive_count in [2, 3, 4] {
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
server.set_env("RUST_LOG", "warn");
server.start_server_at_read_quorum().await?;
let client = server.env.create_s3_client();
let client = Client::from_conf(
client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
.build(),
);
let bucket = format!("missing-volume-{drive_count}");
client.create_bucket().bucket(&bucket).send().await?;
let small = vec![17; 4096];
let large = vec![73; 8 * 1024 * 1024];
for (key, body) in [("small", &small), ("large", &large)] {
client
.put_object()
.bucket(&bucket)
.key(key)
.body(Bytes::copy_from_slice(body).into())
.send()
.await?;
for disk in 0..drive_count {
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
}
}
server.kill_server();
let missing = drive_count - 1;
server.take_disk_offline(missing)?;
server.start_server_at_read_quorum().await?;
for (key, body) in [("small", &small), ("large", &large)] {
assert_quorum_object_body(&client, &bucket, key, body).await?;
let head = client.head_object().bucket(&bucket).key(key).send().await?;
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
}
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
assert_eq!(keys, ["large", "small"]);
let put = client
.put_object()
.bucket(&bucket)
.key("degraded-write")
.body(Bytes::from_static(b"degraded write").into())
.send()
.await;
if drive_count == 2 {
let error = put.expect_err("one of two drives cannot commit a new object");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
let error = client
.delete_object()
.bucket(&bucket)
.key("large")
.send()
.await
.expect_err("one of two drives cannot commit an object deletion");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
} else {
put?;
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
}
assert!(
!server.disk_path(missing).exists(),
"degraded startup and S3 operations must leave the root absent"
);
if drive_count == 2 {
server.kill_server();
std::fs::create_dir(server.disk_path(missing))?;
server.start_server_at_read_quorum().await?;
for (key, body) in [("small", &small), ("large", &large)] {
assert_quorum_object_body(&client, &bucket, key, body).await?;
client.head_object().bucket(&bucket).key(key).send().await?;
}
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
let error = client
.put_object()
.bucket(&bucket)
.key("empty-root-write")
.body(Bytes::from_static(b"must remain unavailable").into())
.send()
.await
.expect_err("an empty replacement cannot restore write quorum");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
assert!(
!server
.disk_path(missing)
.join(crate::storage_api::RUSTFS_META_BUCKET)
.join("format.json")
.exists(),
"read admission must not format the empty replacement"
);
}
server.kill_server();
server.bring_disk_online(missing)?;
server.start_server_at_read_quorum().await?;
for (key, body) in [("small", &small), ("large", &large)] {
assert_quorum_object_body(&client, &bucket, key, body).await?;
}
for key in ["degraded-write", "empty-root-write"] {
let error = client
.head_object()
.bucket(&bucket)
.key(key)
.send()
.await
.expect_err("rejected or deleted degraded write must stay absent after recovery");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
}
client
.put_object()
.bucket(&bucket)
.key("recovery-write")
.body(Bytes::from_static(b"recovered write").into())
.send()
.await?;
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
}
Ok(())
}
#[tokio::test]
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
crate::common::init_logging();
+93 -4
View File
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
let disk = match new_disk(ep, opt).await {
Ok(disk) => disk,
Err(crate::disk::error::DiskError::FileAccessDenied) => {
// LocalDisk already logged the filesystem cause and repair
// hint. Keep this slot unavailable so ECStore initialization
// can apply its existing fatal-error and quorum rules.
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
// Preserve the configured slot when a local volume is unavailable.
// LocalDisk logs the cause; format loading retains authority
// over fatal errors and quorum. Never create a missing root.
global_local_disk_map.insert(ep.to_string(), None);
continue;
}
@@ -888,6 +888,95 @@ mod tests {
disk_map.keys().collect::<Vec<_>>()
);
}
#[tokio::test]
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
use crate::disk::{DiskOption, error::DiskError};
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
for drive_count in [2, 3, 4] {
let directory = tempfile::tempdir().expect("create volume parent");
let mut endpoints = Vec::new();
for index in 0..drive_count {
let path = directory.path().join(format!("disk{index}"));
if index != 0 {
std::fs::create_dir(&path).expect("create surviving volume");
}
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(index);
endpoints.push(endpoint);
}
let missing = endpoints[0].to_string();
let pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: drive_count,
endpoints: endpoints.into(),
cmd_line: "missing-volume".into(),
platform: "test".into(),
}]);
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let options = DiskOption {
cleanup: true,
health_check: false,
};
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
.await
.expect("missing volume should remain offline for quorum evaluation");
let map = ctx.local_disk_map();
let map = map.read().await;
assert_eq!(map.len(), drive_count);
assert!(matches!(map.get(&missing), Some(None)));
assert_eq!(map.values().flatten().count(), drive_count - 1);
drop(map);
let drives = ctx.local_disk_set_drives();
let drives = drives.read().await;
assert_eq!(drives[0][0].len(), drive_count);
assert!(drives[0][0][0].is_none());
assert!(drives[0][0][1..].iter().all(Option::is_some));
drop(drives);
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
assert!(disks[0].is_none());
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
assert!(disks[1..].iter().all(Option::is_some));
assert!(errors[1..].iter().all(Option::is_none));
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
}
}
#[tokio::test]
async fn invalid_local_volume_still_fails_registry_initialization() {
use crate::disk::DiskOption;
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
let directory = tempfile::tempdir().expect("create volume parent");
let path = directory.path().join("regular-file");
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 1,
endpoints: vec![endpoint].into(),
cmd_line: "invalid-volume".into(),
platform: "test".into(),
}]);
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let options = DiskOption {
cleanup: true,
health_check: false,
};
super::initialize_local_disk_maps(&ctx, pools, &options)
.await
.expect_err("a non-directory volume must retain its initialization error");
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
}
#[tokio::test]
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
use crate::disk::DiskOption;
+129 -48
View File
@@ -169,36 +169,18 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
{
return Err(Error::CorruptedFormat);
}
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
{
retain_format_quorum_members(
instance_ctx,
disks,
&format,
&members,
Some(&unformatted_bootstrap_routes),
set_drive_count,
)
.await?;
// Loading an existing format grants no authority to initialize pool
// metadata or migrate formats. The persisted storage classes and each
// object's EC metadata still determine service and operation quorums.
return Ok(LoadedFormat {
format,
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
});
}
let resumable_partial_migration = formats_present > 0
let partial_native_with_blanks = formats_present > 0
&& errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk)))
&& format_quorum.as_ref().is_some_and(Result::is_err)
&& format_quorum.as_ref().is_some_and(Result::is_err);
let resumable_partial_migration = partial_native_with_blanks
&& formats.iter().zip(&errs).all(|(format, error)| {
format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound))
});
if first_disk && (all_unformatted || resumable_partial_migration) {
if (first_disk && all_unformatted) || partial_native_with_blanks {
info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format");
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
let allow_migration = first_disk && (all_unformatted || resumable_partial_migration);
match try_migrate_format(disks, &formats, set_count, set_drive_count, allow_migration).await {
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
info!("Migrated format from MinIO config");
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, None, set_drive_count).await?;
@@ -220,7 +202,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
Ok(LegacyFormatOutcome::None) => {}
Err(e) => return Err(e),
}
if fresh_bootstrap_proven {
if first_disk && fresh_bootstrap_proven {
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
return Ok(LoadedFormat {
format: fm,
@@ -229,6 +211,28 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
}
}
// Resolve legacy migration before admitting native reads. Blank replacement
// slots contribute no votes and receive no formatting/bootstrap authority.
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
{
retain_format_quorum_members(
instance_ctx,
disks,
&format,
&members,
Some(&unformatted_bootstrap_routes),
set_drive_count,
)
.await?;
// Loading an existing format grants no authority to initialize pool
// metadata or migrate formats. The persisted storage classes and each
// object's EC metadata still determine service and operation quorums.
return Ok(LoadedFormat {
format,
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
});
}
info!(
"first_disk: {}, should_init_erasure_disks: {}",
first_disk,
@@ -271,7 +275,7 @@ async fn verified_legacy_adoption_source(
set_count: usize,
set_drive_count: usize,
) -> Result<PoolMetaBootstrapAuthority> {
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count, true).await {
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
Err(err) => {
@@ -451,12 +455,14 @@ enum LegacyFormatOutcome {
///
/// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present
/// and, if so, whether it was compatible. `Err` is returned for read or persist
/// failures that prevent a conclusive migration decision.
/// failures that prevent a conclusive migration decision. With `allow_migration`
/// disabled, a compatible legacy quorum returns `NotFirstDisk` before any writes.
async fn try_migrate_format(
disks: &[Option<DiskStore>],
rustfs_formats: &[Option<FormatV3>],
set_count: usize,
set_drive_count: usize,
allow_migration: bool,
) -> Result<LegacyFormatOutcome> {
let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?;
let mut legacy_seen = false;
@@ -540,6 +546,12 @@ async fn try_migrate_format(
return Ok(LegacyFormatOutcome::Incompatible);
}
// Followers inspect legacy authority before admitting partial native formats,
// but only the elected creator may persist the migration.
if !allow_migration {
return Err(Error::NotFirstDisk);
}
let mut formats_to_write = vec![None; disks.len()];
for (index, disk) in disks.iter().enumerate() {
if disk.is_some() {
@@ -677,7 +689,7 @@ fn select_existing_format_read_quorum(
|| check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err()
|| formats.iter().zip(errors).any(|(format, error)| match (format, error) {
(Some(_), None) => false,
(None, Some(DiskError::DiskNotFound)) => false,
(None, Some(DiskError::DiskNotFound | DiskError::UnformattedDisk)) => false,
(None, Some(error)) => !is_network_like_disk_error(error),
_ => true,
})
@@ -1445,13 +1457,23 @@ mod tests {
}
#[test]
fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() {
fn existing_format_read_quorum_excludes_blank_and_network_unavailable_slots() {
let (formats, mut errors) = half_pool_formats();
errors[2] = Some(DiskError::UnformattedDisk);
errors[3] = Some(DiskError::Timeout);
let (_, members) = select_existing_format_read_quorum(&formats, &errors, 4)
.expect("blank and unreachable slots cannot block an existing native read quorum");
assert!(!members[2] && !members[3]);
assert_eq!(members.iter().filter(|member| **member).count(), 12);
}
#[test]
fn existing_format_read_quorum_rejects_conflicts_and_invalid_slots() {
for failure in [
"conflict",
"wrong_slot",
"nil",
"duplicate",
"blank",
"corrupt",
"access_denied",
"tie",
@@ -1473,7 +1495,6 @@ mod tests {
format.erasure.sets[5][3] = format.erasure.sets[0][0];
}
}
"blank" => errors[2] = Some(DiskError::UnformattedDisk),
"corrupt" => errors[2] = Some(DiskError::CorruptedFormat),
"access_denied" => errors[2] = Some(DiskError::FileAccessDenied),
"tie" => {
@@ -1532,6 +1553,62 @@ mod tests {
}
}
#[tokio::test]
async fn existing_format_read_quorum_quarantines_blank_replacement_without_writes() {
for first_disk in [false, true] {
for surviving_slot in 0..2 {
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
let blank = 1 - surviving_slot;
let mut format = FormatV3::new(1, 2);
format.erasure.this = format.erasure.sets[0][surviving_slot];
save_format_file(&disks[surviving_slot], &Some(format.clone()))
.await
.expect("persist surviving format");
let original = disks.clone();
let before = format_bytes(&original).await;
let loaded = connect_load_init_formats_with_instance_ctx(
&Arc::new(InstanceContext::new()),
first_disk,
&mut disks,
1,
2,
None,
)
.await
.expect("existing native format must load at read quorum with a blank replacement");
assert_eq!(loaded.format.shared_identity(), format.shared_identity());
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
assert!(disks[surviving_slot].is_some());
assert!(disks[blank].is_none(), "blank replacement cannot vote or serve data");
assert_eq!(format_bytes(&original).await, before, "read admission cannot write either format");
assert!(matches!(
load_format_erasure(original[blank].as_ref().expect("blank handle"), false).await,
Err(DiskError::UnformattedDisk)
));
}
}
}
#[tokio::test]
async fn partial_native_format_with_legacy_quorum_waits_for_creator_without_writes() {
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
let legacy = FormatV3::new(1, 2);
write_legacy_majority(&disks, &legacy).await;
let mut native = legacy.clone();
native.erasure.this = native.erasure.sets[0][0];
save_format_file(&disks[0], &Some(native))
.await
.expect("persist partial native format");
let before = format_bytes(&disks).await;
let result =
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 2, None).await;
assert!(
matches!(result, Err(Error::NotFirstDisk)),
"follower must await the legacy migration writer"
);
assert_eq!(format_bytes(&disks).await, before, "follower legacy inspection cannot persist formats");
}
#[tokio::test]
async fn existing_format_load_succeeds_with_a_strict_majority() {
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
@@ -2375,7 +2452,7 @@ mod tests {
#[tokio::test]
async fn legacy_format_migration_resumes_partial_rustfs_writes() {
for drive_count in [3, 4] {
for drive_count in [2, 3, 4] {
let (_temp_dir, mut disks) = local_disks(drive_count).await;
let legacy = FormatV3::new(1, drive_count);
write_legacy_majority(&disks, &legacy).await;
@@ -2434,23 +2511,27 @@ mod tests {
#[tokio::test]
async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() {
let (_temp_dir, mut disks) = local_disks(3).await;
let legacy = FormatV3::new(1, 3);
write_legacy_majority(&disks, &legacy).await;
let mut foreign = FormatV3::new(1, 3);
foreign.erasure.this = foreign.erasure.sets[0][0];
let foreign_id = foreign.id;
save_format_file(&disks[0], &Some(foreign))
.await
.expect("foreign partial RustFS format should be written");
for first_disk in [false, true] {
for drive_count in [2, 3] {
let (_temp_dir, mut disks) = local_disks(drive_count).await;
let legacy = FormatV3::new(1, drive_count);
write_legacy_majority(&disks, &legacy).await;
let mut foreign = FormatV3::new(1, drive_count);
foreign.erasure.this = foreign.erasure.sets[0][0];
let foreign_id = foreign.id;
save_format_file(&disks[0], &Some(foreign))
.await
.expect("foreign partial RustFS format should be written");
assert!(matches!(
connect_load_init_formats(true, &mut disks, 1, 3, None).await,
Err(Error::CorruptedFormat)
));
let (formats, _) = load_format_erasure_all(&disks, false).await;
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
assert!(matches!(
connect_load_init_formats(first_disk, &mut disks, 1, drive_count, None).await,
Err(Error::CorruptedFormat)
));
let (formats, _) = load_format_erasure_all(&disks, false).await;
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
}
}
}
#[tokio::test]
+1 -1
View File
@@ -22,7 +22,7 @@ The outer gate uses the same HTTP/2 and `application/grpc` predicate as the hybr
|---|---|---|---|
| StorageReady | Storage/global config readiness publication plus runtime storage checks. | Yes. | Startup can mark the stage before later runtime readiness rechecks storage. Initial pool metadata reads must complete before object-store publication; see [pool metadata recovery](../operations/pool-metadata-recovery.md). |
| IamReady | Inline IAM bootstrap or deferred IAM recovery publication. | Yes. | Deferred recovery can publish IAM readiness after HTTP has already started. |
| Lock quorum | Per-set write quorum readiness. | Yes. | Do not replace the distributed lock quorum check with node count or endpoint count. |
| Lock quorum | Per-set shared-lock quorum readiness. | Yes. | Startup uses the same read boundary as node readiness. Mutations retain their exclusive-lock requirements. Do not replace the distributed lock quorum check with node count or endpoint count. |
| Peer health | `peer_health_ready` runtime status. | Only when `RUSTFS_HEALTH_PEER_READY_CHECK_ENABLE` is enabled. | The gate is disabled by default; unknown peer health degrades readiness only when enabled. |
| Peer re-admission | A strict majority of the peers in each erasure set this node serves count all of its drives in that set as online (`PeerDriveAdmission` internode RPC). | Startup transition only, while `RUSTFS_STARTUP_PEER_READMISSION_ENABLE` is enabled (default on). | Bounded and fail-open: publishes `FullReady` with a WARN log when `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` elapses, immediately when older peers cannot answer and the majority is out of reach, or after 15 s of unreachable peers keeping it out of reach. Not part of steady-state `/health/ready`. Distinct from peer health, which is this node's view of its peers. |
| KMS compatibility | KMS health compatibility readiness. | Only when `RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE` is enabled (default off; `crates/config/src/constants/health.rs`). | When enabled, `/health/ready` additionally requires the KMS service to be running if a global KMS manager exists. KMS startup fatality and health reporting remain separate from pure docs work. |
+49 -1
View File
@@ -1405,7 +1405,7 @@ mod tests {
#[tokio::test]
async fn node_storage_snapshot_tracks_read_write_quorum_and_recovery() {
for (drive_count, parity) in [(4, 2), (4, 1), (8, 2)] {
for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (4, 1), (8, 2)] {
let store = runtime_inventory(&[(1, drive_count, parity)]).await;
let data = drive_count - parity;
let write_quorum = data + usize::from(data == parity);
@@ -1690,6 +1690,54 @@ mod tests {
assert!(!readiness_gate_blocks_path("/bucket/object", &readiness));
}
#[tokio::test]
async fn two_drive_startup_admits_reads_without_write_quorum_but_requires_iam() {
let mut store = runtime_inventory(&[(1, 2, 1)]).await;
store.disks.get_mut(&DiskSetSelector::new(0, 0)).expect("configured set")[1] = None;
for (online, iam_ready, expected_ready) in [(true, true, true), (true, false, false), (false, true, false)] {
let hosts = if online {
HashSet::from(["node-0:9000".to_string()])
} else {
HashSet::new()
};
let (info, unavailable_drives, missing_drives) = node_storage_snapshot(&store, &hosts)
.await
.expect("snapshot surviving drive and missing slot");
assert_eq!(missing_drives.len(), 1);
let (dependencies, _, details) = apply_node_service_readiness(
StorageWriteReadinessStatus {
ready: false,
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetaWriteBlocked),
},
StorageReadinessDetails {
read_quorum_ready: storage_read_ready_from_runtime_state(&info),
write_quorum_ready: storage_ready_from_runtime_state(&info),
unavailable_drives,
missing_drives,
..Default::default()
},
true,
iam_ready,
true,
);
assert!(!details.write_quorum_ready, "one surviving drive cannot authorize writes");
assert!(!details.pool_metadata_write_ready);
let readiness = GlobalReadiness::new();
assert!(readiness_gate_blocks_path("/bucket/object", &readiness));
let result = wait_for_runtime_readiness_with(
Duration::ZERO,
Duration::from_millis(1),
|| future::ready(dependencies),
|_| readiness.mark_stage(rustfs_common::SystemStage::FullReady),
)
.await;
assert_eq!(result.is_ok(), expected_ready, "online={online}, iam_ready={iam_ready}");
assert_eq!(readiness.is_ready(), expected_ready);
assert_eq!(readiness_gate_blocks_path("/bucket/object", &readiness), !expected_ready);
}
}
#[tokio::test]
async fn node_storage_snapshot_bounds_waits_and_propagates_inventory_failure() {
let mut store = runtime_inventory(&[(1, 4, 2)]).await;