mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-17 13:34:36 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ebe98daaaf |
@@ -170,6 +170,15 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -181,7 +190,13 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,6 +732,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -756,7 +757,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready().await?;
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -770,7 +771,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -782,7 +783,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -800,6 +801,18 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -828,8 +841,16 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -838,7 +859,16 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -283,6 +283,7 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -430,6 +431,138 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
|
||||
let disk = match new_disk(ep, opt).await {
|
||||
Ok(disk) => disk,
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied) => {
|
||||
// LocalDisk already logged the filesystem cause and repair
|
||||
// hint. Keep this slot unavailable so ECStore initialization
|
||||
// can apply its existing fatal-error and quorum rules.
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
|
||||
// Preserve the configured slot when a local volume is unavailable.
|
||||
// LocalDisk logs the cause; format loading retains authority
|
||||
// over fatal errors and quorum. Never create a missing root.
|
||||
global_local_disk_map.insert(ep.to_string(), None);
|
||||
continue;
|
||||
}
|
||||
@@ -888,6 +888,95 @@ mod tests {
|
||||
disk_map.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
|
||||
use crate::disk::{DiskOption, error::DiskError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
for drive_count in [2, 3, 4] {
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..drive_count {
|
||||
let path = directory.path().join(format!("disk{index}"));
|
||||
if index != 0 {
|
||||
std::fs::create_dir(&path).expect("create surviving volume");
|
||||
}
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let missing = endpoints[0].to_string();
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: drive_count,
|
||||
endpoints: endpoints.into(),
|
||||
cmd_line: "missing-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
|
||||
.await
|
||||
.expect("missing volume should remain offline for quorum evaluation");
|
||||
let map = ctx.local_disk_map();
|
||||
let map = map.read().await;
|
||||
assert_eq!(map.len(), drive_count);
|
||||
assert!(matches!(map.get(&missing), Some(None)));
|
||||
assert_eq!(map.values().flatten().count(), drive_count - 1);
|
||||
drop(map);
|
||||
let drives = ctx.local_disk_set_drives();
|
||||
let drives = drives.read().await;
|
||||
assert_eq!(drives[0][0].len(), drive_count);
|
||||
assert!(drives[0][0][0].is_none());
|
||||
assert!(drives[0][0][1..].iter().all(Option::is_some));
|
||||
drop(drives);
|
||||
|
||||
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
|
||||
assert!(disks[0].is_none());
|
||||
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
|
||||
assert!(disks[1..].iter().all(Option::is_some));
|
||||
assert!(errors[1..].iter().all(Option::is_none));
|
||||
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_volume_still_fails_registry_initialization() {
|
||||
use crate::disk::DiskOption;
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let path = directory.path().join("regular-file");
|
||||
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: vec![endpoint].into(),
|
||||
cmd_line: "invalid-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools, &options)
|
||||
.await
|
||||
.expect_err("a non-directory volume must retain its initialization error");
|
||||
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
|
||||
use crate::disk::DiskOption;
|
||||
|
||||
@@ -169,36 +169,18 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
let partial_native_with_blanks = formats_present > 0
|
||||
&& errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk)))
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err)
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err);
|
||||
let resumable_partial_migration = partial_native_with_blanks
|
||||
&& formats.iter().zip(&errs).all(|(format, error)| {
|
||||
format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound))
|
||||
});
|
||||
|
||||
if first_disk && (all_unformatted || resumable_partial_migration) {
|
||||
if (first_disk && all_unformatted) || partial_native_with_blanks {
|
||||
info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format");
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
let allow_migration = first_disk && (all_unformatted || resumable_partial_migration);
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count, allow_migration).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, None, set_drive_count).await?;
|
||||
@@ -220,7 +202,7 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
Ok(LegacyFormatOutcome::None) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
if fresh_bootstrap_proven {
|
||||
if first_disk && fresh_bootstrap_proven {
|
||||
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
@@ -229,6 +211,28 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve legacy migration before admitting native reads. Blank replacement
|
||||
// slots contribute no votes and receive no formatting/bootstrap authority.
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
info!(
|
||||
"first_disk: {}, should_init_erasure_disks: {}",
|
||||
first_disk,
|
||||
@@ -271,7 +275,7 @@ async fn verified_legacy_adoption_source(
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
) -> Result<PoolMetaBootstrapAuthority> {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count, true).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
|
||||
Err(err) => {
|
||||
@@ -451,12 +455,14 @@ enum LegacyFormatOutcome {
|
||||
///
|
||||
/// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present
|
||||
/// and, if so, whether it was compatible. `Err` is returned for read or persist
|
||||
/// failures that prevent a conclusive migration decision.
|
||||
/// failures that prevent a conclusive migration decision. With `allow_migration`
|
||||
/// disabled, a compatible legacy quorum returns `NotFirstDisk` before any writes.
|
||||
async fn try_migrate_format(
|
||||
disks: &[Option<DiskStore>],
|
||||
rustfs_formats: &[Option<FormatV3>],
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
allow_migration: bool,
|
||||
) -> Result<LegacyFormatOutcome> {
|
||||
let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?;
|
||||
let mut legacy_seen = false;
|
||||
@@ -540,6 +546,12 @@ async fn try_migrate_format(
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
|
||||
// Followers inspect legacy authority before admitting partial native formats,
|
||||
// but only the elected creator may persist the migration.
|
||||
if !allow_migration {
|
||||
return Err(Error::NotFirstDisk);
|
||||
}
|
||||
|
||||
let mut formats_to_write = vec![None; disks.len()];
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
if disk.is_some() {
|
||||
@@ -677,7 +689,7 @@ fn select_existing_format_read_quorum(
|
||||
|| check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err()
|
||||
|| formats.iter().zip(errors).any(|(format, error)| match (format, error) {
|
||||
(Some(_), None) => false,
|
||||
(None, Some(DiskError::DiskNotFound)) => false,
|
||||
(None, Some(DiskError::DiskNotFound | DiskError::UnformattedDisk)) => false,
|
||||
(None, Some(error)) => !is_network_like_disk_error(error),
|
||||
_ => true,
|
||||
})
|
||||
@@ -1445,13 +1457,23 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() {
|
||||
fn existing_format_read_quorum_excludes_blank_and_network_unavailable_slots() {
|
||||
let (formats, mut errors) = half_pool_formats();
|
||||
errors[2] = Some(DiskError::UnformattedDisk);
|
||||
errors[3] = Some(DiskError::Timeout);
|
||||
let (_, members) = select_existing_format_read_quorum(&formats, &errors, 4)
|
||||
.expect("blank and unreachable slots cannot block an existing native read quorum");
|
||||
assert!(!members[2] && !members[3]);
|
||||
assert_eq!(members.iter().filter(|member| **member).count(), 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_and_invalid_slots() {
|
||||
for failure in [
|
||||
"conflict",
|
||||
"wrong_slot",
|
||||
"nil",
|
||||
"duplicate",
|
||||
"blank",
|
||||
"corrupt",
|
||||
"access_denied",
|
||||
"tie",
|
||||
@@ -1473,7 +1495,6 @@ mod tests {
|
||||
format.erasure.sets[5][3] = format.erasure.sets[0][0];
|
||||
}
|
||||
}
|
||||
"blank" => errors[2] = Some(DiskError::UnformattedDisk),
|
||||
"corrupt" => errors[2] = Some(DiskError::CorruptedFormat),
|
||||
"access_denied" => errors[2] = Some(DiskError::FileAccessDenied),
|
||||
"tie" => {
|
||||
@@ -1532,6 +1553,62 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_read_quorum_quarantines_blank_replacement_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for surviving_slot in 0..2 {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let blank = 1 - surviving_slot;
|
||||
let mut format = FormatV3::new(1, 2);
|
||||
format.erasure.this = format.erasure.sets[0][surviving_slot];
|
||||
save_format_file(&disks[surviving_slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist surviving format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
2,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("existing native format must load at read quorum with a blank replacement");
|
||||
assert_eq!(loaded.format.shared_identity(), format.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[surviving_slot].is_some());
|
||||
assert!(disks[blank].is_none(), "blank replacement cannot vote or serve data");
|
||||
assert_eq!(format_bytes(&original).await, before, "read admission cannot write either format");
|
||||
assert!(matches!(
|
||||
load_format_erasure(original[blank].as_ref().expect("blank handle"), false).await,
|
||||
Err(DiskError::UnformattedDisk)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_native_format_with_legacy_quorum_waits_for_creator_without_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let legacy = FormatV3::new(1, 2);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut native = legacy.clone();
|
||||
native.erasure.this = native.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(native))
|
||||
.await
|
||||
.expect("persist partial native format");
|
||||
let before = format_bytes(&disks).await;
|
||||
let result =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 2, None).await;
|
||||
assert!(
|
||||
matches!(result, Err(Error::NotFirstDisk)),
|
||||
"follower must await the legacy migration writer"
|
||||
);
|
||||
assert_eq!(format_bytes(&disks).await, before, "follower legacy inspection cannot persist formats");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_load_succeeds_with_a_strict_majority() {
|
||||
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
|
||||
@@ -2375,7 +2452,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_resumes_partial_rustfs_writes() {
|
||||
for drive_count in [3, 4] {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
@@ -2434,23 +2511,27 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() {
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
let legacy = FormatV3::new(1, 3);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, 3);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
for first_disk in [false, true] {
|
||||
for drive_count in [2, 3] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, drive_count);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
|
||||
assert!(matches!(
|
||||
connect_load_init_formats(true, &mut disks, 1, 3, None).await,
|
||||
Err(Error::CorruptedFormat)
|
||||
));
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
assert!(matches!(
|
||||
connect_load_init_formats(first_disk, &mut disks, 1, drive_count, None).await,
|
||||
Err(Error::CorruptedFormat)
|
||||
));
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -22,7 +22,7 @@ The outer gate uses the same HTTP/2 and `application/grpc` predicate as the hybr
|
||||
|---|---|---|---|
|
||||
| StorageReady | Storage/global config readiness publication plus runtime storage checks. | Yes. | Startup can mark the stage before later runtime readiness rechecks storage. Initial pool metadata reads must complete before object-store publication; see [pool metadata recovery](../operations/pool-metadata-recovery.md). |
|
||||
| IamReady | Inline IAM bootstrap or deferred IAM recovery publication. | Yes. | Deferred recovery can publish IAM readiness after HTTP has already started. |
|
||||
| Lock quorum | Per-set write quorum readiness. | Yes. | Do not replace the distributed lock quorum check with node count or endpoint count. |
|
||||
| Lock quorum | Per-set shared-lock quorum readiness. | Yes. | Startup uses the same read boundary as node readiness. Mutations retain their exclusive-lock requirements. Do not replace the distributed lock quorum check with node count or endpoint count. |
|
||||
| Peer health | `peer_health_ready` runtime status. | Only when `RUSTFS_HEALTH_PEER_READY_CHECK_ENABLE` is enabled. | The gate is disabled by default; unknown peer health degrades readiness only when enabled. |
|
||||
| Peer re-admission | A strict majority of the peers in each erasure set this node serves count all of its drives in that set as online (`PeerDriveAdmission` internode RPC). | Startup transition only, while `RUSTFS_STARTUP_PEER_READMISSION_ENABLE` is enabled (default on). | Bounded and fail-open: publishes `FullReady` with a WARN log when `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` elapses, immediately when older peers cannot answer and the majority is out of reach, or after 15 s of unreachable peers keeping it out of reach. Not part of steady-state `/health/ready`. Distinct from peer health, which is this node's view of its peers. |
|
||||
| KMS compatibility | KMS health compatibility readiness. | Only when `RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE` is enabled (default off; `crates/config/src/constants/health.rs`). | When enabled, `/health/ready` additionally requires the KMS service to be running if a global KMS manager exists. KMS startup fatality and health reporting remain separate from pure docs work. |
|
||||
|
||||
@@ -1405,7 +1405,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn node_storage_snapshot_tracks_read_write_quorum_and_recovery() {
|
||||
for (drive_count, parity) in [(4, 2), (4, 1), (8, 2)] {
|
||||
for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (4, 1), (8, 2)] {
|
||||
let store = runtime_inventory(&[(1, drive_count, parity)]).await;
|
||||
let data = drive_count - parity;
|
||||
let write_quorum = data + usize::from(data == parity);
|
||||
@@ -1690,6 +1690,54 @@ mod tests {
|
||||
assert!(!readiness_gate_blocks_path("/bucket/object", &readiness));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_drive_startup_admits_reads_without_write_quorum_but_requires_iam() {
|
||||
let mut store = runtime_inventory(&[(1, 2, 1)]).await;
|
||||
store.disks.get_mut(&DiskSetSelector::new(0, 0)).expect("configured set")[1] = None;
|
||||
|
||||
for (online, iam_ready, expected_ready) in [(true, true, true), (true, false, false), (false, true, false)] {
|
||||
let hosts = if online {
|
||||
HashSet::from(["node-0:9000".to_string()])
|
||||
} else {
|
||||
HashSet::new()
|
||||
};
|
||||
let (info, unavailable_drives, missing_drives) = node_storage_snapshot(&store, &hosts)
|
||||
.await
|
||||
.expect("snapshot surviving drive and missing slot");
|
||||
assert_eq!(missing_drives.len(), 1);
|
||||
let (dependencies, _, details) = apply_node_service_readiness(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetaWriteBlocked),
|
||||
},
|
||||
StorageReadinessDetails {
|
||||
read_quorum_ready: storage_read_ready_from_runtime_state(&info),
|
||||
write_quorum_ready: storage_ready_from_runtime_state(&info),
|
||||
unavailable_drives,
|
||||
missing_drives,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
iam_ready,
|
||||
true,
|
||||
);
|
||||
assert!(!details.write_quorum_ready, "one surviving drive cannot authorize writes");
|
||||
assert!(!details.pool_metadata_write_ready);
|
||||
let readiness = GlobalReadiness::new();
|
||||
assert!(readiness_gate_blocks_path("/bucket/object", &readiness));
|
||||
let result = wait_for_runtime_readiness_with(
|
||||
Duration::ZERO,
|
||||
Duration::from_millis(1),
|
||||
|| future::ready(dependencies),
|
||||
|_| readiness.mark_stage(rustfs_common::SystemStage::FullReady),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.is_ok(), expected_ready, "online={online}, iam_ready={iam_ready}");
|
||||
assert_eq!(readiness.is_ready(), expected_ready);
|
||||
assert_eq!(readiness_gate_blocks_path("/bucket/object", &readiness), !expected_ready);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn node_storage_snapshot_bounds_waits_and_propagates_inventory_failure() {
|
||||
let mut store = runtime_inventory(&[(1, 4, 2)]).await;
|
||||
|
||||
Reference in New Issue
Block a user