mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-18 05:53:13 +00:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4405a75d7c | |||
| 63e7481497 | |||
| 8f12b81f09 | |||
| 92ad98b1e5 | |||
| 69f8b27728 | |||
| 50952e10cf | |||
| 3ddbb80b20 | |||
| 941fa254fc | |||
| d4af3cee3e | |||
| 850640f452 | |||
| 1baaf0b84d | |||
| 83032a3671 | |||
| e110baa3e0 | |||
| f90137b73f | |||
| a5b69359d2 | |||
| c6a52d87cf | |||
| 2654b5331a | |||
| b7b9f630b4 | |||
| 994cd3a8c0 | |||
| ebe98daaaf |
Generated
+9
-9
@@ -10270,7 +10270,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bytes",
|
||||
@@ -10317,7 +10317,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-core"
|
||||
version = "0.63.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -10338,7 +10338,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-dialect-rustfs-admin"
|
||||
version = "0.13.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
@@ -10352,7 +10352,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-http"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
@@ -10370,7 +10370,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-macros"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -10380,7 +10380,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-sig"
|
||||
version = "0.15.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
@@ -10395,7 +10395,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
@@ -10406,7 +10406,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-types"
|
||||
version = "0.42.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
@@ -10429,7 +10429,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-xml"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=fb4d21fca840036faef784aa1b0e965b03451db3#fb4d21fca840036faef784aa1b0e965b03451db3"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=6096b9218ad78202c69367424c5ea1407bb49cc3#6096b9218ad78202c69367424c5ea1407bb49cc3"
|
||||
dependencies = [
|
||||
"quick-xml 0.41.0",
|
||||
]
|
||||
|
||||
+6
-6
@@ -151,16 +151,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "6096b9218ad78202c69367424c5ea1407bb49cc3", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "6096b9218ad78202c69367424c5ea1407bb49cc3", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "6096b9218ad78202c69367424c5ea1407bb49cc3", version = "0.66.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "fb4d21fca840036faef784aa1b0e965b03451db3", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "6096b9218ad78202c69367424c5ea1407bb49cc3", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "6096b9218ad78202c69367424c5ea1407bb49cc3", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "6096b9218ad78202c69367424c5ea1407bb49cc3", version = "0.13.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
|
||||
@@ -170,6 +170,15 @@ impl DiskFaultHarness {
|
||||
/// Start (or restart after [`Self::kill_server`]) the RustFS server with
|
||||
/// all volume directories and the same address.
|
||||
pub async fn start_server(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(false).await
|
||||
}
|
||||
|
||||
/// Start a process once its node read-readiness probe passes.
|
||||
pub async fn start_server_at_read_quorum(&mut self) -> ChaosResult<()> {
|
||||
self.start_server_with_readiness(true).await
|
||||
}
|
||||
|
||||
async fn start_server_with_readiness(&mut self, read_quorum_only: bool) -> ChaosResult<()> {
|
||||
// The common helper always appends `env.temp_dir` as the final storage
|
||||
// path: point it at the last disk for startup and pass the remaining
|
||||
// disks explicitly, then restore it to the root so Drop cleans up
|
||||
@@ -181,7 +190,13 @@ impl DiskFaultHarness {
|
||||
let arg_refs: Vec<&str> = disk_args.iter().map(String::as_str).collect();
|
||||
let env_refs: Vec<(&str, &str)> = self.extra_env.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
|
||||
|
||||
let result = self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await;
|
||||
let result = if read_quorum_only {
|
||||
self.env
|
||||
.start_rustfs_server_with_env_at_read_quorum(arg_refs, &env_refs)
|
||||
.await
|
||||
} else {
|
||||
self.env.start_rustfs_server_with_env(arg_refs, &env_refs).await
|
||||
};
|
||||
self.env.temp_dir = self.root.to_string_lossy().to_string();
|
||||
result
|
||||
}
|
||||
|
||||
@@ -722,7 +722,7 @@ impl RustFSTestEnvironment {
|
||||
cleanup_existing: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing)
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -732,6 +732,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
cleanup_existing: bool,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if cleanup_existing {
|
||||
self.cleanup_existing_processes().await?;
|
||||
@@ -756,7 +757,7 @@ impl RustFSTestEnvironment {
|
||||
self.process = Some(process);
|
||||
|
||||
// Wait for server to be ready
|
||||
self.wait_for_server_ready().await?;
|
||||
self.wait_for_server_ready_with_probe(read_quorum_only).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -770,7 +771,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -782,7 +783,7 @@ impl RustFSTestEnvironment {
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false, false)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -800,6 +801,18 @@ impl RustFSTestEnvironment {
|
||||
self.start_rustfs_server_inner(extra_args, extra_env, true).await
|
||||
}
|
||||
|
||||
/// Start a degraded read-serving process using the node readiness probe.
|
||||
/// ListBuckets retains its namespace write-quorum requirement.
|
||||
pub async fn start_rustfs_server_with_env_at_read_quorum(
|
||||
&mut self,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let binary_path = rustfs_binary_path();
|
||||
self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, true, true)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server without cleaning up other running RustFS processes.
|
||||
///
|
||||
/// This is useful for tests that need multiple independent RustFS instances
|
||||
@@ -828,8 +841,16 @@ impl RustFSTestEnvironment {
|
||||
/// Fails immediately if the spawned server process has already exited
|
||||
/// (e.g. a port bind failure) instead of polling out the full timeout.
|
||||
pub async fn wait_for_server_ready(&mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_server_ready_with_probe(false).await
|
||||
}
|
||||
|
||||
async fn wait_for_server_ready_with_probe(
|
||||
&mut self,
|
||||
read_quorum_only: bool,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
info!("Waiting for RustFS server to be ready on {}", self.address);
|
||||
let client = self.create_s3_client();
|
||||
let health_client = HttpClient::builder().no_proxy().timeout(Duration::from_secs(2)).build()?;
|
||||
|
||||
for i in 0..60 {
|
||||
if let Some(process) = self.process.as_mut()
|
||||
@@ -838,7 +859,16 @@ impl RustFSTestEnvironment {
|
||||
return Err(format!("RustFS server process exited before becoming ready: {status}").into());
|
||||
}
|
||||
|
||||
if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
|
||||
let ready = if read_quorum_only {
|
||||
health_client
|
||||
.get(format!("{}/minio/health/ready", self.url))
|
||||
.send()
|
||||
.await
|
||||
.is_ok_and(|response| response.status().is_success())
|
||||
} else {
|
||||
TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok()
|
||||
};
|
||||
if ready {
|
||||
info!("✅ RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -283,6 +283,7 @@ async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> Te
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
assert_single_node_cold_start_with_missing_volume().await?;
|
||||
|
||||
for (node_count, parity) in [(3, 1), (4, 2), (6, 3), (6, 2), (4, 0)] {
|
||||
let read_quorum = node_count - parity;
|
||||
@@ -430,6 +431,138 @@ async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResul
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_single_node_cold_start_with_missing_volume() -> TestResult {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let mut server = crate::chaos::DiskFaultHarness::new(drive_count).await?;
|
||||
server.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{}", drive_count / 2));
|
||||
server.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
server.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
server.set_env("RUSTFS_HEALTH_ENDPOINT_ENABLE", "true");
|
||||
server.set_env("RUST_LOG", "warn");
|
||||
server.start_server_at_read_quorum().await?;
|
||||
let client = server.env.create_s3_client();
|
||||
let client = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
);
|
||||
let bucket = format!("missing-volume-{drive_count}");
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
let small = vec![17; 4096];
|
||||
let large = vec![73; 8 * 1024 * 1024];
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for disk in 0..drive_count {
|
||||
let census = crate::chaos::census_object_version_on_disk(server.disk_path(disk), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must exist before restart: {census:?}");
|
||||
}
|
||||
}
|
||||
server.kill_server();
|
||||
let missing = drive_count - 1;
|
||||
server.take_disk_offline(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
let head = client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(i64::try_from(body.len())?));
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let keys: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, ["large", "small"]);
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("degraded-write")
|
||||
.body(Bytes::from_static(b"degraded write").into())
|
||||
.send()
|
||||
.await;
|
||||
if drive_count == 2 {
|
||||
let error = put.expect_err("one of two drives cannot commit a new object");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
let error = client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("large")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("one of two drives cannot commit an object deletion");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_quorum_object_body(&client, &bucket, "large", &large).await?;
|
||||
} else {
|
||||
put?;
|
||||
assert_quorum_object_body(&client, &bucket, "degraded-write", b"degraded write").await?;
|
||||
client.delete_object().bucket(&bucket).key("degraded-write").send().await?;
|
||||
}
|
||||
assert!(
|
||||
!server.disk_path(missing).exists(),
|
||||
"degraded startup and S3 operations must leave the root absent"
|
||||
);
|
||||
|
||||
if drive_count == 2 {
|
||||
server.kill_server();
|
||||
std::fs::create_dir(server.disk_path(missing))?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
client.head_object().bucket(&bucket).key(key).send().await?;
|
||||
}
|
||||
assert_eq!(client.list_objects_v2().bucket(&bucket).send().await?.contents().len(), 2);
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("empty-root-write")
|
||||
.body(Bytes::from_static(b"must remain unavailable").into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an empty replacement cannot restore write quorum");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert!(
|
||||
!server
|
||||
.disk_path(missing)
|
||||
.join(crate::storage_api::RUSTFS_META_BUCKET)
|
||||
.join("format.json")
|
||||
.exists(),
|
||||
"read admission must not format the empty replacement"
|
||||
);
|
||||
}
|
||||
server.kill_server();
|
||||
server.bring_disk_online(missing)?;
|
||||
server.start_server_at_read_quorum().await?;
|
||||
for (key, body) in [("small", &small), ("large", &large)] {
|
||||
assert_quorum_object_body(&client, &bucket, key, body).await?;
|
||||
}
|
||||
for key in ["degraded-write", "empty-root-write"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected or deleted degraded write must stay absent after recovery");
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("recovery-write")
|
||||
.body(Bytes::from_static(b"recovered write").into())
|
||||
.send()
|
||||
.await?;
|
||||
assert_quorum_object_body(&client, &bucket, "recovery-write", b"recovered write").await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_start_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -11423,6 +11423,20 @@ impl LocalDisk {
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
fn part_path_resolver(&self) -> impl Fn(&str, &str) -> Result<PathBuf> + Send + use<> {
|
||||
let root = self.io_root().to_owned();
|
||||
#[cfg(target_os = "linux")]
|
||||
let mount_lease = self.mount_lease.clone();
|
||||
move |bucket, key| {
|
||||
let (bucket_path, path) = build_local_disk_object_path(&root, bucket, key);
|
||||
#[cfg(target_os = "linux")]
|
||||
check_local_disk_valid_object_path_at(&root, &mount_lease, &bucket_path, &path)?;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
check_local_disk_valid_object_path(&root, &bucket_path, &path)?;
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_part_metadata(&self, bucket: &str, volume_dir: &Path, path_str: &str) -> Result<ObjectPartInfo> {
|
||||
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
|
||||
let path = Path::new(path_str);
|
||||
@@ -11435,8 +11449,10 @@ impl LocalDisk {
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or_default();
|
||||
let parent_str = path.parent().unwrap_or_else(|| Path::new("")).to_string_lossy();
|
||||
let data_path = self.io_get_object_path(bucket, &path_join_buf(&[parent_str.as_ref(), &format!("part.{num}")]))?;
|
||||
let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref());
|
||||
let data_key = path_join_buf(&[parent_str.as_ref(), &format!("part.{num}")]);
|
||||
let metadata_key = path_str.to_owned();
|
||||
let volume = bucket.to_owned();
|
||||
let resolve = self.part_path_resolver();
|
||||
// A part's existence check, metadata read and decode share one dispatch.
|
||||
// Keep open errors unmapped for the existing missing-volume fallback.
|
||||
let root = self.publication_root.clone();
|
||||
@@ -11446,12 +11462,13 @@ impl LocalDisk {
|
||||
error: Some(error),
|
||||
..Default::default()
|
||||
};
|
||||
let data_path = resolve(&volume, &data_key)?;
|
||||
if let Err(err) = std::fs::metadata(data_path) {
|
||||
return Ok(Ok(part_error(err.to_string())));
|
||||
}
|
||||
// Invalid metadata paths remain request errors, but missing data wins
|
||||
// first, as it does in the serial reader.
|
||||
let metadata_path = metadata_path?;
|
||||
let metadata_path = resolve(&volume, &metadata_key)?;
|
||||
Ok(read_all_data_std(root.open_readonly(&metadata_path))
|
||||
.map(|(data, _)| ObjectPartInfo::unmarshal(&data).unwrap_or_else(|err| part_error(err.to_string()))))
|
||||
})
|
||||
@@ -12944,7 +12961,10 @@ impl LocalDisk {
|
||||
let name = entry.trim_end_matches(SLASH_SEPARATOR);
|
||||
let name = decode_dir_object(format!("{}/{}", current, name).as_str());
|
||||
|
||||
if opts.limit <= 0 || metadata_counts_toward_limit(&metadata) {
|
||||
if opts.limit <= 0
|
||||
|| (opts.forward_to.as_ref().is_none_or(|marker| &name >= marker)
|
||||
&& metadata_counts_toward_limit(&metadata))
|
||||
{
|
||||
*objs_returned += 1;
|
||||
}
|
||||
|
||||
@@ -13214,7 +13234,9 @@ impl LocalDisk {
|
||||
|
||||
meta.metadata = read.bytes;
|
||||
|
||||
if opts.limit <= 0 || read.counts_toward_limit {
|
||||
if opts.limit <= 0
|
||||
|| (opts.forward_to.as_ref().is_none_or(|marker| &meta.name >= marker) && read.counts_toward_limit)
|
||||
{
|
||||
*objs_returned += 1;
|
||||
}
|
||||
|
||||
@@ -15487,42 +15509,36 @@ impl DiskAPI for LocalDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
|
||||
if bucket == RUSTFS_META_MULTIPART_BUCKET && !paths.is_empty() {
|
||||
let mut cached_parts = Vec::with_capacity(paths.len());
|
||||
let mut all_cached = true;
|
||||
let mut verified_source: Option<(&str, PathBuf)> = None;
|
||||
for path_str in paths {
|
||||
let Some(num) = Path::new(path_str)
|
||||
.file_name()
|
||||
.and_then(|v| v.to_str())
|
||||
.and_then(|v| v.strip_prefix("part."))
|
||||
.and_then(|v| v.strip_suffix(".meta"))
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
let scope_root = multipart_staging::upload_scope_root(path_str, true);
|
||||
let (source, head_already_verified) = match verified_source.as_ref() {
|
||||
Some((prev_root, prev_source)) if *prev_root == scope_root => (prev_source, true),
|
||||
_ => {
|
||||
let Ok(source) = self.io_get_object_open_path(bucket, scope_root) else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
verified_source = Some((scope_root, source));
|
||||
(&verified_source.as_ref().expect("just initialized").1, false)
|
||||
let permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
|
||||
let resolve = self.part_path_resolver();
|
||||
let keys = paths.to_owned();
|
||||
let cached = spawn_blocking_with_disk_io_permit(permit, move || {
|
||||
let mut parts = Vec::with_capacity(keys.len());
|
||||
let mut verified = None;
|
||||
for key in &keys {
|
||||
let num = Path::new(key)
|
||||
.file_name()?
|
||||
.to_str()?
|
||||
.strip_prefix("part.")?
|
||||
.strip_suffix(".meta")?
|
||||
.parse()
|
||||
.ok()?;
|
||||
let scope = multipart_staging::upload_scope_root(key, true);
|
||||
let same_head = verified.as_ref().is_some_and(|(prev, _)| *prev == scope);
|
||||
if !same_head {
|
||||
verified = Some((scope, resolve(RUSTFS_META_MULTIPART_BUCKET, scope).ok()?));
|
||||
}
|
||||
};
|
||||
let Some(part) =
|
||||
multipart_source_state::get_active_multipart_source_part_with_head(source, num, head_already_verified)
|
||||
else {
|
||||
all_cached = false;
|
||||
break;
|
||||
};
|
||||
cached_parts.push(part);
|
||||
}
|
||||
if all_cached {
|
||||
return Ok(cached_parts);
|
||||
let (_, source) = verified.as_ref()?;
|
||||
parts.push(multipart_source_state::get_active_multipart_source_part_with_head(
|
||||
source, num, same_head,
|
||||
)?);
|
||||
}
|
||||
Some(parts)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
if let Some(parts) = cached {
|
||||
return Ok(parts);
|
||||
}
|
||||
}
|
||||
let volume_dir = self.io_get_bucket_path(bucket)?;
|
||||
@@ -17957,10 +17973,13 @@ fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::c
|
||||
}
|
||||
|
||||
let part = &fi.parts[0];
|
||||
if !usize::try_from(part.actual_size).is_ok_and(|size| size <= fi.erasure.block_size) {
|
||||
if part.size > fi.erasure.block_size || !usize::try_from(part.actual_size).is_ok_and(|size| size <= fi.erasure.block_size) {
|
||||
return false;
|
||||
}
|
||||
let shard_size = fi.shard_file_size(part.actual_size);
|
||||
let Ok(stored_size) = i64::try_from(part.size) else {
|
||||
return false;
|
||||
};
|
||||
let shard_size = fi.shard_file_size(stored_size.max(part.actual_size));
|
||||
let versioned = fi.versioned || fi.version_id.is_some_and(|version_id| !version_id.is_nil());
|
||||
storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned)
|
||||
}
|
||||
@@ -18727,6 +18746,8 @@ mod test {
|
||||
"a multi-block external object can still fit the per-shard budget"
|
||||
);
|
||||
assert!(!should_read_legacy_inline_part(&ec12, &config));
|
||||
ec12.parts[0].actual_size = 1;
|
||||
assert!(!should_read_legacy_inline_part(&ec12, &config));
|
||||
}
|
||||
|
||||
fn guarded_delete_group(fi: FileInfo, expected_current: CurrentDeleteGeneration) -> GuardedFileInfoVersions {
|
||||
@@ -31781,13 +31802,14 @@ mod test {
|
||||
Endpoint::try_from(dir.path().to_str().expect("operation should succeed")).expect("operation should succeed");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("operation should succeed");
|
||||
|
||||
async fn scan_names(disk: &LocalDisk, bucket: &str, base_dir: &str, forward_to: &str) -> (Vec<String>, i32) {
|
||||
async fn scan_names(disk: &LocalDisk, bucket: &str, base_dir: &str, forward_to: &str, limit: i32) -> (Vec<String>, i32) {
|
||||
let (reader, mut writer) = tokio::io::duplex(4096);
|
||||
let mut out = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: base_dir.to_string(),
|
||||
recursive: true,
|
||||
limit,
|
||||
forward_to: Some(forward_to.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -31812,7 +31834,7 @@ mod test {
|
||||
}
|
||||
|
||||
let (engineering_names, engineering_count) =
|
||||
scan_names(&disk, bucket, "engineering/", "engineering/engineering/engineering/repo-0001").await;
|
||||
scan_names(&disk, bucket, "engineering/", "engineering/engineering/engineering/repo-0001", 0).await;
|
||||
|
||||
assert_eq!(
|
||||
engineering_names,
|
||||
@@ -31828,7 +31850,7 @@ mod test {
|
||||
assert_eq!(engineering_count as usize, engineering_names.len());
|
||||
|
||||
let (different_names, different_count) =
|
||||
scan_names(&disk, bucket, "different/", "different/prefix/prefix/repo-0001").await;
|
||||
scan_names(&disk, bucket, "different/", "different/prefix/prefix/repo-0001", 0).await;
|
||||
|
||||
assert_eq!(
|
||||
different_names,
|
||||
@@ -31840,7 +31862,8 @@ mod test {
|
||||
);
|
||||
assert_eq!(different_count as usize, different_names.len());
|
||||
|
||||
let (double_names, double_count) = scan_names(&disk, bucket, "engineering/", "engineering/engineering/repo-0001").await;
|
||||
let (double_names, double_count) =
|
||||
scan_names(&disk, bucket, "engineering/", "engineering/engineering/repo-0001", 0).await;
|
||||
|
||||
assert_eq!(
|
||||
double_names,
|
||||
@@ -31852,6 +31875,13 @@ mod test {
|
||||
"forward_to must not skip a child directory whose name repeats the base prefix"
|
||||
);
|
||||
assert_eq!(double_count as usize, double_names.len());
|
||||
|
||||
fs::write(bucket_dir.join("engineering/engineering").join(STORAGE_FORMAT_FILE), b"meta")
|
||||
.await
|
||||
.expect("write ancestor metadata");
|
||||
let (names, count) = scan_names(&disk, bucket, "engineering/", "engineering/engineering/repo-0001", 2).await;
|
||||
assert_eq!(count, 2);
|
||||
assert!(names.iter().any(|name| name == "engineering/engineering/repo-0002"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -604,10 +604,10 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
|
||||
let disk = match new_disk(ep, opt).await {
|
||||
Ok(disk) => disk,
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied) => {
|
||||
// LocalDisk already logged the filesystem cause and repair
|
||||
// hint. Keep this slot unavailable so ECStore initialization
|
||||
// can apply its existing fatal-error and quorum rules.
|
||||
Err(crate::disk::error::DiskError::FileAccessDenied | crate::disk::error::DiskError::VolumeNotFound) => {
|
||||
// Preserve the configured slot when a local volume is unavailable.
|
||||
// LocalDisk logs the cause; format loading retains authority
|
||||
// over fatal errors and quorum. Never create a missing root.
|
||||
global_local_disk_map.insert(ep.to_string(), None);
|
||||
continue;
|
||||
}
|
||||
@@ -888,6 +888,95 @@ mod tests {
|
||||
disk_map.keys().collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn missing_local_volume_preserves_offline_slot_without_creating_root() {
|
||||
use crate::disk::{DiskOption, error::DiskError};
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
for drive_count in [2, 3, 4] {
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..drive_count {
|
||||
let path = directory.path().join(format!("disk{index}"));
|
||||
if index != 0 {
|
||||
std::fs::create_dir(&path).expect("create surviving volume");
|
||||
}
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let missing = endpoints[0].to_string();
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: drive_count,
|
||||
endpoints: endpoints.into(),
|
||||
cmd_line: "missing-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools.clone(), &options)
|
||||
.await
|
||||
.expect("missing volume should remain offline for quorum evaluation");
|
||||
let map = ctx.local_disk_map();
|
||||
let map = map.read().await;
|
||||
assert_eq!(map.len(), drive_count);
|
||||
assert!(matches!(map.get(&missing), Some(None)));
|
||||
assert_eq!(map.values().flatten().count(), drive_count - 1);
|
||||
drop(map);
|
||||
let drives = ctx.local_disk_set_drives();
|
||||
let drives = drives.read().await;
|
||||
assert_eq!(drives[0][0].len(), drive_count);
|
||||
assert!(drives[0][0][0].is_none());
|
||||
assert!(drives[0][0][1..].iter().all(Option::is_some));
|
||||
drop(drives);
|
||||
|
||||
let (disks, errors) = crate::store::init_format::init_disks(&ctx, &pools.0[0].endpoints, &options).await;
|
||||
assert!(disks[0].is_none());
|
||||
assert_eq!(errors[0], Some(DiskError::VolumeNotFound));
|
||||
assert!(disks[1..].iter().all(Option::is_some));
|
||||
assert!(errors[1..].iter().all(Option::is_none));
|
||||
assert!(!directory.path().join("disk0").exists(), "startup must not create a missing mount root");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_local_volume_still_fails_registry_initialization() {
|
||||
use crate::disk::DiskOption;
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
|
||||
let directory = tempfile::tempdir().expect("create volume parent");
|
||||
let path = directory.path().join("regular-file");
|
||||
std::fs::write(&path, b"preserve existing file").expect("create invalid volume");
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: vec![endpoint].into(),
|
||||
cmd_line: "invalid-volume".into(),
|
||||
platform: "test".into(),
|
||||
}]);
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let options = DiskOption {
|
||||
cleanup: true,
|
||||
health_check: false,
|
||||
};
|
||||
super::initialize_local_disk_maps(&ctx, pools, &options)
|
||||
.await
|
||||
.expect_err("a non-directory volume must retain its initialization error");
|
||||
assert_eq!(std::fs::read(&path).expect("read existing file"), b"preserve existing file");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_invalid_registry_pool_index_has_no_authority_or_registry_effect() {
|
||||
use crate::disk::DiskOption;
|
||||
|
||||
@@ -752,18 +752,60 @@ impl ECStore {
|
||||
Ok(ec)
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
async fn init_pool_metadata_for_startup(self: &Arc<Self>) -> Result<(PoolMeta, PoolMetaReplicaState)> {
|
||||
let should_persist_pool_meta = self
|
||||
.pools
|
||||
.first()
|
||||
.is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints));
|
||||
// Lock order is save gate -> distributed metadata writer, matching
|
||||
// Lock order is save gate -> distributed metadata fence, matching
|
||||
// runtime mutations. RPC locks are available before ECStore publication
|
||||
// and do not depend on the later fleet proof.
|
||||
let mut startup_write_state = self.pool_meta_save_gate.write().await;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let startup_attempt = uuid::Uuid::new_v4();
|
||||
// A committed snapshot needs only a shared namespace fence. A writer
|
||||
// transition releases this reader and re-reads every observation below
|
||||
// under the existing exclusive fence before applying side effects.
|
||||
if !startup_write_state.bootstrap_identity_proven() && startup_write_state.attested_pool_indices().is_empty() {
|
||||
let pool = self
|
||||
.pools
|
||||
.first()
|
||||
.ok_or_else(|| Error::other("startup has no storage pools"))?;
|
||||
let lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, "pool.bin").await?;
|
||||
let reader = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let load = load_pool_meta_for_startup(self.pools.clone(), &mut startup_write_state);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load);
|
||||
match load.await {
|
||||
Ok((meta, replicas)) => {
|
||||
let topology_update = meta.validate(self.pools.clone())?;
|
||||
let requires_writer = should_persist_pool_meta
|
||||
&& (topology_update
|
||||
|| (replicas.needs_repair && replicas.repair_write_safe)
|
||||
|| startup_write_state.identity_requires_repair());
|
||||
if !topology_update && !requires_writer {
|
||||
startup_write_state.ensure_startup_identity_initialized()?;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
crate::core::pools::startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "startup-classifier", "attempt": startup_attempt,
|
||||
"elected_writer": should_persist_pool_meta,
|
||||
"needs_repair": replicas.needs_repair,
|
||||
"repair_write_safe": replicas.repair_write_safe,
|
||||
"topology_update": topology_update,
|
||||
}));
|
||||
self.install_startup_pool_metadata(&meta, Some(&reader)).await?;
|
||||
return Ok((meta, replicas));
|
||||
}
|
||||
}
|
||||
Err(err)
|
||||
if should_persist_pool_meta
|
||||
&& err
|
||||
.pool_metadata_failure()
|
||||
.is_some_and(|failure| failure.kind == crate::error::PoolMetadataFailure::ReadUnavailable) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
drop(reader);
|
||||
}
|
||||
let startup_pool_meta_guard = if should_persist_pool_meta {
|
||||
let pool = self
|
||||
.pools
|
||||
@@ -824,7 +866,6 @@ impl ECStore {
|
||||
"repair_write_safe": pool_meta_replica_state.repair_write_safe,
|
||||
"topology_update": update,
|
||||
}));
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
let mut installed_pool_meta = if update {
|
||||
PoolMeta::new(&self.pools, &meta)?
|
||||
@@ -881,19 +922,34 @@ impl ECStore {
|
||||
installed_pool_meta = persist.await?;
|
||||
}
|
||||
|
||||
if startup_pool_meta_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost() || guard.is_released())
|
||||
{
|
||||
return Err(Error::other("startup pool metadata writer fence was lost before publication"));
|
||||
}
|
||||
{
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
*pool_meta = installed_pool_meta.clone();
|
||||
}
|
||||
self.install_startup_pool_metadata(&installed_pool_meta, startup_pool_meta_guard.as_deref())
|
||||
.await?;
|
||||
drop(startup_pool_meta_guard);
|
||||
drop(startup_write_state);
|
||||
|
||||
Ok((installed_pool_meta, pool_meta_replica_state))
|
||||
}
|
||||
|
||||
async fn install_startup_pool_metadata(
|
||||
&self,
|
||||
metadata: &PoolMeta,
|
||||
fence: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
if fence.is_some_and(|guard| guard.is_lock_lost() || guard.is_released()) {
|
||||
return Err(Error::other("startup pool metadata fence was lost before publication"));
|
||||
}
|
||||
*pool_meta = metadata.clone();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, rx))]
|
||||
pub async fn init(self: &Arc<Self>, rx: CancellationToken) -> Result<()> {
|
||||
runtime_sources::ensure_boot_time().await;
|
||||
|
||||
let (installed_pool_meta, pool_meta_replica_state) = self.init_pool_metadata_for_startup().await?;
|
||||
let endpoints = runtime_sources::endpoint_pools_or_default();
|
||||
|
||||
resolve_store_init_stage_result(self.load_rebalance_meta().await, "load_rebalance_meta")?;
|
||||
let rebalance_resume_required = {
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
@@ -3731,6 +3787,89 @@ mod tests {
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn committed_startup_pool_metadata_uses_shared_fence_without_disk_writes() {
|
||||
use crate::disk::{FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
|
||||
for reopen in [false, true] {
|
||||
let directory = tempfile::tempdir().expect("create committed metadata deployment");
|
||||
let (_, seeded, seed_shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
let (store, shutdown) = if reopen {
|
||||
seed_shutdown.cancel();
|
||||
drop(seeded);
|
||||
let (_, reopened, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "startup-read-fence", &[4])).await;
|
||||
(reopened, shutdown)
|
||||
} else {
|
||||
(seeded, seed_shutdown)
|
||||
};
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let snapshot = || {
|
||||
let mut files = std::collections::BTreeMap::new();
|
||||
for (index, slot) in disks.iter().enumerate() {
|
||||
let disk = slot.as_ref().expect("committed drive handle");
|
||||
for object in [POOL_META_NAME, POOL_META_IDENTITY_NAME, FORMAT_CONFIG_FILE] {
|
||||
let root = disk.path().join(RUSTFS_META_BUCKET).join(object);
|
||||
assert!(root.exists(), "committed authority must be present: {root:?}");
|
||||
let mut pending = vec![root];
|
||||
while let Some(path) = pending.pop() {
|
||||
let metadata = std::fs::symlink_metadata(&path).expect("inspect authority file type");
|
||||
assert!(!metadata.file_type().is_symlink(), "authority fixture must not follow symlinks");
|
||||
if metadata.is_dir() {
|
||||
for entry in std::fs::read_dir(&path).expect("inspect authority object directory") {
|
||||
pending.push(entry.expect("authority directory entry").path());
|
||||
}
|
||||
} else {
|
||||
assert!(metadata.is_file(), "authority fixture must contain regular files");
|
||||
let contents = std::fs::read(&path).expect("read physical authority bytes");
|
||||
let relative = path
|
||||
.strip_prefix(disk.path())
|
||||
.expect("authority stays under fixture drive")
|
||||
.to_path_buf();
|
||||
files.insert((index, relative), contents);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
files
|
||||
};
|
||||
let before = snapshot();
|
||||
let lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("metadata namespace lock");
|
||||
let mut reader = lock
|
||||
.get_read_lock(Duration::from_secs(1))
|
||||
.await
|
||||
.expect("hold a competing metadata reader");
|
||||
let (metadata, replicas) = tokio::time::timeout(Duration::from_secs(2), store.init_pool_metadata_for_startup())
|
||||
.await
|
||||
.expect("committed startup must not wait for an exclusive lock")
|
||||
.expect("committed startup metadata is readable under a shared fence");
|
||||
assert_eq!(metadata.pools[0].cmd_line, "startup-read-fence-pool-0");
|
||||
assert!(!replicas.needs_repair);
|
||||
assert!(!reader.is_lock_lost());
|
||||
let after = snapshot();
|
||||
assert_eq!(
|
||||
after, before,
|
||||
"shared startup observation cannot mutate metadata, identity, or format authority"
|
||||
);
|
||||
assert!(reader.release(), "release the snapshot's shared fence");
|
||||
let mut invalid = metadata.clone();
|
||||
invalid.pools[0].cmd_line = "must-not-be-published".to_string();
|
||||
store
|
||||
.install_startup_pool_metadata(&invalid, Some(&reader))
|
||||
.await
|
||||
.expect_err("a released fence cannot publish startup metadata");
|
||||
assert_eq!(store.pool_meta.read().await.pools[0].cmd_line, metadata.pools[0].cmd_line);
|
||||
drop(reader);
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn interrupted_format_bootstrap_recovers_two_pools_through_store_init() {
|
||||
|
||||
@@ -169,36 +169,34 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
{
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum)))
|
||||
&& let Some((format, members)) = select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
{
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
let resumable_partial_migration = formats_present > 0
|
||||
let native_read_candidate = if matches!(format_quorum, Some(Err(Error::ErasureReadQuorum))) {
|
||||
select_existing_format_read_quorum(&formats, &errs, set_drive_count)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let partial_native_with_blanks = formats_present > 0
|
||||
&& errs.iter().any(|error| matches!(error, Some(DiskError::UnformattedDisk)))
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err)
|
||||
&& format_quorum.as_ref().is_some_and(Result::is_err);
|
||||
let resumable_partial_migration = partial_native_with_blanks
|
||||
&& formats.iter().zip(&errs).all(|(format, error)| {
|
||||
format.is_some() || matches!(error, Some(DiskError::UnformattedDisk | DiskError::DiskNotFound))
|
||||
});
|
||||
|
||||
if first_disk && (all_unformatted || resumable_partial_migration) {
|
||||
let allow_migration = first_disk && (all_unformatted || resumable_partial_migration);
|
||||
if partial_native_with_blanks && !allow_migration && native_read_candidate.is_none() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
if allow_migration || (partial_native_with_blanks && native_read_candidate.is_some()) {
|
||||
info!(all_unformatted, resumable_partial_migration, "checking for a legacy storage format");
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count).await {
|
||||
let mode = match native_read_candidate.as_ref() {
|
||||
Some((reference, _)) => LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors: &errs,
|
||||
allow_migration,
|
||||
},
|
||||
None => LegacyFormatMode::Migration,
|
||||
};
|
||||
match try_migrate_format(disks, &formats, set_count, set_drive_count, mode).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { format, quorum_members }) => {
|
||||
info!("Migrated format from MinIO config");
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &quorum_members, None, set_drive_count).await?;
|
||||
@@ -217,10 +215,10 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
);
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
Ok(LegacyFormatOutcome::None) => {}
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
if fresh_bootstrap_proven {
|
||||
if first_disk && fresh_bootstrap_proven {
|
||||
let fm = init_format_erasure(instance_ctx, disks, set_count, set_drive_count, deployment_id).await?;
|
||||
return Ok(LoadedFormat {
|
||||
format: fm,
|
||||
@@ -229,6 +227,26 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve legacy migration before admitting native reads. Blank replacement
|
||||
// slots contribute no votes and receive no formatting/bootstrap authority.
|
||||
if let Some((format, members)) = native_read_candidate {
|
||||
retain_format_quorum_members(
|
||||
instance_ctx,
|
||||
disks,
|
||||
&format,
|
||||
&members,
|
||||
Some(&unformatted_bootstrap_routes),
|
||||
set_drive_count,
|
||||
)
|
||||
.await?;
|
||||
// Loading an existing format grants no authority to initialize pool
|
||||
// metadata or migrate formats. The persisted storage classes and each
|
||||
// object's EC metadata still determine service and operation quorums.
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::None,
|
||||
});
|
||||
}
|
||||
info!(
|
||||
"first_disk: {}, should_init_erasure_disks: {}",
|
||||
first_disk,
|
||||
@@ -271,9 +289,11 @@ async fn verified_legacy_adoption_source(
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
) -> Result<PoolMetaBootstrapAuthority> {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count).await {
|
||||
match try_migrate_format(disks, rustfs_formats, set_count, set_drive_count, LegacyFormatMode::Migration).await {
|
||||
Ok(LegacyFormatOutcome::Migrated { .. }) => Ok(PoolMetaBootstrapAuthority::LegacyAdoption),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::Incompatible) => Ok(PoolMetaBootstrapAuthority::None),
|
||||
Ok(LegacyFormatOutcome::None | LegacyFormatOutcome::ReadCompatible | LegacyFormatOutcome::Incompatible) => {
|
||||
Ok(PoolMetaBootstrapAuthority::None)
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
error = %err,
|
||||
@@ -443,25 +463,49 @@ enum LegacyFormatOutcome {
|
||||
/// version mismatch, or a parse failure). The caller must decide how to
|
||||
/// proceed; creating a fresh format would leave the legacy objects unreadable.
|
||||
Incompatible,
|
||||
/// Observed legacy formats agree with the native read quorum, without granting writer authority.
|
||||
ReadCompatible,
|
||||
/// No MinIO `format.json` was present on any disk (a normal fresh install).
|
||||
None,
|
||||
}
|
||||
|
||||
/// Native reads validate observed legacy identities without electing a migration writer.
|
||||
enum LegacyFormatMode<'a> {
|
||||
Migration,
|
||||
NativeRead {
|
||||
reference: &'a FormatV3,
|
||||
native_errors: &'a [Option<DiskError>],
|
||||
allow_migration: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Tries to migrate an on-disk MinIO `format.json` into RustFS format files.
|
||||
///
|
||||
/// Returns [`LegacyFormatOutcome`] describing whether a legacy format was present
|
||||
/// and, if so, whether it was compatible. `Err` is returned for read or persist
|
||||
/// failures that prevent a conclusive migration decision.
|
||||
/// failures that prevent a conclusive migration decision. Native read admission
|
||||
/// checks every observed legacy format against its existing layout and tolerates
|
||||
/// unavailable nonvoters. Migration requires complete observations and a strict majority.
|
||||
async fn try_migrate_format(
|
||||
disks: &[Option<DiskStore>],
|
||||
rustfs_formats: &[Option<FormatV3>],
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
mode: LegacyFormatMode<'_>,
|
||||
) -> Result<LegacyFormatOutcome> {
|
||||
let (native_reference, native_errors, allow_migration) = match mode {
|
||||
LegacyFormatMode::Migration => (None, None, true),
|
||||
LegacyFormatMode::NativeRead {
|
||||
reference,
|
||||
native_errors,
|
||||
allow_migration,
|
||||
} => (Some(reference), Some(native_errors), allow_migration),
|
||||
};
|
||||
let legacy_format_max_bytes = legacy_format_max_bytes(disks.len())?;
|
||||
let mut legacy_seen = false;
|
||||
let mut invalid_legacy_format = false;
|
||||
let mut read_error: Option<Error> = None;
|
||||
let mut unavailable_error = None;
|
||||
let mut legacy_formats = vec![None; disks.len()];
|
||||
|
||||
let reads = stream::iter(
|
||||
@@ -473,6 +517,13 @@ async fn try_migrate_format(
|
||||
let Some(disk) = disk else {
|
||||
return (index, None);
|
||||
};
|
||||
// Native format loading already proved this peer unavailable.
|
||||
// Reuse that observation and keep it ineligible for migration.
|
||||
if let Some(error) = native_errors.and_then(|errors| errors.get(index)).and_then(Option::as_ref)
|
||||
&& is_network_like_disk_error(error)
|
||||
{
|
||||
return (index, Some(Err(error.clone())));
|
||||
}
|
||||
let reader = match disk.read_file(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE).await {
|
||||
Ok(reader) => reader,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return (index, None),
|
||||
@@ -512,6 +563,9 @@ async fn try_migrate_format(
|
||||
match read {
|
||||
Ok(Some(format)) => legacy_formats[index] = Some(format),
|
||||
Ok(None) => invalid_legacy_format = true,
|
||||
Err(err) if native_reference.is_some() && is_network_like_disk_error(&err) => {
|
||||
unavailable_error.get_or_insert(err);
|
||||
}
|
||||
Err(err) => {
|
||||
read_error.get_or_insert_with(|| err.into());
|
||||
}
|
||||
@@ -528,6 +582,15 @@ async fn try_migrate_format(
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
|
||||
if let Some(reference) = native_reference {
|
||||
if !formats_match_reference_slots(&legacy_formats, reference, 0) {
|
||||
return Ok(LegacyFormatOutcome::Incompatible);
|
||||
}
|
||||
if !allow_migration || unavailable_error.is_some() || legacy_formats.iter().flatten().count() <= disks.len() / 2 {
|
||||
return Ok(LegacyFormatOutcome::ReadCompatible);
|
||||
}
|
||||
}
|
||||
|
||||
let format = match get_format_erasure_in_quorum(&legacy_formats, 0) {
|
||||
Ok(format) => format,
|
||||
Err(_) => return Ok(LegacyFormatOutcome::Incompatible),
|
||||
@@ -677,7 +740,7 @@ fn select_existing_format_read_quorum(
|
||||
|| check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count).is_err()
|
||||
|| formats.iter().zip(errors).any(|(format, error)| match (format, error) {
|
||||
(Some(_), None) => false,
|
||||
(None, Some(DiskError::DiskNotFound)) => false,
|
||||
(None, Some(DiskError::DiskNotFound | DiskError::UnformattedDisk)) => false,
|
||||
(None, Some(error)) => !is_network_like_disk_error(error),
|
||||
_ => true,
|
||||
})
|
||||
@@ -1445,13 +1508,23 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_invalid_slots_and_blank_disks() {
|
||||
fn existing_format_read_quorum_excludes_blank_and_network_unavailable_slots() {
|
||||
let (formats, mut errors) = half_pool_formats();
|
||||
errors[2] = Some(DiskError::UnformattedDisk);
|
||||
errors[3] = Some(DiskError::Timeout);
|
||||
let (_, members) = select_existing_format_read_quorum(&formats, &errors, 4)
|
||||
.expect("blank and unreachable slots cannot block an existing native read quorum");
|
||||
assert!(!members[2] && !members[3]);
|
||||
assert_eq!(members.iter().filter(|member| **member).count(), 12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_format_read_quorum_rejects_conflicts_and_invalid_slots() {
|
||||
for failure in [
|
||||
"conflict",
|
||||
"wrong_slot",
|
||||
"nil",
|
||||
"duplicate",
|
||||
"blank",
|
||||
"corrupt",
|
||||
"access_denied",
|
||||
"tie",
|
||||
@@ -1473,7 +1546,6 @@ mod tests {
|
||||
format.erasure.sets[5][3] = format.erasure.sets[0][0];
|
||||
}
|
||||
}
|
||||
"blank" => errors[2] = Some(DiskError::UnformattedDisk),
|
||||
"corrupt" => errors[2] = Some(DiskError::CorruptedFormat),
|
||||
"access_denied" => errors[2] = Some(DiskError::FileAccessDenied),
|
||||
"tie" => {
|
||||
@@ -1532,6 +1604,186 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_read_quorum_quarantines_blank_replacement_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for surviving_slot in 0..2 {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let blank = 1 - surviving_slot;
|
||||
let mut format = FormatV3::new(1, 2);
|
||||
format.erasure.this = format.erasure.sets[0][surviving_slot];
|
||||
save_format_file(&disks[surviving_slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist surviving format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
2,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("existing native format must load at read quorum with a blank replacement");
|
||||
assert_eq!(loaded.format.shared_identity(), format.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[surviving_slot].is_some());
|
||||
assert!(disks[blank].is_none(), "blank replacement cannot vote or serve data");
|
||||
assert_eq!(format_bytes(&original).await, before, "read admission cannot write either format");
|
||||
assert!(matches!(
|
||||
load_format_erasure(original[blank].as_ref().expect("blank handle"), false).await,
|
||||
Err(DiskError::UnformattedDisk)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_read_quorum_with_blank_and_unavailable_slots_validates_legacy_without_writes() {
|
||||
for first_disk in [false, true] {
|
||||
for scenario in [
|
||||
"native-remote",
|
||||
"legacy-missing",
|
||||
"foreign-remote",
|
||||
"corrupt-remote",
|
||||
"wrong-slot-missing",
|
||||
] {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(4, 4).await;
|
||||
let reference = FormatV3::new(1, 4);
|
||||
for slot in 0..2 {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = format.erasure.sets[0][slot];
|
||||
save_format_file(&disks[slot], &Some(format.clone()))
|
||||
.await
|
||||
.expect("persist native read quorum");
|
||||
if scenario != "native-remote" {
|
||||
write_legacy_format(&disks[slot], &format).await;
|
||||
}
|
||||
}
|
||||
match scenario {
|
||||
"foreign-remote" => {
|
||||
let mut foreign = FormatV3::new(1, 4);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][2];
|
||||
write_legacy_format(&disks[2], &foreign).await;
|
||||
}
|
||||
"corrupt-remote" => write_legacy_bytes(&disks[2], bytes::Bytes::from_static(b"corrupt legacy")).await,
|
||||
"wrong-slot-missing" => {
|
||||
let mut wrong_slot = reference.clone();
|
||||
wrong_slot.erasure.this = reference.erasure.sets[0][0];
|
||||
write_legacy_format(&disks[2], &wrong_slot).await;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&original).await;
|
||||
if scenario.ends_with("remote") {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve unavailable peer address");
|
||||
let address = listener.local_addr().expect("peer address");
|
||||
drop(listener);
|
||||
let mut endpoint = Endpoint::try_from(format!("http://{address}/disk").as_str()).expect("remote endpoint");
|
||||
endpoint.is_local = false;
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(3);
|
||||
let remote = crate::cluster::rpc::RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(crate::cluster::rpc::TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("construct unavailable remote handle");
|
||||
let remote = Arc::new(crate::disk::Disk::Remote(Box::new(remote)));
|
||||
let error = remote
|
||||
.read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect_err("remote peer must be unavailable");
|
||||
assert!(is_network_like_disk_error(&error), "expected unavailable peer, got {error:?}");
|
||||
disks[3] = Some(remote);
|
||||
} else {
|
||||
disks[3] = None;
|
||||
}
|
||||
let result = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
first_disk,
|
||||
&mut disks,
|
||||
1,
|
||||
4,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
if matches!(scenario, "native-remote" | "legacy-missing") {
|
||||
let loaded = result.unwrap_or_else(|error| panic!("{scenario} must admit native reads: {error:?}"));
|
||||
assert_eq!(loaded.format.shared_identity(), reference.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[..2].iter().all(Option::is_some));
|
||||
assert!(disks[2..].iter().all(Option::is_none), "unavailable and blank slots cannot vote");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"{scenario} observed legacy evidence must reject reads"
|
||||
);
|
||||
}
|
||||
assert_eq!(format_bytes(&original).await, before, "{scenario} cannot write native formats");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_native_format_with_legacy_quorum_reads_without_migration_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(2, 2).await;
|
||||
let legacy = FormatV3::new(1, 2);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut native = legacy.clone();
|
||||
native.erasure.this = native.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(native))
|
||||
.await
|
||||
.expect("persist partial native format");
|
||||
let original = disks.clone();
|
||||
let before = format_bytes(&disks).await;
|
||||
let loaded =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 2, None)
|
||||
.await
|
||||
.expect("follower can read the existing native layout without migrating legacy formats");
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert!(disks[0].is_some() && disks[1].is_none(), "legacy-only slots cannot vote for native reads");
|
||||
assert_eq!(format_bytes(&original).await, before, "follower legacy inspection cannot persist formats");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn follower_below_native_read_quorum_skips_legacy_reads_without_writes() {
|
||||
let (_directory, mut disks) = local_disks_with_set_width(3, 3).await;
|
||||
let mut format = FormatV3::new(1, 3);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(format))
|
||||
.await
|
||||
.expect("persist one native format below the two-drive read quorum");
|
||||
for disk in &disks {
|
||||
write_legacy_bytes(disk, bytes::Bytes::from_static(b"invalid legacy format")).await;
|
||||
}
|
||||
let before = format_bytes(&disks).await;
|
||||
|
||||
let result = connect_load_init_formats(false, &mut disks, 1, 3, None).await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::ErasureReadQuorum)),
|
||||
"a non-reader must reject quorum before probing corrupt legacy files"
|
||||
);
|
||||
assert_eq!(format_bytes(&disks).await, before);
|
||||
for disk in disks.iter().flatten() {
|
||||
assert_eq!(
|
||||
disk.read_all(MIGRATING_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect("legacy bytes must remain untouched"),
|
||||
bytes::Bytes::from_static(b"invalid legacy format")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_load_succeeds_with_a_strict_majority() {
|
||||
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
|
||||
@@ -2375,7 +2627,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_resumes_partial_rustfs_writes() {
|
||||
for drive_count in [3, 4] {
|
||||
for drive_count in [2, 3, 4] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
@@ -2434,23 +2686,32 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_format_migration_rejects_a_foreign_partial_rustfs_format() {
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
let legacy = FormatV3::new(1, 3);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, 3);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
for first_disk in [false, true] {
|
||||
for drive_count in [2, 3] {
|
||||
let (_temp_dir, mut disks) = local_disks(drive_count).await;
|
||||
let legacy = FormatV3::new(1, drive_count);
|
||||
write_legacy_majority(&disks, &legacy).await;
|
||||
let mut foreign = FormatV3::new(1, drive_count);
|
||||
foreign.erasure.this = foreign.erasure.sets[0][0];
|
||||
let foreign_id = foreign.id;
|
||||
save_format_file(&disks[0], &Some(foreign))
|
||||
.await
|
||||
.expect("foreign partial RustFS format should be written");
|
||||
|
||||
assert!(matches!(
|
||||
connect_load_init_formats(true, &mut disks, 1, 3, None).await,
|
||||
Err(Error::CorruptedFormat)
|
||||
));
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
let result = connect_load_init_formats(first_disk, &mut disks, 1, drive_count, None).await;
|
||||
if !first_disk && drive_count == 3 {
|
||||
assert!(matches!(result, Err(Error::ErasureReadQuorum)), "a follower below read quorum must wait");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(result, Err(Error::CorruptedFormat)),
|
||||
"an eligible reader or migration writer must reject the foreign legacy layout"
|
||||
);
|
||||
}
|
||||
let (formats, _) = load_format_erasure_all(&disks, false).await;
|
||||
assert_eq!(formats[0].as_ref().map(|format| format.id), Some(foreign_id));
|
||||
assert!(formats[1..].iter().all(Option::is_none), "foreign partial state must not be extended");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1590,6 +1590,15 @@ where
|
||||
args: &AddOrUpdateUserReq,
|
||||
updated_at: OffsetDateTime,
|
||||
) -> Result<(OffsetDateTime, bool, tokio::sync::OwnedMutexGuard<crate::cache::MutationLockState>)> {
|
||||
// A virtual OIDC parent is a canonical base64url-encoded SHA-256 digest.
|
||||
// It has no stored user record to collide with in the checks below.
|
||||
if access_key.len() == 43
|
||||
&& base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(access_key.as_bytes())
|
||||
.is_ok_and(|bytes| bytes.len() == 32)
|
||||
{
|
||||
return Err(Error::AccessKeyAlreadyExists);
|
||||
}
|
||||
let parent_lock = self.cache.parent_mutation_lock(access_key);
|
||||
let parent_guard = parent_lock.lock_owned().await;
|
||||
let _service_guard = self.cache.service_account_mutation_lock().lock().await;
|
||||
|
||||
@@ -143,7 +143,6 @@ const SVC_ACC_LIST_KEY: &str = "service-accounts/";
|
||||
const GROUPS_LIST_KEY: &str = "groups/";
|
||||
const POLICIES_LIST_KEY: &str = "policies/";
|
||||
const STS_LIST_KEY: &str = "sts/";
|
||||
const POLICY_DB_PREFIX: &str = "policydb/";
|
||||
const POLICY_DB_USERS_LIST_KEY: &str = "policydb/users/";
|
||||
const POLICY_DB_STS_USERS_LIST_KEY: &str = "policydb/sts-users/";
|
||||
const POLICY_DB_GROUPS_LIST_KEY: &str = "policydb/groups/";
|
||||
@@ -173,17 +172,6 @@ struct LazyRewriteEntry {
|
||||
static IAM_LAZY_REWRITE_TRACKER: LazyLock<Mutex<HashMap<String, LazyRewriteEntry>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
// split_path splits a path into a top-level directory and a child item. The
|
||||
// parent directory retains the trailing slash.
|
||||
fn split_path(s: &str, last_index: bool) -> (&str, &str) {
|
||||
let i = if last_index { s.rfind('/') } else { s.find('/') };
|
||||
|
||||
match i {
|
||||
Some(index) => (&s[..index + 1], &s[index + 1..]),
|
||||
None => (s, ""),
|
||||
}
|
||||
}
|
||||
|
||||
/// Attempts to decrypt an at-rest IAM config blob using the same key sources as
|
||||
/// the IAM load path: the RustFS master keys (`RUSTFS_IAM_MASTER_KEY[_OLD_KEYS]`)
|
||||
/// and the MinIO-compatible legacy keys derived from the root credentials
|
||||
@@ -490,36 +478,41 @@ impl ObjectStore {
|
||||
|
||||
#[hotpath::measure]
|
||||
async fn list_all_iamconfig_items(&self) -> Result<HashMap<String, Vec<String>>> {
|
||||
let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
|
||||
self.list_iam_config_items(format!("{}/", *IAM_CONFIG_PREFIX).as_str(), ctx.clone(), tx)
|
||||
.await;
|
||||
|
||||
let mut res = HashMap::new();
|
||||
let ctx = CancellationToken::new();
|
||||
// Walk only cache partitions. Other children of config/iam (notably
|
||||
// plugin delegations) have independent lifetimes and cardinalities.
|
||||
for key in [
|
||||
USERS_LIST_KEY,
|
||||
GROUPS_LIST_KEY,
|
||||
POLICIES_LIST_KEY,
|
||||
SVC_ACC_LIST_KEY,
|
||||
STS_LIST_KEY,
|
||||
POLICY_DB_USERS_LIST_KEY,
|
||||
POLICY_DB_GROUPS_LIST_KEY,
|
||||
POLICY_DB_STS_USERS_LIST_KEY,
|
||||
] {
|
||||
let (tx, mut rx) = mpsc::channel::<StringOrErr>(100);
|
||||
self.list_iam_config_items(&format!("{}/{key}", *IAM_CONFIG_PREFIX), ctx.clone(), tx)
|
||||
.await;
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!(
|
||||
path_kind = "iam_config",
|
||||
operation = "list_items",
|
||||
reason = "walk_result",
|
||||
error = %err,
|
||||
"system path list failed"
|
||||
);
|
||||
record_system_path_failure("iam_config", "list_items", "walk_result");
|
||||
ctx.cancel();
|
||||
|
||||
while let Some(v) = rx.recv().await {
|
||||
if let Some(err) = v.err {
|
||||
warn!(
|
||||
path_kind = "iam_config",
|
||||
operation = "list_items",
|
||||
reason = "walk_result",
|
||||
error = %err,
|
||||
"system path list failed"
|
||||
);
|
||||
record_system_path_failure("iam_config", "list_items", "walk_result");
|
||||
ctx.cancel();
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(item) = v.item {
|
||||
let last_index = item.starts_with(POLICY_DB_PREFIX);
|
||||
let (list_key, trimmed_item) = split_path(&item, last_index);
|
||||
res.entry(list_key.to_owned())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(trimmed_item.to_owned());
|
||||
if let Some(item) = v.item {
|
||||
res.entry(key.to_owned()).or_insert_with(Vec::new).push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1413,6 +1406,12 @@ mod tests {
|
||||
)
|
||||
.await;
|
||||
let store = ObjectStore::new(env.ecstore);
|
||||
store
|
||||
.save_iam_config("fixture", "config/iam/plugin-delegations/v1/fixture")
|
||||
.await
|
||||
.expect("seed unrelated partition");
|
||||
let listed = store.list_all_iamconfig_items().await.expect("list IAM partitions");
|
||||
assert!(!listed.contains_key("plugin-delegations/"));
|
||||
store
|
||||
.save_group_info("reload-group", GroupInfo::new(vec!["reload-user".to_string()]))
|
||||
.await
|
||||
|
||||
@@ -4204,6 +4204,11 @@ mod tests {
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
};
|
||||
let virtual_parent = base64_simd::URL_SAFE_NO_PAD.encode_to_string([42; 32]);
|
||||
assert!(matches!(
|
||||
iam.create_user(&virtual_parent, &args).await,
|
||||
Err(IamError::AccessKeyAlreadyExists)
|
||||
));
|
||||
for service in [false, true] {
|
||||
for expired in [false, true] {
|
||||
let key = format!("collision-{service}-{expired}");
|
||||
|
||||
@@ -22,7 +22,7 @@ The outer gate uses the same HTTP/2 and `application/grpc` predicate as the hybr
|
||||
|---|---|---|---|
|
||||
| StorageReady | Storage/global config readiness publication plus runtime storage checks. | Yes. | Startup can mark the stage before later runtime readiness rechecks storage. Initial pool metadata reads must complete before object-store publication; see [pool metadata recovery](../operations/pool-metadata-recovery.md). |
|
||||
| IamReady | Inline IAM bootstrap or deferred IAM recovery publication. | Yes. | Deferred recovery can publish IAM readiness after HTTP has already started. |
|
||||
| Lock quorum | Per-set write quorum readiness. | Yes. | Do not replace the distributed lock quorum check with node count or endpoint count. |
|
||||
| Lock quorum | Per-set shared-lock quorum readiness. | Yes. | Startup uses the same read boundary as node readiness. Mutations retain their exclusive-lock requirements. Do not replace the distributed lock quorum check with node count or endpoint count. |
|
||||
| Peer health | `peer_health_ready` runtime status. | Only when `RUSTFS_HEALTH_PEER_READY_CHECK_ENABLE` is enabled. | The gate is disabled by default; unknown peer health degrades readiness only when enabled. |
|
||||
| Peer re-admission | A strict majority of the peers in each erasure set this node serves count all of its drives in that set as online (`PeerDriveAdmission` internode RPC). | Startup transition only, while `RUSTFS_STARTUP_PEER_READMISSION_ENABLE` is enabled (default on). | Bounded and fail-open: publishes `FullReady` with a WARN log when `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` elapses, immediately when older peers cannot answer and the majority is out of reach, or after 15 s of unreachable peers keeping it out of reach. Not part of steady-state `/health/ready`. Distinct from peer health, which is this node's view of its peers. |
|
||||
| KMS compatibility | KMS health compatibility readiness. | Only when `RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE` is enabled (default off; `crates/config/src/constants/health.rs`). | When enabled, `/health/ready` additionally requires the KMS service to be running if a global KMS manager exists. KMS startup fatality and health reporting remain separate from pure docs work. |
|
||||
|
||||
@@ -62,6 +62,8 @@ The native CAS, undo and recovery RPC methods require all participating storage
|
||||
|
||||
A runtime metadata or identity read failure before any write is dispatched rejects that operation but does not permanently block subsequent retries. Errors retain their typed cause; pool metadata unavailability reaches S3 as `503 ServiceUnavailable`, without exposing internal error details. Cancellation during preflight or a rejected first conditional write is also retryable. After any identity, prepare, or commit write may have started, cancellation, an uncertain write result, or abandoned runtime publication blocks further metadata-dependent mutations.
|
||||
|
||||
An existing deployment with committed, initialized identity and unchanged topology can publish its startup metadata under a shared `pool.bin` namespace fence when no writer-side repair or completion is required. Pending bootstrap, identity completion, topology changes, and recovery retain the exclusive writer fence. Observing an initialized identity clears bootstrap provenance, so later initialization attempts use the same read-admission path. Startup releases the shared fence before acquiring the writer and reloads the identity and replicas under exclusive ownership; observations made by the reader never authorize later writes. This avoids requiring write-lock quorum merely to start a readable survivor.
|
||||
|
||||
During startup, an unreadable identity or `pool.bin` replica returns typed `ReadUnavailable` before a partial metadata view is installed or the object store is published. Initialization retries from the complete identity and metadata read at the existing five-second retry interval, within the existing startup attempt budget. A later readable result must still pass the original identity, generation, topology, conditional-write and readback checks. Read-only unavailability does not create a permanent startup block; an existing validation or uncertain-write block is never cleared by a clean retry.
|
||||
|
||||
The node checks for interrupted pool metadata transactions every five seconds. Failed recovery attempts back off to at most sixty seconds; each attempt has a thirty-second budget and stops on shutdown. Healthy nodes do not read metadata for this worker. Recovery:
|
||||
|
||||
@@ -147,6 +147,7 @@ struct ObjectZipDownloadTokenPayload {
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
struct ObjectZipDownloadReqInfoSnapshot {
|
||||
session_present: bool,
|
||||
cred: Option<rustfs_credentials::Credentials>,
|
||||
is_owner: bool,
|
||||
bucket: Option<String>,
|
||||
@@ -187,12 +188,16 @@ struct PreparedZipArchive {
|
||||
impl From<&ReqInfo> for ObjectZipDownloadReqInfoSnapshot {
|
||||
fn from(req_info: &ReqInfo) -> Self {
|
||||
let mut cred = req_info.cred.clone();
|
||||
let session_present = req_info
|
||||
.keystone_session_present
|
||||
.unwrap_or_else(|| cred.as_ref().is_some_and(|cred| !cred.session_token.is_empty()));
|
||||
if let Some(cred) = cred.as_mut() {
|
||||
cred.secret_key.clear();
|
||||
cred.session_token.clear();
|
||||
}
|
||||
|
||||
Self {
|
||||
session_present,
|
||||
cred,
|
||||
is_owner: req_info.is_owner,
|
||||
bucket: req_info.bucket.clone(),
|
||||
@@ -207,7 +212,7 @@ impl From<ObjectZipDownloadReqInfoSnapshot> for ReqInfo {
|
||||
Self {
|
||||
cred: snapshot.cred,
|
||||
is_owner: snapshot.is_owner,
|
||||
keystone_session_present: None,
|
||||
keystone_session_present: Some(snapshot.session_present),
|
||||
bucket: snapshot.bucket,
|
||||
object: snapshot.object,
|
||||
version_id: snapshot.version_id,
|
||||
@@ -382,7 +387,14 @@ fn validate_download_token(
|
||||
}
|
||||
let expires_at = OffsetDateTime::from_unix_timestamp(payload.expires_at_unix)
|
||||
.map_err(|_| s3_error!(AccessDenied, "invalid or expired download token"))?;
|
||||
if expires_at <= now {
|
||||
if expires_at <= now
|
||||
|| payload
|
||||
.req_info
|
||||
.cred
|
||||
.as_ref()
|
||||
.and_then(|cred| cred.expiration)
|
||||
.is_some_and(|expiration| expiration <= now)
|
||||
{
|
||||
return Err(s3_error!(AccessDenied, "invalid or expired download token"));
|
||||
}
|
||||
|
||||
@@ -446,15 +458,9 @@ fn authenticated_zip_download_principal(req: &AdminRequest) -> S3Result<String>
|
||||
}
|
||||
|
||||
fn authenticated_zip_download_req_info(req: &AdminRequest) -> S3Result<ReqInfo> {
|
||||
let mut req_info = req
|
||||
.req_info()
|
||||
req.req_info()
|
||||
.cloned()
|
||||
.ok_or_else(|| s3_error!(AccessDenied, "authentication required"))?;
|
||||
if let Some(cred) = req_info.cred.as_mut() {
|
||||
cred.secret_key.clear();
|
||||
cred.session_token.clear();
|
||||
}
|
||||
Ok(req_info)
|
||||
.ok_or_else(|| s3_error!(AccessDenied, "authentication required"))
|
||||
}
|
||||
|
||||
fn authenticated_zip_download_auth_context(req: &AdminRequest) -> ObjectZipDownloadAuthContext {
|
||||
@@ -1328,6 +1334,8 @@ mod tests {
|
||||
let record =
|
||||
validate_download_token(&created.id, &created.token, now, &credentials).expect("matching token should be accepted");
|
||||
|
||||
assert_eq!(record.req_info.keystone_session_present, Some(true));
|
||||
assert!(record.req_info.cred.as_ref().expect("credential").session_token.is_empty());
|
||||
assert_eq!(record.id, created.id);
|
||||
assert_eq!(record.principal, "alice");
|
||||
assert_eq!(record.request.bucket, "photos");
|
||||
@@ -1680,14 +1688,13 @@ mod tests {
|
||||
}
|
||||
|
||||
fn test_req_info() -> ReqInfo {
|
||||
let mut credentials = rustfs_credentials::Credentials {
|
||||
let credentials = rustfs_credentials::Credentials {
|
||||
access_key: "alice".to_string(),
|
||||
secret_key: "should-not-be-serialized".to_string(),
|
||||
session_token: "should-not-be-serialized".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
credentials.secret_key.clear();
|
||||
credentials.session_token.clear();
|
||||
|
||||
ReqInfo {
|
||||
cred: Some(credentials),
|
||||
is_owner: false,
|
||||
|
||||
@@ -970,7 +970,7 @@ impl TableCredentialIssuer for IamTableCredentialIssuer {
|
||||
bind_table_credential_parent(&mut credential, principal);
|
||||
|
||||
self.iam_store
|
||||
.set_temp_user(&credential.access_key, &credential, None)
|
||||
.set_temp_user_from_parent(&credential, principal)
|
||||
.await
|
||||
.map_err(|_| s3_error!(InternalError, "failed to store table credentials"))?;
|
||||
|
||||
|
||||
+3
-1
@@ -2154,7 +2154,9 @@ mod tests {
|
||||
);
|
||||
non_utf8.insert("If-Match", HeaderValue::from_bytes(b"\xff").expect("obs-text is a valid header value"));
|
||||
|
||||
for headers in [HeaderMap::new(), blank, non_utf8] {
|
||||
let mut existence_only = HeaderMap::new();
|
||||
existence_only.insert(IF_MATCH, HeaderValue::from_static(" * "));
|
||||
for headers in [HeaderMap::new(), blank, non_utf8, existence_only] {
|
||||
let conditions = get_condition_values(&headers, &cred, None, None, None);
|
||||
|
||||
assert!(!conditions.contains_key("if-none-match"), "headers {headers:?}");
|
||||
|
||||
@@ -556,7 +556,7 @@ pub(crate) fn base_conditions(principal: &AuthzPrincipal<'_>, facts: &AuthzFacts
|
||||
.get(name)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
.filter(|value| !value.is_empty() && !(name == IF_MATCH.as_str() && *value == "*"))
|
||||
{
|
||||
args.insert(name.to_owned(), vec![value.to_owned()]);
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ use http::header::{CACHE_CONTROL, CONTENT_LOCATION, CONTENT_RANGE, CONTENT_TYPE,
|
||||
use http::{HeaderMap, HeaderName, HeaderValue};
|
||||
use rustfs_gateway::{
|
||||
ErrorHeader, HandlerError, HandlerErrorContext, HttpDate, LegacyRustfsFacts, LegacyRustfsRefusal, RedirectTarget,
|
||||
RegionLabel, StoredHeaderValue, VersionIdLabel,
|
||||
RegionLabel, RustfsErrorCause, StoredHeaderValue, VersionIdLabel,
|
||||
};
|
||||
use rustfs_gateway_types::{ETag, ErrorCode, Timestamp, TimestampFormat};
|
||||
use rustfs_s3_types::S3Error;
|
||||
@@ -84,6 +84,10 @@ pub(crate) fn handler_error(error: S3Error) -> HandlerError {
|
||||
}
|
||||
let mut mapped = HandlerError::new(code, message.unwrap_or(Cow::Borrowed("")));
|
||||
for (name, value) in &headers {
|
||||
// The error document's `Content-Type` is the gateway's to write.
|
||||
if name == CONTENT_TYPE {
|
||||
continue;
|
||||
}
|
||||
let Some(header) = error_header(name, value) else {
|
||||
return HandlerError::internal_error(UNWRITABLE_HEADER);
|
||||
};
|
||||
@@ -179,6 +183,15 @@ fn error_header(name: &HeaderName, value: &HeaderValue) -> Option<ErrorHeader> {
|
||||
"retry-after" => Some(ErrorHeader::RetryAfter {
|
||||
seconds: value.parse().ok()?,
|
||||
}),
|
||||
// The gateway spells the closed set of causes; an unknown one fails the whole error.
|
||||
"x-rustfs-error-cause" => [
|
||||
RustfsErrorCause::NamespaceLockTimeout,
|
||||
RustfsErrorCause::NamespaceLockUnavailable,
|
||||
RustfsErrorCause::StorageWriteQuorum,
|
||||
]
|
||||
.into_iter()
|
||||
.find(|cause| cause.as_str() == value)
|
||||
.map(|cause| ErrorHeader::RustfsErrorCause { cause }),
|
||||
"x-amz-bucket-region" => Some(ErrorHeader::BucketRegion {
|
||||
region: RegionLabel::new(value).ok()?,
|
||||
}),
|
||||
@@ -382,6 +395,52 @@ mod tests {
|
||||
assert_eq!(rendered[6], "Tue, 14 Nov 2023 22:13:20 GMT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_content_type_the_raiser_set_is_left_to_the_gateway() {
|
||||
let mapped = with_headers(
|
||||
S3ErrorCode::SlowDown,
|
||||
&[(header::CONTENT_TYPE, "application/xml"), (header::RETRY_AFTER, "1")],
|
||||
);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "SlowDown");
|
||||
assert_eq!(mapped.headers(), [ErrorHeader::RetryAfter { seconds: 1 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unavailable_error_states_its_cause_beside_the_retry_advice() {
|
||||
let cause = HeaderName::from_static("x-rustfs-error-cause");
|
||||
let mapped = with_headers(
|
||||
S3ErrorCode::ServiceUnavailable,
|
||||
&[(cause.clone(), "namespace_lock_timeout"), (header::RETRY_AFTER, "1")],
|
||||
);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "ServiceUnavailable");
|
||||
assert!(mapped.headers().contains(&ErrorHeader::RustfsErrorCause {
|
||||
cause: RustfsErrorCause::NamespaceLockTimeout
|
||||
}));
|
||||
assert!(mapped.headers().contains(&ErrorHeader::RetryAfter { seconds: 1 }));
|
||||
for (value, expected) in [
|
||||
("namespace_lock_unavailable", RustfsErrorCause::NamespaceLockUnavailable),
|
||||
("storage_write_quorum", RustfsErrorCause::StorageWriteQuorum),
|
||||
] {
|
||||
let mapped = with_headers(S3ErrorCode::ServiceUnavailable, &[(cause.clone(), value)]);
|
||||
assert_eq!(mapped.headers(), [ErrorHeader::RustfsErrorCause { cause: expected }], "{value}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cause_outside_the_closed_set_fails_the_whole_error() {
|
||||
for value in ["storage_read_quorum", "NAMESPACE_LOCK_TIMEOUT", ""] {
|
||||
let mapped = with_headers(
|
||||
S3ErrorCode::ServiceUnavailable,
|
||||
&[(HeaderName::from_static("x-rustfs-error-cause"), value)],
|
||||
);
|
||||
|
||||
assert_eq!(mapped.code(), &ErrorCode::INTERNAL_ERROR, "{value:?}");
|
||||
assert_eq!(mapped.message(), UNWRITABLE_HEADER, "{value:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_outside_the_closed_set_fails_the_whole_error() {
|
||||
let mapped = with_headers(S3ErrorCode::AccessDenied, &[(HeaderName::from_static("x-rustfs-unlisted"), "value")]);
|
||||
|
||||
@@ -1405,7 +1405,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn node_storage_snapshot_tracks_read_write_quorum_and_recovery() {
|
||||
for (drive_count, parity) in [(4, 2), (4, 1), (8, 2)] {
|
||||
for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (4, 1), (8, 2)] {
|
||||
let store = runtime_inventory(&[(1, drive_count, parity)]).await;
|
||||
let data = drive_count - parity;
|
||||
let write_quorum = data + usize::from(data == parity);
|
||||
@@ -1690,6 +1690,54 @@ mod tests {
|
||||
assert!(!readiness_gate_blocks_path("/bucket/object", &readiness));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_drive_startup_admits_reads_without_write_quorum_but_requires_iam() {
|
||||
let mut store = runtime_inventory(&[(1, 2, 1)]).await;
|
||||
store.disks.get_mut(&DiskSetSelector::new(0, 0)).expect("configured set")[1] = None;
|
||||
|
||||
for (online, iam_ready, expected_ready) in [(true, true, true), (true, false, false), (false, true, false)] {
|
||||
let hosts = if online {
|
||||
HashSet::from(["node-0:9000".to_string()])
|
||||
} else {
|
||||
HashSet::new()
|
||||
};
|
||||
let (info, unavailable_drives, missing_drives) = node_storage_snapshot(&store, &hosts)
|
||||
.await
|
||||
.expect("snapshot surviving drive and missing slot");
|
||||
assert_eq!(missing_drives.len(), 1);
|
||||
let (dependencies, _, details) = apply_node_service_readiness(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetaWriteBlocked),
|
||||
},
|
||||
StorageReadinessDetails {
|
||||
read_quorum_ready: storage_read_ready_from_runtime_state(&info),
|
||||
write_quorum_ready: storage_ready_from_runtime_state(&info),
|
||||
unavailable_drives,
|
||||
missing_drives,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
iam_ready,
|
||||
true,
|
||||
);
|
||||
assert!(!details.write_quorum_ready, "one surviving drive cannot authorize writes");
|
||||
assert!(!details.pool_metadata_write_ready);
|
||||
let readiness = GlobalReadiness::new();
|
||||
assert!(readiness_gate_blocks_path("/bucket/object", &readiness));
|
||||
let result = wait_for_runtime_readiness_with(
|
||||
Duration::ZERO,
|
||||
Duration::from_millis(1),
|
||||
|| future::ready(dependencies),
|
||||
|_| readiness.mark_stage(rustfs_common::SystemStage::FullReady),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.is_ok(), expected_ready, "online={online}, iam_ready={iam_ready}");
|
||||
assert_eq!(readiness.is_ready(), expected_ready);
|
||||
assert_eq!(readiness_gate_blocks_path("/bucket/object", &readiness), !expected_ready);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn node_storage_snapshot_bounds_waits_and_propagates_inventory_failure() {
|
||||
let mut store = runtime_inventory(&[(1, 4, 2)]).await;
|
||||
|
||||
Reference in New Issue
Block a user