diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index f971dae9b..8c5132487 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -800,11 +800,22 @@ where if log_error { error!("save_config_with_opts: err: {:?}, file: {}", err, file); } - Err(err) + Err(map_system_metadata_write_error(err, file)) } } } +/// A system metadata volume outage must remain retryable instead of being +/// exposed as the user-facing bucket-not-found response. +pub(crate) fn map_system_metadata_write_error(err: Error, file: &str) -> Error { + match err { + Error::BucketNotFound(_) | Error::VolumeNotFound => { + Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), file.to_string()) + } + other => other, + } +} + fn new_server_config() -> Config { Config::new() } @@ -2796,14 +2807,14 @@ mod tests { use super::{ SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, build_scalar_config_object, config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, - heal_config_descriptor, is_standard_object_server_config, lookup_configs, new_and_save_server_config, read_config, - read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, read_config_with_metadata, - read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot, - save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section, - storage_class_kvs_mut, + heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error, + new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, + read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner, + save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation, + server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut, }; use crate::config::{audit, heal, notify, oidc, scanner}; - use crate::disk::endpoint::Endpoint; + use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint}; use crate::error::{Error, Result}; use crate::layout::endpoints::SetupType; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; @@ -2835,6 +2846,72 @@ mod tests { assert!(rendered.contains("panicked")); assert!(!rendered.contains("do-not-expose-payload")); } + + #[test] + fn system_metadata_volume_failures_map_to_retryable_write_errors() { + for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] { + assert_eq!( + map_system_metadata_write_error(error, "buckets/example/.metadata.bin"), + Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string()) + ); + } + + let other = Error::other("metadata encoding failed"); + assert_eq!(map_system_metadata_write_error(other.clone(), "buckets/example/.metadata.bin"), other); + } + + #[derive(Debug, Default)] + struct MetadataWriteStore { + error: Option, + } + + #[async_trait::async_trait] + impl crate::storage_api_contracts::object::ObjectIO for MetadataWriteStore { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + + async fn get_object_reader( + &self, + _bucket: &str, + _object: &str, + _range: Option, + _headers: Self::HeaderMap, + _opts: &Self::ObjectOptions, + ) -> core::result::Result { + Err(Error::FileNotFound) + } + + async fn put_object( + &self, + _bucket: &str, + _object: &str, + _data: &mut Self::PutObjectReader, + _opts: &Self::ObjectOptions, + ) -> core::result::Result { + Err(self.error.clone().expect("test store error should be configured")) + } + } + + #[tokio::test] + async fn save_config_preserves_retryable_system_volume_errors() { + let store = Arc::new(MetadataWriteStore { + error: Some(Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())), + }); + let error = + save_config_with_opts_inner(store, "buckets/example/.metadata.bin", Vec::new(), &ObjectOptions::default(), false) + .await + .expect_err("missing metadata volume must fail"); + + assert_eq!( + error, + Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string()) + ); + } use rustfs_lock::client::LockClient; use rustfs_lock::client::local::LocalClient; use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats}; diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 719fb8f27..734e1a982 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -422,9 +422,7 @@ async fn save_data_usage_in_backend( if publication_epoch != expected_publication_epoch { return Err(Error::other("data usage publication epoch changed before save")); } - crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data) - .await - .map_err(Error::other)?; + crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await?; drop(publication_guard); cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref())) @@ -641,7 +639,7 @@ where { Ok(reader) => reader, Err(Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::ConfigNotFound) => return Ok(None), - Err(err) => return Err(err), + Err(err) => return Err(map_data_usage_metadata_read_error(err, object)), }; let revision = reader .object_info @@ -656,6 +654,18 @@ where Ok(Some((data_usage_info, revision))) } +/// A missing usage object is harmless during bucket creation, but a missing +/// system metadata volume is a storage outage. Keep the latter retryable and +/// distinguishable from the user bucket not existing. +fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error { + match err { + Error::BucketNotFound(_) | Error::VolumeNotFound => { + Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), object.to_string()) + } + other => other, + } +} + fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool { data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket) } @@ -912,7 +922,7 @@ where ) .await; drop(publication_guard); - match save_result { + match save_result.map_err(|err| crate::config::com::map_system_metadata_write_error(err, object)) { Ok(_) => return Ok(()), Err(err) => { if let Some((observed, observed_revision)) = load_data_usage_for_bucket_removal(store, object).await? { @@ -2759,6 +2769,7 @@ mod tests { struct UsageCacheReadStore { transient_failures: Mutex, reads: Mutex>, + terminal_error: Mutex>, } impl UsageCacheReadStore { @@ -2766,6 +2777,15 @@ mod tests { Self { transient_failures: Mutex::new(n), reads: Mutex::new(Vec::new()), + terminal_error: Mutex::new(None), + } + } + + fn with_terminal_error(error: Error) -> Self { + Self { + transient_failures: Mutex::new(0), + reads: Mutex::new(Vec::new()), + terminal_error: Mutex::new(Some(error)), } } @@ -2793,6 +2813,9 @@ mod tests { _opts: &Self::ObjectOptions, ) -> Result { self.reads.lock().await.push(object.to_string()); + if let Some(error) = self.terminal_error.lock().await.clone() { + return Err(error); + } let mut remaining = self.transient_failures.lock().await; if *remaining > 0 { *remaining -= 1; @@ -2857,6 +2880,22 @@ mod tests { assert!(!is_data_usage_cache_absent(&Error::DiskNotFound)); } + #[test] + fn data_usage_removal_maps_missing_system_volume_to_read_quorum() { + for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] { + assert_eq!( + map_data_usage_metadata_read_error(error, "bucket-metadata/.usage.json"), + Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string()) + ); + } + + let missing_object = Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string()); + assert_eq!( + map_data_usage_metadata_read_error(missing_object.clone(), "bucket-metadata/.usage.json"), + missing_object + ); + } + #[tokio::test] async fn load_data_usage_cache_treats_absence_as_an_empty_cache_without_retrying() { let name = "usage-cache"; @@ -2872,6 +2911,22 @@ mod tests { ); } + #[tokio::test] + async fn data_usage_removal_surfaces_missing_system_volume_as_read_quorum() { + for cause in [Error::BucketNotFound(RUSTFS_META_BUCKET.to_string()), Error::VolumeNotFound] { + let store = UsageCacheReadStore::with_terminal_error(cause); + + let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json") + .await + .expect_err("missing system metadata volume must not be treated as an absent usage object"); + + assert_eq!( + error, + Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string()) + ); + } + } + #[tokio::test] async fn load_data_usage_cache_retries_a_transient_failure() { let name = "usage-cache";