From daca7294c7793a7fa153886003e6c3588933116a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 22 Jul 2026 13:26:09 +0800 Subject: [PATCH] fix(ecstore): stop logging not-found listing quorum miss at error (#5111) A metacache listing that misses quorum purely because the volume or path is absent on a quorum of drives (every drive reports VolumeNotFound or FileNotFound) is a benign, expected outcome: list_path_raw already returns VolumeNotFound/FileNotFound and lets the caller decide how to react. The common trigger is a startup race where the system bucket (.rustfs.sys) is not yet created on every drive when an early reader such as the IAM config loader lists config/iam/. Logging that at error prints a scary message during normal boot and, in #5076, misled a user into blaming it for unrelated upload failures. Add is_benign_not_found_listing_failure() and demote the pure not-found case to debug (state = "quorum_not_found"), keeping error for listings that failed for a real reason (I/O, timeout, corruption). Return value and control flow are unchanged. Add a unit test for the classifier. Refs: #5076 --- .../ecstore/src/cache_value/metacache_set.rs | 74 ++++++++++++++++--- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/crates/ecstore/src/cache_value/metacache_set.rs b/crates/ecstore/src/cache_value/metacache_set.rs index 00ad1d6b4..b48fd6ae0 100644 --- a/crates/ecstore/src/cache_value/metacache_set.rs +++ b/crates/ecstore/src/cache_value/metacache_set.rs @@ -124,6 +124,24 @@ fn classify_listing_quorum_failure(errors: &[DiskError]) -> DiskError { DiskError::ErasureReadQuorum } +/// Returns true when a metacache listing missed quorum purely because the +/// volume or path is absent on a quorum of drives, i.e. every recorded failure +/// is [`DiskError::VolumeNotFound`] or [`DiskError::FileNotFound`]. +/// +/// This is a benign, expected outcome: the caller is handed +/// `VolumeNotFound`/`FileNotFound` and decides how to react. The most common +/// trigger is a startup race where the system bucket has not yet been created +/// on every drive when an early reader (e.g. the IAM config loader) lists it. +/// It must be distinguished from a listing that failed for a real reason (I/O, +/// timeout, corruption) so the former is not surfaced at `error`. See +/// rustfs/rustfs#5076. +fn is_benign_not_found_listing_failure(errors: &[DiskError]) -> bool { + !errors.is_empty() + && errors + .iter() + .all(|err| matches!(err, DiskError::VolumeNotFound | DiskError::FileNotFound)) +} + struct PublishedBytesWriter { inner: W, published: bool, @@ -840,17 +858,38 @@ async fn list_path_raw_inner( _ => {} }); - error!( - event = EVENT_METACACHE_LISTING, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_METACACHE, - bucket = %opts.bucket, - path = %opts.path, - state = "quorum_failed", - error = %combined_err.join(", "), - "Metacache listing quorum failed" - ); let failures = errs.iter().flatten().cloned().collect::>(); + // A listing that misses quorum purely because the volume/path is + // absent on a quorum of drives is benign and expected — the caller + // receives VolumeNotFound/FileNotFound and decides how to react. + // The common trigger is a startup race where the system bucket is + // not yet created on every drive when an early reader (e.g. the IAM + // config loader) lists it, so surfacing it at `error` is misleading + // noise (rustfs/rustfs#5076). Keep `error` for listings that failed + // for a real reason (I/O, timeout, corruption). + if is_benign_not_found_listing_failure(&failures) { + debug!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + bucket = %opts.bucket, + path = %opts.path, + state = "quorum_not_found", + error = %combined_err.join(", "), + "Metacache listing quorum not reached (volume/path absent)" + ); + } else { + error!( + event = EVENT_METACACHE_LISTING, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_METACACHE, + bucket = %opts.bucket, + path = %opts.path, + state = "quorum_failed", + error = %combined_err.join(", "), + "Metacache listing quorum failed" + ); + } return Err(classify_listing_quorum_failure(&failures)); } @@ -1037,6 +1076,21 @@ mod tests { use time::OffsetDateTime; use uuid::Uuid; + #[test] + fn benign_not_found_listing_failure_detection() { + // Pure not-found quorum misses are benign (the volume/path simply does + // not exist on a quorum of drives) and must not be logged at ERROR. + assert!(is_benign_not_found_listing_failure(&[DiskError::VolumeNotFound])); + assert!(is_benign_not_found_listing_failure( + &[DiskError::VolumeNotFound, DiskError::FileNotFound,] + )); + // No recorded failure is not a not-found case. + assert!(!is_benign_not_found_listing_failure(&[])); + // Any real error must keep the failure at ERROR severity. + assert!(!is_benign_not_found_listing_failure(&[DiskError::VolumeNotFound, DiskError::Timeout,])); + assert!(!is_benign_not_found_listing_failure(&[DiskError::DiskNotFound])); + } + #[tokio::test] async fn list_path_raw_empty_disks_returns_read_quorum() { let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())