mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 07:36:53 +00:00
fix(object-capacity): mark partial-failure refreshes degraded and merge over disk cache (#4499)
A full refresh with partial disk failures used to commit the surviving subset's sum as a fresh exact cluster total (no field carried the partial-failure fact), while the complete disk cache kept the failed disk's old value — so the reported capacity oscillated between the partial sum and the cache-merged total on alternating refreshes. - Add a degraded flag to CapacityUpdate/CachedCapacity, set when the scan behind the update had partial errors; expose it in refresh logs, admin capacity logs and a new rustfs_capacity_degraded_readings_total counter. - On a degraded full refresh, surface only the disks whose own scan fully succeeded and merge them over a complete disk cache, so failed disks keep their last-known values and the published total no longer dips and bounces back. The disk cache is never replaced from a degraded refresh. - Without a complete cache, keep the partial sum (unchanged #805 non-pollution behavior) but mark the reading degraded. Ref: rustfs/backlog#1014 (S06 from audit rustfs/backlog#1010)
This commit is contained in:
@@ -127,8 +127,8 @@ pub async fn resolve_admin_used_capacity(disks: &[rustfs_madmin::Disk], fallback
|
||||
if cache_age < fast_update_threshold {
|
||||
record_capacity_cache_served("fresh");
|
||||
debug!(
|
||||
"Using cached capacity: {} bytes (age: {:?}, source: {:?}, files={}, estimated={})",
|
||||
cached.total_used, cache_age, cached.source, cached.file_count, cached.is_estimated
|
||||
"Using cached capacity: {} bytes (age: {:?}, source: {:?}, files={}, estimated={}, degraded={})",
|
||||
cached.total_used, cache_age, cached.source, cached.file_count, cached.is_estimated, cached.degraded
|
||||
);
|
||||
return cached.total_used;
|
||||
}
|
||||
@@ -157,6 +157,7 @@ pub async fn resolve_admin_used_capacity(disks: &[rustfs_madmin::Disk], fallback
|
||||
duration_ms = elapsed.as_millis() as u64,
|
||||
file_count = update.file_count,
|
||||
is_estimated = update.is_estimated,
|
||||
degraded = update.degraded,
|
||||
"Capacity refresh completed"
|
||||
);
|
||||
update.total_used
|
||||
@@ -188,6 +189,7 @@ pub async fn resolve_admin_used_capacity(disks: &[rustfs_madmin::Disk], fallback
|
||||
source = ?cached.source,
|
||||
file_count = cached.file_count,
|
||||
is_estimated = cached.is_estimated,
|
||||
degraded = cached.degraded,
|
||||
needs_update,
|
||||
blocking = should_block,
|
||||
"Served cached capacity"
|
||||
@@ -233,6 +235,7 @@ pub async fn resolve_admin_used_capacity(disks: &[rustfs_madmin::Disk], fallback
|
||||
duration_ms = elapsed.as_millis() as u64,
|
||||
file_count = update.file_count,
|
||||
is_estimated = update.is_estimated,
|
||||
degraded = update.degraded,
|
||||
"Capacity refresh completed"
|
||||
);
|
||||
update.total_used
|
||||
|
||||
Reference in New Issue
Block a user