fix(object-capacity): harden write timing, interval bounds, and scope expiry (#4575)

fix(object-capacity): harden clock handling in write window, background intervals and scope registry

Three low-severity robustness gaps (S32+S33+#35):

- WriteRecord keyed its 60-bucket write window by wall-clock unix
  seconds while the debounce used Instant. An NTP step backwards marked
  recent buckets as future and silently suppressed write-triggered
  refreshes until the wall clock caught up; a forward step aged the
  whole window at once. Bucket keys now derive from a monotonic
  per-record epoch, immune to clock steps.
- Background refresh/metrics intervals were only clamped at zero;
  RUSTFS_CAPACITY_SCHEDULED_INTERVAL=u64::MAX overflowed
  Instant + Duration and panicked the spawned task, silently killing
  scheduled refreshes. Intervals now clamp into [1s, 30 days] via one
  helper with a structured warn.
- record_capacity_scope merged new disks into an expired entry and
  refreshed its recorded_at, resurrecting a scope take_capacity_scope
  would have discarded (and the merge path never pruned). Expired
  entries are now replaced, not merged into.

Ref: rustfs/backlog#1022 (S32+S33+#35 from audit rustfs/backlog#1010)
This commit is contained in:
Zhengchao An
2026-07-09 05:27:36 +08:00
committed by GitHub
parent c92d47df97
commit ddb9d8ca3e
2 changed files with 114 additions and 45 deletions
+44 -2
View File
@@ -83,8 +83,15 @@ pub fn record_capacity_scope(token: Uuid, scope: CapacityScope) {
enforce_hard_limit(&mut entries, CAPACITY_SCOPE_REGISTRY_HARD_LIMIT);
}
if let Some(entry) = entries.get_mut(&token) {
merge_capacity_scopes(&mut entry.scope, scope);
entry.recorded_at = now;
// An expired entry would already be discarded by take_capacity_scope;
// merging into it (and refreshing recorded_at) would resurrect stale
// disks. Replace it with the fresh scope instead (backlog#1022 #35).
if now.duration_since(entry.recorded_at) > CAPACITY_SCOPE_TTL {
*entry = CapacityScopeEntry { scope, recorded_at: now };
} else {
merge_capacity_scopes(&mut entry.scope, scope);
entry.recorded_at = now;
}
} else {
entries.insert(token, CapacityScopeEntry { scope, recorded_at: now });
}
@@ -156,6 +163,41 @@ mod tests {
.join();
}
#[test]
fn record_capacity_scope_replaces_expired_entry_instead_of_merging() {
let _guard = test_lock().lock().expect("test lock poisoned");
clear_capacity_scope_registry_for_test();
let token = Uuid::new_v4();
let stale_disk = CapacityScopeDisk {
endpoint: "node-old".to_string(),
drive_path: "/tmp/disk-old".to_string(),
};
let fresh_disk = CapacityScopeDisk {
endpoint: "node-new".to_string(),
drive_path: "/tmp/disk-new".to_string(),
};
record_capacity_scope(token, CapacityScope { disks: vec![stale_disk] });
// Backdate the entry beyond the TTL: take_capacity_scope would drop
// it, so a new record for the same token must not resurrect it.
capacity_scope_registry()
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.get_mut(&token)
.expect("entry must exist")
.recorded_at = Instant::now() - CAPACITY_SCOPE_TTL - Duration::from_secs(1);
record_capacity_scope(
token,
CapacityScope {
disks: vec![fresh_disk.clone()],
},
);
assert_eq!(take_capacity_scope(token), Some(CapacityScope { disks: vec![fresh_disk] }));
clear_capacity_scope_registry_for_test();
}
#[test]
fn record_and_take_capacity_scope_round_trips() {
let _guard = test_lock().lock().expect("test lock poisoned");