gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 16:37:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(capacity): skip idle scheduled disk scans (#6541)

Browse Source 
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-08-25 14:30:29 +08:00
committed by GitHub
parent 76a861f815
commit cf37bc418c
4 changed files with 182 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/object-capacity/src/capacity_manager.rs
+74 -24
View File
@@ -14,7 +14,7 @@
//! Hybrid Capacity Manager for efficient capacity statistics
use super::scan::refresh_capacity_with_scope;
use super::scan::{ScheduledCapacityRefresh, refresh_capacity_with_scope, select_scheduled_capacity_refresh};
use super::types::CapacityDiskRef;
use crate::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty_scopes, take_capacity_scope};
use futures::FutureExt;
@@ -1021,6 +1021,7 @@ impl HybridCapacityManager {
/// remote or removed disks would otherwise stay marked forever and keep
/// the dirty-disk gauge permanently non-zero (backlog#1020 S30).
pub async fn retain_dirty_disks_within(&self, local: &HashSet<CapacityScopeDisk>) {
self.sync_global_dirty_scopes().await;
let mut dirty_disks = self.dirty_disks.write().await;
let before = dirty_disks.len();
dirty_disks.retain(|disk, _| local.contains(disk));
@@ -1378,6 +1379,44 @@ where
}
}
async fn run_scheduled_capacity_refresh(manager: Arc<HybridCapacityManager>, disks: Vec<CapacityDiskRef>) -> bool {
let start = Instant::now();
match select_scheduled_capacity_refresh(manager.as_ref(), &disks).await {
ScheduledCapacityRefresh::Idle => {
debug!(
event = EVENT_CAPACITY_REFRESH_SCHEDULED,
component = LOG_COMPONENT_CAPACITY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "skipped",
source = DataSource::Scheduled.as_metric_label(),
reason = "no_dirty_disks",
disk_count = disks.len(),
"capacity refresh scheduled"
);
true
}
ScheduledCapacityRefresh::Scan { disks, dirty_subset } => {
debug!(
event = EVENT_CAPACITY_REFRESH_SCHEDULED,
component = LOG_COMPONENT_CAPACITY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "started",
source = DataSource::Scheduled.as_metric_label(),
refresh_scope = if dirty_subset { "dirty_subset" } else { "full" },
disk_count = disks.len(),
enqueue_latency_ms = start.elapsed().as_millis() as u64,
"capacity refresh scheduled"
);
let result = manager
.refresh_or_join(DataSource::Scheduled, move || async move {
refresh_capacity_with_scope(disks, dirty_subset).await
})
.await;
scheduled_refresh_was_clean(&result)
}
}
}
/// Owned capacity scheduler tasks for one server runtime.
#[must_use = "capacity background tasks stop when their lifecycle handle is dropped"]
pub struct CapacityBackgroundTasks {
@@ -1429,29 +1468,7 @@ pub async fn start_background_tasks(disks: Vec<CapacityDiskRef>) -> CapacityBack
tasks.spawn(async move {
run_scheduled_refresh_loop(refresh_interval, refresh_shutdown, move || {
let start = Instant::now();
let manager = manager_for_refresh.clone();
let disks = disks.clone();
let disk_count = disks.len();
async move {
debug!(
event = EVENT_CAPACITY_REFRESH_SCHEDULED,
component = LOG_COMPONENT_CAPACITY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "started",
source = DataSource::Scheduled.as_metric_label(),
disk_count,
enqueue_latency_ms = start.elapsed().as_millis() as u64,
"capacity refresh scheduled"
);
let result = manager
.refresh_or_join(
DataSource::Scheduled,
move || async move { refresh_capacity_with_scope(disks, false).await },
)
.await;
scheduled_refresh_was_clean(&result)
}
run_scheduled_capacity_refresh(manager_for_refresh.clone(), disks.clone())
})
.await;
});
@@ -1526,6 +1543,39 @@ mod tests {
assert!(!scheduled_refresh_was_clean(&Err("scan failed".to_string())));
}
#[tokio::test]
async fn test_scheduled_capacity_refresh_skips_clean_cache_then_scans_dirty_disk() {
let temp_dir = tempfile::TempDir::new().expect("capacity test directory should be created");
std::fs::write(temp_dir.path().join("object.bin"), b"capacity-bytes").expect("capacity fixture should be written");
let disk = CapacityDiskRef {
endpoint: "node-a".to_string(),
drive_path: temp_dir.path().display().to_string(),
};
let manager = create_isolated_manager(HybridStrategyConfig::default());
manager
.update_capacity(CapacityUpdate::estimated(123, 1), DataSource::RealTime)
.await;
assert!(run_scheduled_capacity_refresh(manager.clone(), vec![disk.clone()]).await);
let cached = manager.get_capacity().await.expect("cached capacity should remain available");
assert_eq!(cached.total_used, 123);
assert_eq!(cached.source, DataSource::RealTime);
manager
.mark_dirty_scope(&CapacityScope {
disks: vec![CapacityScopeDisk {
endpoint: disk.endpoint.clone(),
drive_path: disk.drive_path.clone(),
}],
})
.await;
assert!(run_scheduled_capacity_refresh(manager.clone(), vec![disk]).await);
let cached = manager.get_capacity().await.expect("dirty refresh should update the cache");
assert_eq!(cached.source, DataSource::Scheduled);
assert!(manager.get_dirty_disks().await.is_empty());
}
#[tokio::test(start_paused = true)]
async fn test_scheduled_refresh_loop_applies_backoff_and_reset() {
use std::collections::VecDeque;
crates/object-capacity/src/scan.rs
+102 -26
View File
@@ -73,6 +73,15 @@ struct CapacityScanReport {
per_disk: Vec<DiskCapacityScanResult>,
}
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum ScheduledCapacityRefresh {
Idle,
Scan {
disks: Vec<CapacityDiskRef>,
dirty_subset: bool,
},
}
impl CapacityScanReport {
fn into_capacity_update(self, expected_disk_count: usize, replaces_disk_cache: bool) -> CapacityUpdate {
let mut update = if self.summary.is_estimated {
@@ -94,7 +103,13 @@ impl CapacityScanReport {
.collect();
// Skipped metadata or timeout fallback estimates can update an
// existing complete cache, but must not establish a new baseline.
if !self.summary.timed_out && !self.summary.metadata_incomplete {
if self.summary.timed_out {
// A timeout estimate is still the best bounded refresh for a
// disk too large to enumerate. Acknowledge dirty marks that
// predate this attempt so an idle disk does not loop forever;
// update_capacity preserves marks recorded during the scan.
update.clear_dirty_disks = update.per_disk.iter().map(|entry| entry.disk.clone()).collect();
} else if !self.summary.metadata_incomplete {
update.expected_disk_count = Some(expected_disk_count);
update.replaces_disk_cache = replaces_disk_cache;
update.clear_dirty_disks = update.per_disk.iter().map(|entry| entry.disk.clone()).collect();
@@ -329,23 +344,33 @@ pub(crate) async fn calculate_data_dir_used_capacity(
Ok(calculate_data_dir_used_capacity_report(disks).await?.summary)
}
pub async fn select_capacity_refresh_disks(
pub(crate) async fn select_scheduled_capacity_refresh(
capacity_manager: &HybridCapacityManager,
disks: &[CapacityDiskRef],
) -> (Vec<CapacityDiskRef>, bool) {
) -> ScheduledCapacityRefresh {
// The write side marks every disk of an EC set dirty, including remote
// peers, but only local disks are ever scanned and cleared — drop ghost
// entries so the dirty gauge reflects local pending work (backlog#1020).
let local_set: HashSet<CapacityScopeDisk> = disks.iter().map(disk_scope_key).collect();
capacity_manager.retain_dirty_disks_within(&local_set).await;
if !capacity_manager.can_refresh_dirty_subset().await {
return (disks.to_vec(), false);
}
let dirty_disks = capacity_manager.get_dirty_disks().await;
if dirty_disks.is_empty() {
return (disks.to_vec(), false);
return if disks.is_empty() || capacity_manager.get_capacity().await.is_some() {
ScheduledCapacityRefresh::Idle
} else {
ScheduledCapacityRefresh::Scan {
disks: disks.to_vec(),
dirty_subset: false,
}
};
}
if !capacity_manager.can_refresh_dirty_subset().await {
return ScheduledCapacityRefresh::Scan {
disks: disks.to_vec(),
dirty_subset: false,
};
}
let dirty_set: HashSet<CapacityScopeDisk> = dirty_disks.into_iter().collect();
@@ -356,9 +381,27 @@ pub async fn select_capacity_refresh_disks(
.collect();
if selected.is_empty() || selected.len() >= disks.len() {
(disks.to_vec(), false)
ScheduledCapacityRefresh::Scan {
disks: disks.to_vec(),
dirty_subset: false,
}
} else {
(selected, true)
ScheduledCapacityRefresh::Scan {
disks: selected,
dirty_subset: true,
}
}
}
pub async fn select_capacity_refresh_disks(
capacity_manager: &HybridCapacityManager,
disks: &[CapacityDiskRef],
) -> (Vec<CapacityDiskRef>, bool) {
match select_scheduled_capacity_refresh(capacity_manager, disks).await {
// Preserve the public selector's historical contract. The background
// scheduler consumes the richer internal plan and can remain idle.
ScheduledCapacityRefresh::Idle => (disks.to_vec(), false),
ScheduledCapacityRefresh::Scan { disks, dirty_subset } => (disks, dirty_subset),
}
}
@@ -1606,21 +1649,27 @@ mod tests {
#[test]
fn test_into_capacity_update_incomplete_results_do_not_replace_disk_cache() {
for scan in [
CapacityScanResult {
used_bytes: 100,
file_count: 10,
is_estimated: true,
metadata_incomplete: true,
..Default::default()
},
CapacityScanResult {
used_bytes: 100,
file_count: 10,
is_estimated: true,
timed_out: true,
..Default::default()
},
for (scan, clears_dirty) in [
(
CapacityScanResult {
used_bytes: 100,
file_count: 10,
is_estimated: true,
metadata_incomplete: true,
..Default::default()
},
false,
),
(
CapacityScanResult {
used_bytes: 100,
file_count: 10,
is_estimated: true,
timed_out: true,
..Default::default()
},
true,
),
] {
let disk = CapacityScopeDisk {
endpoint: "node-a".to_string(),
@@ -1641,7 +1690,11 @@ mod tests {
assert_eq!(update.per_disk[0].disk, disk);
assert_eq!(update.expected_disk_count, None);
assert!(!update.replaces_disk_cache);
assert!(update.clear_dirty_disks.is_empty());
if clears_dirty {
assert_eq!(update.clear_dirty_disks, vec![disk]);
} else {
assert!(update.clear_dirty_disks.is_empty());
}
}
}
@@ -1712,6 +1765,29 @@ mod tests {
assert_eq!(selected.len(), 2);
}
#[tokio::test]
async fn test_scheduled_capacity_refresh_returns_idle_with_cached_incomplete_baseline() {
let manager = create_isolated_manager(HybridStrategyConfig::default());
manager
.update_capacity(CapacityUpdate::estimated(100, 10), DataSource::Scheduled)
.await;
let disks = vec![CapacityDiskRef {
endpoint: "disk-1".to_string(),
drive_path: "/tmp/disk-1".to_string(),
}];
assert_eq!(
select_scheduled_capacity_refresh(manager.as_ref(), &disks).await,
ScheduledCapacityRefresh::Idle
);
let (selected, dirty_subset) = select_capacity_refresh_disks(manager.as_ref(), &disks).await;
assert_eq!(selected, disks);
assert!(!dirty_subset);
assert!(!manager.can_refresh_dirty_subset().await);
}
#[tokio::test]
async fn test_select_capacity_refresh_disks_returns_dirty_subset_when_cache_complete() {
let manager = create_isolated_manager(HybridStrategyConfig::default());