fix(scanner): defer usage publication during pool recovery (#6333)

* fix(scanner): defer usage publication during pool recovery

* fix(scanner): preserve metrics when publication is deferred

* fix(scanner): route test types through storage boundary

* fix(scanner): keep cache floor deferred during movement
This commit is contained in:
cxymds
2026-08-21 17:32:59 +08:00
committed by GitHub
parent cdfac5d7e3
commit adb90fc6e1
7 changed files with 622 additions and 83 deletions
+19
View File
@@ -49,6 +49,25 @@ impl ScannerIOCycle for ECStore {
) -> Result<ScannerCycleResult> {
let child_token = ctx.child_token();
// Check the local pool metadata before listing buckets. A failed or
// canceled decommission remains suspended after its worker exits, so
// starting a scan in that state could build a snapshot that cannot be
// routed to the authoritative metadata object.
if self.scanner_data_usage_publication_blocked().await {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "cycle_data_usage_route_blocked",
"Scanner cycle deferred while data usage metadata remains hidden by data movement"
);
return Ok(ScannerCycleResult::new(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
None,
));
}
let distributed = self.setup_is_dist_erasure().await;
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
ScannerActivityPreflight::Ready(snapshot) => snapshot,
+40 -1
View File
@@ -17,7 +17,9 @@ use super::io_disk::tier_stats_template;
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_folder::ScannerItem;
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
use crate::storage_api::owner::{
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
};
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
use crate::{
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
@@ -182,6 +184,39 @@ async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
}
#[tokio::test]
#[serial]
async fn scanner_cycle_is_deferred_while_terminal_decommission_is_blocked() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
for decommission in [
EcstorePoolDecommissionInfo {
failed: true,
..Default::default()
},
EcstorePoolDecommissionInfo {
canceled: true,
..Default::default()
},
] {
store.pool_meta.write().await.pools[0].decommission = Some(decommission);
assert!(store.scanner_data_usage_publication_blocked().await);
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
Duration::from_secs(30),
ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
)
.await
.expect("terminal-decommission-deferred scanner cycle should finish")
.expect("terminal-decommission-deferred scanner cycle should succeed");
assert_eq!(result.status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement));
assert!(receiver.recv().await.is_none(), "blocked cycle must not publish usage");
}
}
#[tokio::test]
async fn data_usage_publish_fails_when_receiver_is_closed() {
let (updates, receiver) = mpsc::channel(1);
@@ -236,6 +271,10 @@ async fn multi_pool_scanner_cycle_publishes_combined_usage() {
assert_eq!(bucket_usage.size, 11);
assert_eq!(usage.objects_total_count, 2);
assert_eq!(usage.objects_total_size, 11);
assert!(
receiver.recv().await.is_none(),
"a scanner cycle must publish at most one terminal usage snapshot"
);
}
#[tokio::test]