Compare commits

...

6 Commits

Author SHA1 Message Date
Hauser Zhi 212cb2a47e refactor(scanner): scope heal inventory to coordinator workers
Build the inventory only for coordinator-owned cross-drive healing and expose the regression marker directly through the scan storage domain.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 01:24:53 +08:00
houseme 3c0a0cf7f0 fix(scanner): use scan domain for test marker import
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 16:30:48 +00:00
Hauser Zhi 7f3ea0c73c perf(scanner): skip unused heal disk inventory
Avoid building and cloning the cross-drive inventory for single-drive scans and scans with healing suppressed. Exercise durable usage publication and cursor advancement through the runtime cycle.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 00:24:56 +08:00
houseme f9cafcb8a2 fix(scanner): route healing marker through storage boundary
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 15:58:58 +00:00
houseme 156defd1f1 fix(scanner): scan online single drives marked as healing
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 14:45:52 +00:00
houseme 4398882cc9 Initial plan 2026-11-19 14:37:06 +00:00
3 changed files with 110 additions and 7 deletions
+88
View File
@@ -1635,6 +1635,94 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
clear_dirty_usage_buckets_for_tests();
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
let bucket = "marked-drive-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("fixture bucket should be created");
let mut reader = PutObjReader::from_vec(vec![7; 16]);
store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
.expect("fixture object should persist");
wait_for_namespace_commit_tails(store.as_ref()).await;
let marker = temp_dir
.path()
.join("pool0/disk0")
.join(RUSTFS_META_BUCKET)
.join(HEALING_MARKER_PATH);
let marker_contents = b"leftover marker";
tokio::fs::write(&marker, marker_contents)
.await
.expect("fixture healing marker should persist");
let set = &store.pools[0].disk_set[0];
tokio::time::timeout(Duration::from_secs(10), async {
while !set.get_online_disks_with_healing(true).await.1 {
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.expect("disk info should observe the marker after its cache expires");
let ctx = CancellationToken::new();
let mut cycle_info = CurrentCycle {
next: 1,
..Default::default()
};
let mut revision = DataUsageCacheRevision::Missing;
let leader_epoch = 11;
let outcome = tokio::time::timeout(
Duration::from_secs(60),
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
)
.await
.expect("scanner cycle should finish");
assert!(
matches!(
outcome,
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
),
"unexpected cycle outcome: {outcome:?}",
);
let mut set_snapshot = DataUsageCache::default();
set_snapshot
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
.await
.expect("set usage snapshot should be readable");
assert!(set_snapshot.info.snapshot_complete);
assert_eq!(set_snapshot.info.next_cycle, 1);
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("global usage snapshot should be durable");
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
assert!(usage.usage_snapshot_complete);
assert!(usage.last_update.is_some());
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("scanner cycle cursor should be durable");
let (persisted_cycle, persisted_epoch) =
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
assert_eq!(persisted_epoch, leader_epoch);
assert_eq!(persisted_cycle.next, 2);
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
clear_dirty_usage_buckets_for_tests();
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn coordinator_publishes_authority_during_pending_put() {
+20 -7
View File
@@ -620,7 +620,10 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
let single_drive = self.set_drive_count == 1;
// A single-drive set has no alternate scan source. A heal marker
// still disables scanner heals, but must not exclude its only usage walk.
let (disks, healing) = self.get_online_disks_with_healing(single_drive).await;
#[cfg(test)]
let disks = super::tests::usage_remote_drain::walker_disks(&self, &buckets, disks);
if disks.is_empty() {
@@ -757,7 +760,18 @@ impl ScannerIOCache for SetDisks {
let _ = updates.send(incomplete_scope).await;
return Ok(ScannerSourceTurnReceipt::default());
}
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
// Coordinator workers need the full inventory only for cross-drive
// healing. Remote workers own their disk inventory on the peer.
let heal_disk_inventory = if !single_drive
&& !healing
&& workers
.iter()
.any(|(_, mode)| *mode == NamespaceScannerWorkerMode::Coordinator)
{
Some(Arc::new(scanner_set_disk_inventory(self.as_ref()).await))
} else {
None
};
if unsupported_remote_disks > 0 {
debug!(
target: "rustfs::scanner::io",
@@ -1010,7 +1024,6 @@ impl ScannerIOCache for SetDisks {
let store_clone_clone = self.clone();
let bucket_result_tx_clone = bucket_result_tx.clone();
let bucket_observations_clone = bucket_observations.clone();
let set_disk_inventory_clone = set_disk_inventory.clone();
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
@@ -1026,9 +1039,9 @@ impl ScannerIOCache for SetDisks {
let worker_retired_buckets = retired_buckets.clone();
#[cfg(feature = "e2e-test-hooks")]
let worker_incarnations = current_bucket_incarnations.clone();
let remote_capabilities = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
NamespaceScannerWorkerMode::Coordinator => None,
let (remote_capabilities, heal_disk_inventory) = match worker_mode {
NamespaceScannerWorkerMode::Coordinator => (None, heal_disk_inventory.clone()),
NamespaceScannerWorkerMode::RemoteV4(capabilities) => (Some(capabilities), None),
};
#[cfg(test)]
let worker_index = futs.len();
@@ -1671,7 +1684,7 @@ impl ScannerIOCache for SetDisks {
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
scan_ctx.clone(),
bucket_budget.clone(),
set_disk_inventory_clone.as_ref().clone(),
heal_disk_inventory.as_deref().cloned().unwrap_or_default(),
cache,
None,
ScannerDiskScanOptions {
+2
View File
@@ -422,6 +422,8 @@ pub(crate) mod scan {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence as ecstore_scanner_ack_fence;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::HEALING_MARKER_PATH;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::notification::new_global_notification_sys;
}