gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 16:37:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence system metadata publication (#6444)

Browse Source 
* feat(scanner): fence usage publication during data movement

* fix(scanner): detect movement refresh state changes

* fix(scanner): fence publication during data movement

* fix(scanner): close movement epoch publication races

* fix(scanner): fence movement-sensitive publication paths

* fix(scanner): fence cache and heal recovery paths

* fix(scanner): carry publication epoch through scan cycle

* fix(scanner): recheck remote cache epoch after save

* fix(scanner): recheck local cache epoch before publish

* fix(scanner): fence data usage writers and baseline

* fix(scanner): expose decommission activity to publication fence

* fix(scanner): release publication gate before reads

* fix(scanner): complete publication fence integration

* fix(scanner): avoid empty usage baseline publication

* chore(scanner): gate test-only helpers

* fix: use decommission canceler in reload test

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-08-23 17:28:21 +08:00
committed by GitHub
parent 9cda615519
commit e196a134cc
26 changed files with 2465 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_io/cache.rs
+20 -1
View File
@@ -395,6 +395,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache,
cache_cycle_floor: &AtomicU64,
expected_publication_epoch: u64,
) -> Option<SystemTime> {
let source = cache_snapshot.info.source?;
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
@@ -489,7 +490,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot
.save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions)
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
.await
{
error!(
@@ -519,6 +520,24 @@ pub(super) async fn persist_and_publish_cache_snapshot(
);
return None;
}
// The persisted-root fast path performs no PUT, so it also needs the
// cycle token re-admission before forwarding the root to the aggregate.
// This final check covers both the fast path and a successful save.
if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch)
.await
.is_none()
{
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "publication_epoch_changed_before_publish",
"Scanner cache root publish skipped after movement epoch change"
);
return None;
}
drop(guard);
let last_update = cache_snapshot.info.last_update;
crates/scanner/src/scanner_io/io_cache.rs
+79 -8
View File
@@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks {
all_buckets,
digest: scan_plan_digest,
leader_epoch,
publication_epoch,
dirty_usage_buckets,
bucket_failures,
pending_maintenance_work,
@@ -40,6 +41,12 @@ impl ScannerIOCache for SetDisks {
let set_label = self.set_index.to_string();
let source = DataUsageCacheSource::new(self.pool_index, self.set_index);
let expected_publication_epoch = match publication_epoch {
Some(epoch) => epoch,
None => scanner_publication_epoch(self.clone())
.await
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default();
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
warn!(
@@ -76,10 +83,16 @@ impl ScannerIOCache for SetDisks {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref())
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
return persist_and_publish_cache_snapshot(
self,
&updates,
cache,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
@@ -414,6 +427,7 @@ impl ScannerIOCache for SetDisks {
let pending_maintenance_work_clone = pending_maintenance_work.clone();
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
let cache_cycle_floor_clone = cache_cycle_floor.clone();
let expected_publication_epoch_clone = expected_publication_epoch;
let remote_server_epoch = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::Coordinator => None,
@@ -753,6 +767,23 @@ impl ScannerIOCache for SetDisks {
);
continue;
}
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_before_reuse",
"Current scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
@@ -901,7 +932,12 @@ impl ScannerIOCache for SetDisks {
{
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await
{
error!(
@@ -958,7 +994,12 @@ impl ScannerIOCache for SetDisks {
false
} else {
match partial_cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await
{
Ok(()) => true,
@@ -1029,7 +1070,12 @@ impl ScannerIOCache for SetDisks {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
&cache_name,
&revisions,
expected_publication_epoch_clone,
)
.await
{
done_save();
@@ -1064,6 +1110,24 @@ impl ScannerIOCache for SetDisks {
continue;
}
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch_clone)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_after_save",
"Scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
@@ -1159,7 +1223,14 @@ impl ScannerIOCache for SetDisks {
cache.info.lkg_scan_plan_digest = None;
cache.clone()
};
let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await;
let _ = persist_and_publish_cache_snapshot(
self.clone(),
&updates,
cache_snapshot,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await;
} else {
let mut incomplete_scope = cache_mutex.lock().await.clone();
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();
crates/scanner/src/scanner_io/io_cycle.rs
+21 -3
View File
@@ -68,6 +68,19 @@ impl ScannerIOCycle for ECStore {
));
}
// Capture one storage-owned movement epoch for the entire cycle. Set
// workers must not each observe a fresh epoch: a movement transition
// between sets would otherwise allow a mixed-generation aggregate.
let publication_epoch = match self.scanner_data_usage_publication_admission().await {
Some(admission) => Some(admission.epoch()),
None => {
return Ok(ScannerCycleResult::new(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
None,
));
}
};
let distributed = self.setup_is_dist_erasure().await;
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
ScannerActivityPreflight::Ready(snapshot) => snapshot,
@@ -131,7 +144,9 @@ impl ScannerIOCycle for ECStore {
if all_buckets.is_empty() {
reset_set_scan_gauges();
if !bucket_plan_complete {
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
return Ok(
ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)
);
}
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
@@ -155,7 +170,7 @@ impl ScannerIOCycle for ECStore {
)
.await?
{
return Ok(ScannerCycleResult::new(status, None));
return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
}
let dirty_usage_clear =
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
@@ -165,6 +180,7 @@ impl ScannerIOCycle for ECStore {
Vec::new()
};
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
}
@@ -180,7 +196,7 @@ impl ScannerIOCycle for ECStore {
"Scanner set state update detected missing disk sets"
);
reset_set_scan_gauges();
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
}
let set_scan_limit = scanner_budgeted_concurrency_limit(
@@ -250,6 +266,7 @@ impl ScannerIOCycle for ECStore {
all_buckets: Arc::clone(&all_buckets),
digest: scan_plan_digest,
leader_epoch,
publication_epoch,
dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(),
bucket_failures: bucket_failures.clone(),
pending_maintenance_work: pending_maintenance_work.clone(),
@@ -430,6 +447,7 @@ impl ScannerIOCycle for ECStore {
Vec::new()
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
.with_failed_dirty_usage(!failed_buckets.is_empty())
.with_pending_maintenance_work(pending_maintenance_work)
crates/scanner/src/scanner_io/tests.rs
+44
View File
@@ -145,6 +145,50 @@ async fn scanner_cache_locks_allow_cross_source_workers() {
assert!(!second.is_lock_lost());
}
#[tokio::test]
async fn scanner_set_cache_admission_tracks_owner_snapshot_and_fails_closed() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let set = store.pools[0].disk_set[0].clone();
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_none(),
"a set must not publish before the owner has refreshed its movement snapshot"
);
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_some(),
"an idle owner snapshot should admit the set cache"
);
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
pool_stats[0] = EcstoreRebalanceStats {
participating: true,
info: EcstoreRebalanceInfo {
start_time: Some(OffsetDateTime::now_utc()),
status: EcstoreRebalStatus::Started,
..Default::default()
},
..Default::default()
};
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
id: Uuid::new_v4().to_string(),
pool_stats,
..Default::default()
});
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_none(),
"active movement must keep set cache publication blocked"
);
*store.rebalance_meta.write().await = None;
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_some(),
"an idle owner refresh must make set cache publication live again"
);
}
#[tokio::test]
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;