From e196a134cc3e8d9947d20ab77f6fd77ae5621141 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 17:28:21 +0800 Subject: [PATCH] fix(scanner): fence system metadata publication (#6444) * feat(scanner): fence usage publication during data movement * fix(scanner): detect movement refresh state changes * fix(scanner): fence publication during data movement * fix(scanner): close movement epoch publication races * fix(scanner): fence movement-sensitive publication paths * fix(scanner): fence cache and heal recovery paths * fix(scanner): carry publication epoch through scan cycle * fix(scanner): recheck remote cache epoch after save * fix(scanner): recheck local cache epoch before publish * fix(scanner): fence data usage writers and baseline * fix(scanner): expose decommission activity to publication fence * fix(scanner): release publication gate before reads * fix(scanner): complete publication fence integration * fix(scanner): avoid empty usage baseline publication * chore(scanner): gate test-only helpers * fix: use decommission canceler in reload test --------- Co-authored-by: houseme --- crates/ecstore/src/core/pools.rs | 188 ++++++++-- crates/ecstore/src/data_usage/mod.rs | 290 +++++++++++++-- crates/ecstore/src/runtime/instance.rs | 84 ++++- .../ecstore/src/services/rebalance/control.rs | 87 ++++- .../ecstore/src/services/rebalance/runtime.rs | 134 ++++--- crates/ecstore/src/set_disk/mod.rs | 20 + crates/ecstore/src/store/bucket.rs | 2 +- crates/ecstore/src/store/init.rs | 5 + crates/ecstore/src/store/mod.rs | 154 +++++++- crates/ecstore/src/store/rebalance.rs | 16 +- crates/scanner/src/data_usage_define.rs | 26 +- .../src/data_usage_define/persistence.rs | 91 ++++- crates/scanner/src/data_usage_define/tests.rs | 48 +++ crates/scanner/src/lib.rs | 124 +++++++ crates/scanner/src/remote_scanner/stream.rs | 27 +- crates/scanner/src/scanner.rs | 247 +++++++++++-- crates/scanner/src/scanner/cycle_state.rs | 346 +++++++++++++++--- crates/scanner/src/scanner/heal_info.rs | 125 ++++++- crates/scanner/src/scanner/leadership.rs | 139 ++++++- crates/scanner/src/scanner/tests.rs | 213 +++++++++-- crates/scanner/src/scanner/usage_store.rs | 271 +++++++++++--- crates/scanner/src/scanner_io.rs | 22 +- crates/scanner/src/scanner_io/cache.rs | 21 +- crates/scanner/src/scanner_io/io_cache.rs | 87 ++++- crates/scanner/src/scanner_io/io_cycle.rs | 24 +- crates/scanner/src/scanner_io/tests.rs | 44 +++ 26 files changed, 2465 insertions(+), 370 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 16e4394a5..e4b8880e9 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -135,7 +135,7 @@ struct DecommissionOperation { } impl DecommissionCanceler { - fn new(token: CancellationToken) -> Self { + pub(crate) fn new(token: CancellationToken) -> Self { Self { operation: Arc::new(DecommissionOperation { token, @@ -153,7 +153,7 @@ impl DecommissionCanceler { &self.operation.token } - fn is_active(&self) -> bool { + pub(crate) fn is_active(&self) -> bool { self.operation.active.load(Ordering::Acquire) } @@ -1444,6 +1444,30 @@ fn should_replace_pool_status_for_status_refresh( !has_active_worker && persisted.last_update > current.last_update } +fn pool_decommission_movement_snapshot( + info: Option<&PoolDecommissionInfo>, +) -> (bool, bool, bool, bool, bool, Option) { + info.map(|info| { + ( + info.has_decommission_state(), + info.complete, + info.failed, + info.canceled, + info.queued, + info.start_time, + ) + }) + .unwrap_or_default() +} + +pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool { + before.pools.len() != after.pools.len() + || before.pools.iter().zip(after.pools.iter()).any(|(before, after)| { + pool_decommission_movement_snapshot(before.decommission.as_ref()) + != pool_decommission_movement_snapshot(after.decommission.as_ref()) + }) +} + /// Merges a persisted pool metadata snapshot into `current` monotonically: /// a pool entry is replaced only when no active worker covers it and the /// snapshot is strictly newer, so delayed snapshots never roll back local @@ -3358,6 +3382,8 @@ impl ECStore { .first() .cloned() .ok_or_else(|| Error::other("refresh_pool_status_meta: no pools available"))?; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let mut persisted = PoolMeta::default(); persisted.load(pool, self.pools.clone()).await?; @@ -3370,7 +3396,9 @@ impl ECStore { }; let mut pool_meta = self.pool_meta.write().await; - merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers); + if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) { + self.ctx.advance_data_movement_operation_epoch(); + } Ok(()) } @@ -3489,11 +3517,19 @@ impl ECStore { async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; let _start_guard = self.start_gate.lock().await; - let operation_gate = self.ctx.decommission_operation_gate(); - let operation_guard = operation_gate.write().await; - - // Lock order: decommission_cancelers before pool_meta. Holding both makes - // owner validation and the terminal transition one atomic operation. + // Signal cancellation before waiting for the movement writer. A worker + // may still hold a publication read guard while it observes this + // signal; waiting for the writer first would deadlock that handoff. + if let Some(owner) = owner { + owner.cancel(); + } else if let Some(canceler) = self.decommission_cancelers.read().await.get(idx).and_then(Option::as_ref) { + canceler.cancel(); + } + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + // Lock order: movement gate, then decommission_cancelers, then pool_meta. + // Holding both state locks makes owner validation and the terminal + // transition one atomic operation. let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut lock = self.pool_meta.write().await; @@ -3562,9 +3598,26 @@ impl ECStore { self.release_decommission_canceler_slot(idx, canceler).await; } + if should_save_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("decommission_cancel for pool {idx}"); - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission cancel saved locally but pool meta reload failed" + ); + } } Ok(()) @@ -3589,7 +3642,11 @@ impl ECStore { .unwrap_or((false, false, false, false)); ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?; } - self.cancel_decommission_routines_and_wait(&[idx]).await; + // Cancel workers before waiting for the movement writer so active + // object operations can observe the signal and release read guards. + self.cancel_decommission_routines(&[idx]).await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let (should_reload_pool_meta, previous_pool_meta) = { let mut pool_meta = self.pool_meta.write().await; @@ -3606,17 +3663,37 @@ impl ECStore { return Err(err); } + if should_reload_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("clear_decommission for pool {idx}"); - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission clear saved locally but pool meta reload failed" + ); + } } Ok(()) } async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result { + // Serialize promotion and generation capture with clear/restart transitions. let (changed, generation, save_error) = { let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); @@ -3644,6 +3721,9 @@ impl ECStore { return Err(err); } + if changed { + self.ctx.advance_data_movement_operation_epoch(); + } if changed && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("promote_queued_decommission for pool {idx}"); if let Err(err) = @@ -3668,6 +3748,8 @@ impl ECStore { } async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> { + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let changed = { let mut pool_meta = self.pool_meta.write().await; pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())? @@ -3708,19 +3790,20 @@ impl ECStore { is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx)) } + #[cfg(test)] async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) { + self.cancel_decommission_routines(indices).await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + } + + async fn cancel_decommission_routines(&self, indices: &[usize]) { { let mut cancelers = self.decommission_cancelers.write().await; for idx in indices { take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx); } } - self.wait_for_decommission_side_effects().await; - } - - async fn wait_for_decommission_side_effects(&self) { - let operation_gate = self.ctx.decommission_operation_gate(); - let _operation_guard = operation_gate.write().await; } async fn reserve_decommission_routines( @@ -4218,7 +4301,7 @@ impl ECStore { } decommission_cancel_signal_result(rx.is_cancelled())?; self.ensure_decommission_generation_current(idx, generation).await?; - let operation_gate = self.ctx.decommission_operation_gate(); + let operation_gate = self.ctx.data_movement_operation_gate(); let bucket_incarnation_fence = match expected_bucket_incarnation_id { Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?), @@ -5113,9 +5196,12 @@ impl ECStore { { ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?; let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; - // Lock order: decommission_cancelers before pool_meta. Holding both makes - // owner validation and the terminal transition one atomic operation. + // Lock order: movement gate, then decommission_cancelers, then pool_meta. + // Holding both state locks makes owner validation and the terminal + // transition one atomic operation. let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; @@ -5151,6 +5237,12 @@ impl ECStore { if let Some(canceler) = terminal_canceler.as_ref() { self.release_decommission_canceler_slot(idx, canceler).await; } + + if should_reload_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("decommission_failed for pool {idx}"); if let Some(err) = observe_decommission_terminal_reload_result( @@ -5208,7 +5300,12 @@ impl ECStore { } self.verify_decommission_durable_ilm_receipts(idx).await?; let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + // Lock order: movement gate, then decommission_cancelers, then pool_meta. + // Holding both state locks makes owner validation and the terminal + // transition one atomic operation. let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; @@ -5249,6 +5346,12 @@ impl ECStore { if let Some(canceler) = terminal_canceler.as_ref() { self.release_decommission_canceler_slot(idx, canceler).await; } + + if should_reload_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("complete_decommission for pool {idx}"); if let Some(err) = observe_decommission_terminal_reload_result( @@ -5479,11 +5582,16 @@ impl ECStore { self.ensure_decommission_rebalance_idle_after_refresh().await?; let all_space_infos = self.get_decommission_all_pool_space_infos().await?; - self.cancel_decommission_routines_and_wait(&indices).await; + // Signal cancellation before waiting for the movement writer so active + // object operations can observe the signal and release read guards. + self.cancel_decommission_routines(&indices).await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let index_cancelers = if let Some((rx, local_indices)) = reservation { - // Lock order matches terminal transitions: decommission_cancelers - // before pool_meta while start_gate excludes another start. + // Lock order matches terminal transitions: movement gate, then + // decommission_cancelers, then pool_meta while start_gate excludes + // another start. let mut cancelers = self.decommission_cancelers.write().await; let pool_meta = self.pool_meta.read().await; ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?; @@ -5505,6 +5613,10 @@ impl ECStore { let previous_pool_meta = self .save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets) .await?; + self.ctx.advance_data_movement_operation_epoch(); + // The local durable transition is now fenced. Release the writer + // before any peer RPC; remote reload must not block scanner admission. + drop(_movement_guard); if let Some(notification_sys) = runtime_sources::notification_sys() && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await) @@ -5519,11 +5631,20 @@ impl ECStore { "Decommission start failed after pool metadata save" ); - { - let mut pool_meta = self.pool_meta.write().await; - rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone()); - } - if let Err(rollback_save_err) = self.save_current_pool_meta().await { + let rollback_result = { + let movement_guard = movement_gate.write().await; + { + let mut pool_meta = self.pool_meta.write().await; + rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone()); + } + let rollback_result = self.save_current_pool_meta().await; + if rollback_result.is_ok() { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(movement_guard); + rollback_result + }; + if let Err(rollback_save_err) = rollback_result { error!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -6527,7 +6648,7 @@ impl ECStore { generation: OffsetDateTime, ) -> Result<()> { self.ensure_decommission_generation_current(idx, generation).await?; - let operation_gate = self.ctx.decommission_operation_gate(); + let operation_gate = self.ctx.data_movement_operation_gate(); run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await } @@ -8077,7 +8198,7 @@ mod pools_tests { ..Default::default() }; - merge_pool_status_refresh(&mut current, persisted, &[false]); + assert!(merge_pool_status_refresh(&mut current, persisted, &[false])); let info = current.pools[0] .decommission @@ -8121,7 +8242,7 @@ mod pools_tests { ..Default::default() }; - merge_pool_status_refresh(&mut current, persisted, &[true]); + assert!(!merge_pool_status_refresh(&mut current, persisted, &[true])); let info = current.pools[0] .decommission @@ -8162,7 +8283,7 @@ mod pools_tests { ..Default::default() }; - merge_pool_status_refresh(&mut current, persisted, &[true]); + assert!(!merge_pool_status_refresh(&mut current, persisted, &[true])); let info = current.pools[0] .decommission @@ -8591,7 +8712,7 @@ mod pools_tests { #[tokio::test] async fn test_decommission_transition_waits_without_registered_canceler() { let store = decommission_worker_test_store(PoolMeta::default(), vec![None]); - let operation_gate = store.ctx.decommission_operation_gate(); + let operation_gate = store.ctx.data_movement_operation_gate(); let operation_guard = operation_gate.read().await; let transition = tokio::spawn({ let store = store.clone(); @@ -11338,6 +11459,7 @@ mod pools_tests { assert!(store.decommission_cancelers.read().await[0].is_none()); assert!(!canceler.is_active()); assert!(canceler.is_cancelled()); + assert_eq!(store.ctx.data_movement_operation_epoch(), 1); } #[test] diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 12cf3117b..0a613e02f 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -388,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: "nonconverged data usage observations cannot replace the quota-authoritative snapshot", )); } + let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else { + return Err(Error::other("data usage publication is blocked by data movement")); + }; // Prevent older data from overwriting newer persisted stats - if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await + let existing_snapshot = load_data_usage_snapshot(store.clone()).await; + if let Ok((existing, source)) = existing_snapshot && source.is_authoritative() && let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now()) { @@ -400,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: return Ok(()); } - save_data_usage_in_backend(data_usage_info, store).await + save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await } -async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc) -> Result<(), Error> { +async fn save_data_usage_in_backend( + data_usage_info: DataUsageInfo, + store: Arc, + expected_publication_epoch: u64, +) -> Result<(), Error> { let data = serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?; // Save to backend using the same mechanism as original code + let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else { + return Err(Error::other("data usage publication is blocked by data movement")); + }; + if publication_epoch != expected_publication_epoch { + return Err(Error::other("data usage publication epoch changed before save")); + } crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data) .await .map_err(Error::other)?; + drop(publication_guard); - cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await; + cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref())) + .await; // Invalidate the cached snapshot so readers observe the new save on their // next request instead of waiting out the remaining TTL. The next cached @@ -449,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore { } } -async fn cleanup_observed_data_usage_after_authoritative_save(store: &S, authoritative: &DataUsageInfo) -where +async fn cleanup_observed_data_usage_after_authoritative_save_with_publication( + store: &S, + authoritative: &DataUsageInfo, + publication_store: Option<&ECStore>, +) where S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized, { - let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { + let observed_read_epoch = match publication_store { + Some(publication_store) => { + let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else { + return; + }; + Some(epoch) + } + None => None, + }; + let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await; + let (observed, revision) = match observed_snapshot { Ok(Some(snapshot)) => snapshot, Ok(None) => return, Err(err) => { @@ -469,6 +498,19 @@ where return; } + let publication_guard = match publication_store { + Some(publication_store) => { + let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await + else { + return; + }; + if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) { + return; + } + Some(guard) + } + None => None, + }; match store.delete_observed_data_usage_snapshot(&revision).await { Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {} Err(err) => { @@ -479,6 +521,15 @@ where ); } } + drop(publication_guard); +} + +#[cfg(test)] +async fn cleanup_observed_data_usage_after_authoritative_save(store: &S, authoritative: &DataUsageInfo) +where + S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized, +{ + cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await; } fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) { @@ -519,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc, bucket: &str) pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> { prepare_bucket_usage_for_namespace_change(bucket, None).await?; - remove_bucket_usage_from_backend_with_guard(store, bucket, None).await + remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await } pub(crate) async fn prepare_bucket_usage_for_namespace_change( @@ -542,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change( Ok(()) } +#[cfg(test)] pub(crate) async fn remove_bucket_usage_from_backend_with_guard( store: &S, bucket: &str, @@ -551,6 +603,24 @@ where S: EcstoreObjectIO + ?Sized, { let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await; + invalidate_bucket_usage_snapshot_caches(guard, bucket).await?; + result +} + +pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced( + store: &ECStore, + bucket: &str, + guard: Option<&rustfs_lock::NamespaceLockGuard>, +) -> Result<(), Error> { + let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await; + invalidate_bucket_usage_snapshot_caches(guard, bucket).await?; + result +} + +async fn invalidate_bucket_usage_snapshot_caches( + guard: Option<&rustfs_lock::NamespaceLockGuard>, + bucket: &str, +) -> Result<(), Error> { let mut snapshot_cache = data_usage_snapshot_cache().write().await; ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?; clear_data_usage_snapshot_cache(&mut snapshot_cache); @@ -558,7 +628,7 @@ where let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await; ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?; clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache); - result + Ok(()) } async fn load_data_usage_for_bucket_removal(store: &S, object: &str) -> Result, Error> @@ -617,48 +687,83 @@ fn ensure_bucket_namespace_guard( Ok(()) } +#[cfg(test)] async fn remove_bucket_usage_from_backend_with_store_and_guard( store: &S, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>, ) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await +} + +async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication( + store: &S, + bucket: &str, + guard: Option<&rustfs_lock::NamespaceLockGuard>, + publication_store: Option<&ECStore>, +) -> Result<(), Error> where S: EcstoreObjectIO + ?Sized, { ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?; + let primary_seed_epoch = match publication_store { + Some(publication_store) => Some( + publication_store + .scanner_data_usage_publication_epoch() + .await + .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?, + ), + None => None, + }; let primary_seed = load_data_usage_seed_for_missing_primary(store).await?; - remove_bucket_usage_from_object_with_retries( + remove_bucket_usage_from_object_with_retries_and_publication( store, DATA_USAGE_OBJ_NAME_PATH.as_str(), bucket, DATA_USAGE_REMOVE_CAS_RETRIES, - Some(&primary_seed), + primary_seed.as_ref(), guard, + publication_store.map(|store| (store, primary_seed_epoch)), ) .await?; ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?; + let backup_seed_epoch = match publication_store { + Some(publication_store) => Some( + publication_store + .scanner_data_usage_publication_epoch() + .await + .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?, + ), + None => None, + }; let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) .await? - .map_or(primary_seed, |(data_usage_info, _)| data_usage_info); - remove_bucket_usage_from_object_with_retries( + .map(|(data_usage_info, _)| data_usage_info) + .or_else(|| primary_seed.clone()); + remove_bucket_usage_from_object_with_retries_and_publication( store, DATA_USAGE_OBJ_BACKUP_PATH.as_str(), bucket, DATA_USAGE_REMOVE_CAS_RETRIES, - Some(&backup_seed), + backup_seed.as_ref(), guard, + publication_store.map(|store| (store, backup_seed_epoch)), ) .await?; ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?; - if let Err(err) = remove_bucket_usage_from_object_with_retries( + if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication( store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard, + publication_store.map(|store| (store, None)), ) .await { @@ -672,12 +777,21 @@ where LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), ] { - remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?; + remove_bucket_usage_from_object_with_retries_and_publication( + store, + object, + bucket, + DATA_USAGE_REMOVE_CAS_RETRIES, + None, + guard, + publication_store.map(|store| (store, None)), + ) + .await?; } Ok(()) } -async fn load_data_usage_seed_for_missing_primary(store: &S) -> Result +async fn load_data_usage_seed_for_missing_primary(store: &S) -> Result, Error> where S: EcstoreObjectIO + ?Sized, { @@ -690,12 +804,13 @@ where if !authoritative { data_usage_info.usage_snapshot_complete = false; } - return Ok(data_usage_info); + return Ok(Some(data_usage_info)); } } - Ok(DataUsageInfo::default()) + Ok(None) } +#[cfg(test)] async fn remove_bucket_usage_from_object_with_retries( store: &S, object: &str, @@ -704,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries( missing_seed: Option<&DataUsageInfo>, guard: Option<&rustfs_lock::NamespaceLockGuard>, ) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None) + .await +} + +async fn remove_bucket_usage_from_object_with_retries_and_publication( + store: &S, + object: &str, + bucket: &str, + cas_retries: usize, + missing_seed: Option<&DataUsageInfo>, + guard: Option<&rustfs_lock::NamespaceLockGuard>, + publication: Option<(&ECStore, Option)>, +) -> Result<(), Error> where S: EcstoreObjectIO + ?Sized, { for attempt in 0..=cas_retries { ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?; - let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? { + let read_epoch = match publication { + Some((publication_store, expected_publication_epoch)) => { + let epoch = publication_store + .scanner_data_usage_publication_epoch() + .await + .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?; + if expected_publication_epoch.is_some_and(|expected| expected != epoch) { + return Err(Error::other("data usage publication epoch changed before snapshot read")); + } + Some(epoch) + } + None => None, + }; + let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?; + let (mut data_usage_info, revision) = match loaded_snapshot { Some((data_usage_info, revision)) => (data_usage_info, Some(revision)), None => match missing_seed { Some(data_usage_info) => (data_usage_info.clone(), None), @@ -733,6 +878,22 @@ where }, }; ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?; + let publication_guard = match publication { + Some((publication_store, expected_publication_epoch)) => { + let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await + else { + return Err(Error::other("data usage publication is blocked by data movement")); + }; + if expected_publication_epoch + .or(read_epoch) + .is_some_and(|expected| expected != publication_epoch) + { + return Err(Error::other("data usage publication epoch changed before snapshot commit")); + } + Some(guard) + } + None => None, + }; let save_result = store .put_object( RUSTFS_META_BUCKET, @@ -745,6 +906,7 @@ where }, ) .await; + drop(publication_guard); match save_result { Ok(_) => return Ok(()), Err(err) => { @@ -2286,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc) { #[cfg(test)] mod tests { use super::*; + use crate::layout::endpoints::EndpointServerPools; + use crate::runtime::instance::InstanceContext; use crate::storage_api_contracts::object::ObjectIO as _; use rustfs_data_usage::BucketUsageInfo; use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey}; @@ -2308,6 +2472,7 @@ mod tests { error_after_commit_put: Option, advance_time_on_put: Option, advance_time_after_get: Option<(UsageObjectSlot, Duration)>, + advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc)>, advance_time_before_put: Option<(usize, Duration)>, advance_time_after_put: Option<(usize, Duration)>, put_count: usize, @@ -2385,10 +2550,21 @@ mod tests { } _ => None, }; + let advance_publication_epoch = match state.advance_publication_epoch_after_get { + Some((expected_slot, ref ctx)) if expected_slot == slot => { + let ctx = Arc::clone(ctx); + state.advance_publication_epoch_after_get = None; + Some(ctx) + } + _ => None, + }; drop(state); if let Some(duration) = advance { tokio::time::advance(duration).await; } + if let Some(ctx) = advance_publication_epoch { + ctx.advance_data_movement_operation_epoch(); + } Ok(crate::object_api::GetObjectReader { stream: Box::new(Cursor::new(data)), object_info: ObjectInfo { @@ -2589,6 +2765,23 @@ mod tests { .to_string() } + fn build_publication_store(ctx: Arc) -> Arc { + let endpoint_pools = EndpointServerPools::default(); + Arc::new(ECStore { + id: uuid::Uuid::new_v4(), + disk_map: HashMap::new(), + pools: Vec::new(), + peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()), + pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()), + rebalance_meta: RwLock::new(None), + decommission_cancelers: RwLock::new(Vec::new()), + start_gate: TokioMutex::new(()), + pool_meta_save_gate: TokioMutex::new(()), + ctx, + bucket_fence_registry: Arc::default(), + }) + } + #[test] fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() { // `to_object_err` rewrites the raw storage variants before they reach @@ -4333,7 +4526,15 @@ mod tests { .expect("namespace lock acquisition should not fail") .expect("namespace lock should be acquired"), ); - let store = Arc::new(UsageCasStore::default()); + let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now()); + let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode"); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((encoded.clone(), 1)), + backup_object: Some((encoded, 1)), + ..Default::default() + }), + }); let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now()); let mut snapshot_cache = data_usage_snapshot_cache().write().await; *snapshot_cache = Some(CachedDataUsageSnapshot { @@ -4432,21 +4633,17 @@ mod tests { } #[tokio::test] - async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() { + async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() { let store = Arc::new(UsageCasStore::default()); remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") .await - .expect("bucket removal should create both usage fences"); + .expect("bucket removal should remain a no-op without a usage baseline"); let state = store.state.lock().await; - assert_eq!(state.put_count, 2); - for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() { - let saved = serde_json::from_slice::(data).expect("saved usage snapshot should decode"); - assert_eq!(*revision, 1); - assert!(saved.last_update.is_some()); - assert!(!data_usage_contains_bucket(&saved, "bucket-a")); - } + assert_eq!(state.put_count, 0); + assert!(state.object.is_none()); + assert!(state.backup_object.is_none()); } #[tokio::test] @@ -4598,6 +4795,39 @@ mod tests { assert_eq!(backup_err, Error::PreconditionFailed); } + #[tokio::test] + async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() { + let ctx = Arc::new(InstanceContext::new()); + let publication_store = build_publication_store(ctx.clone()); + let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)), + advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)), + ..Default::default() + }), + }); + let expected_epoch = publication_store + .scanner_data_usage_publication_epoch() + .await + .expect("idle publication store should admit the initial read"); + + let err = remove_bucket_usage_from_object_with_retries_and_publication( + store.as_ref(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + "bucket-a", + 0, + None, + None, + Some((publication_store.as_ref(), Some(expected_epoch))), + ) + .await + .expect_err("a movement epoch flip during the read must fence the commit"); + + assert!(err.to_string().contains("epoch changed")); + assert_eq!(store.state.lock().await.put_count, 0); + } + #[tokio::test] async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() { let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 9453ed02b..fedf2d7fb 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr; use rustfs_lock::{GlobalLockManager, get_global_lock_manager}; use s3s::region::Region; use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, OnceLock}; +use std::sync::{ + Arc, OnceLock, + atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, +}; use tokio::sync::{OnceCell, RwLock}; use tokio_util::sync::CancellationToken; use uuid::Uuid; +const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0; +const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1; +const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2; + /// Runtime state owned by a single `ECStore` instance. /// /// This is intentionally minimal in the first migration slice; subsequent @@ -160,10 +167,22 @@ pub struct InstanceContext { /// workers (scanner/heal/tier/lifecycle) without touching another instance. /// Replaces the process-global cancel-token static. background_cancel_token: OnceLock, - /// Serializes decommission data-movement operations with cancellation and - /// a subsequent restart. Readers are held across one object side effect; - /// the transition path takes the writer after cancelling the routine. - decommission_operation_gate: Arc>, + /// Serializes data-movement transitions with scanner publication commits. + /// Readers are held across one publication commit; movement transitions + /// take the writer at their durable state commit boundary. + data_movement_operation_gate: Arc>, + /// Monotonic admission epoch paired with the operation gate. A + /// publication admitted before a movement transition must never be + /// mistaken for one admitted after the transition. + data_movement_operation_epoch: AtomicU64, + /// Once the admission epoch reaches its reserved terminal value, no new + /// publication may be admitted. Keeping this state separate from the + /// saturating counter prevents an unchanged `u64::MAX` value from being + /// mistaken for a fresh epoch after overflow. + data_movement_operation_epoch_exhausted: AtomicBool, + /// Last storage-owned movement snapshot observed under the operation + /// gate. SetDisks cache writers fail closed until ECStore refreshes it. + scanner_publication_state: AtomicU8, /// Resolves object-encryption material at the application boundary. object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, @@ -204,7 +223,10 @@ impl InstanceContext { local_disk_set_drives: Arc::new(RwLock::new(Vec::new())), bucket_metadata_sys: std::sync::Mutex::new(None), background_cancel_token: OnceLock::new(), - decommission_operation_gate: Arc::new(RwLock::new(())), + data_movement_operation_gate: Arc::new(RwLock::new(())), + data_movement_operation_epoch: AtomicU64::new(0), + data_movement_operation_epoch_exhausted: AtomicBool::new(false), + scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN), object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), @@ -223,8 +245,54 @@ impl InstanceContext { self.lock_manager.clone() } - pub(crate) fn decommission_operation_gate(&self) -> Arc> { - Arc::clone(&self.decommission_operation_gate) + pub(crate) fn data_movement_operation_gate(&self) -> Arc> { + Arc::clone(&self.data_movement_operation_gate) + } + + pub(crate) fn data_movement_operation_epoch(&self) -> u64 { + self.data_movement_operation_epoch.load(Ordering::Acquire) + } + + pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool { + self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire) + } + + pub(crate) fn scanner_publication_state_allowed(&self) -> bool { + !self.data_movement_operation_epoch_exhausted() + && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED + } + + pub(crate) fn set_scanner_publication_state(&self, blocked: bool) { + self.scanner_publication_state.store( + if blocked { + SCANNER_PUBLICATION_STATE_BLOCKED + } else { + SCANNER_PUBLICATION_STATE_ALLOWED + }, + Ordering::Release, + ); + } + + pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 { + self.scanner_publication_state + .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release); + let _ = self + .data_movement_operation_epoch + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1))); + let result = self.data_movement_operation_epoch.load(Ordering::Acquire); + if result == u64::MAX { + self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release); + } + result + } + + #[cfg(test)] + pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) { + self.data_movement_operation_epoch.store(epoch, Ordering::Release); + self.data_movement_operation_epoch_exhausted + .store(epoch == u64::MAX, Ordering::Release); + self.scanner_publication_state + .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release); } /// Install the application-owned object-encryption resolver once. diff --git a/crates/ecstore/src/services/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs index 09e1993e7..5e13a1933 100644 --- a/crates/ecstore/src/services/rebalance/control.rs +++ b/crates/ecstore/src/services/rebalance/control.rs @@ -48,12 +48,12 @@ fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usi .unwrap_or_default() } -fn merge_rebalance_status_refresh(current: &mut Option, persisted: RebalanceMeta) { +fn merge_rebalance_status_refresh(current: &mut Option, persisted: RebalanceMeta) -> bool { if persisted.id.is_empty() && persisted.pool_stats.is_empty() { - clear_rebalance_status_refresh(current); - return; + return clear_rebalance_status_refresh(current); } + let before = current.clone(); match current.as_mut() { Some(current_meta) => { if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict @@ -66,14 +66,41 @@ fn merge_rebalance_status_refresh(current: &mut Option, persisted *current = Some(persisted); } } + + match (before.as_ref(), current.as_ref()) { + (None, None) => false, + (None, Some(_)) | (Some(_), None) => true, + (Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after), + } } -fn clear_rebalance_status_refresh(current: &mut Option) { +fn clear_rebalance_status_refresh(current: &mut Option) -> bool { if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) { - *current = None; + current.take().is_some() + } else { + false } } +fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool { + let Some(current) = current else { + return true; + }; + + current.id != persisted.id + || current.stopped_at != persisted.stopped_at + || current.pool_stats.len() != persisted.pool_stats.len() + || current + .pool_stats + .iter() + .zip(persisted.pool_stats.iter()) + .any(|(current, persisted)| { + current.participating != persisted.participating + || current.info.status != persisted.info.status + || current.info.stopping != persisted.info.stopping + }) +} + impl ECStore { pub(super) async fn save_rebalance_meta_with_merge( &self, @@ -121,7 +148,10 @@ impl ECStore { "Loading rebalance metadata" ); let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; if resolve_rebalance_meta_load_result(meta.load(pool).await)? { + let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta); { let mut rebalance_meta = self.rebalance_meta.write().await; @@ -130,6 +160,10 @@ impl ECStore { drop(rebalance_meta); } + if movement_changed { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?; debug!( event = EVENT_REBALANCE_STATE, @@ -139,10 +173,15 @@ impl ECStore { "Loaded rebalance metadata" ); } else { + let movement_changed = self.rebalance_meta.read().await.is_some(); { let mut rebalance_meta = self.rebalance_meta.write().await; *rebalance_meta = None; } + if movement_changed { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); debug!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, @@ -160,14 +199,20 @@ impl ECStore { pub async fn refresh_rebalance_status_meta(&self) -> Result<()> { let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?; let mut persisted = RebalanceMeta::new(); + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; match persisted.load(pool).await { Ok(()) => { let mut rebalance_meta = self.rebalance_meta.write().await; - merge_rebalance_status_refresh(&mut rebalance_meta, persisted); + if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) { + self.ctx.advance_data_movement_operation_epoch(); + } } Err(Error::ConfigNotFound) => { let mut rebalance_meta = self.rebalance_meta.write().await; - clear_rebalance_status_refresh(&mut rebalance_meta); + if clear_rebalance_status_refresh(&mut rebalance_meta) { + self.ctx.advance_data_movement_operation_epoch(); + } } Err(err) => { return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}"))); @@ -349,6 +394,8 @@ impl ECStore { #[tracing::instrument(skip(self, bucktes))] pub async fn init_rebalance_start(self: &Arc, bucktes: Vec) -> Result { let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let decommission_running = self.is_decommission_running().await; { @@ -356,12 +403,16 @@ impl ECStore { validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?; } - self.init_rebalance_meta(bucktes).await + let id = self.init_rebalance_meta(bucktes).await?; + self.ctx.advance_data_movement_operation_epoch(); + Ok(id) } #[tracing::instrument(skip(self))] pub async fn start_rebalance_for_id(self: &Arc, expected_id: &str) -> Result<()> { let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; { let rebalance_meta = self.rebalance_meta.read().await; @@ -379,7 +430,10 @@ impl ECStore { } } - self.start_rebalance().await + if self.start_rebalance_inner().await? { + self.ctx.advance_data_movement_operation_epoch(); + } + Ok(()) } pub async fn rollback_rebalance_start_for_id(self: &Arc, expected_id: Option<&str>, start_error: String) -> Result<()> { @@ -548,6 +602,9 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn stop_rebalance_for_id(self: &Arc, expected_id: Option<&str>) -> Result<()> { + let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id) @@ -560,6 +617,7 @@ impl ECStore { .await, "stop_rebalance", )?; + self.ctx.advance_data_movement_operation_epoch(); } Ok(()) @@ -570,6 +628,9 @@ impl ECStore { expected_id: Option<&str>, start_error: String, ) -> Result<()> { + let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; rollback_rebalance_start_meta_snapshot_for_id( @@ -587,6 +648,7 @@ impl ECStore { .await, "rollback_rebalance_start", )?; + self.ctx.advance_data_movement_operation_epoch(); } Ok(()) @@ -597,6 +659,8 @@ impl ECStore { return Ok(()); } + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let encoded_error = encode_rebalance_stop_propagation_record(&record); let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; @@ -610,6 +674,7 @@ impl ECStore { .await, "record_rebalance_stop_propagation", )?; + self.ctx.advance_data_movement_operation_epoch(); } Ok(()) @@ -682,7 +747,7 @@ mod tests { ..Default::default() }; - merge_rebalance_status_refresh(&mut current, persisted); + assert!(merge_rebalance_status_refresh(&mut current, persisted)); let refreshed = current.as_ref().expect("refresh should keep rebalance metadata"); assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed); @@ -721,7 +786,7 @@ mod tests { ..Default::default() }; - merge_rebalance_status_refresh(&mut current, persisted); + assert!(!merge_rebalance_status_refresh(&mut current, persisted)); assert!( current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(), diff --git a/crates/ecstore/src/services/rebalance/runtime.rs b/crates/ecstore/src/services/rebalance/runtime.rs index 906e92922..83a2a87c3 100644 --- a/crates/ecstore/src/services/rebalance/runtime.rs +++ b/crates/ecstore/src/services/rebalance/runtime.rs @@ -42,6 +42,15 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap) -> Result<()> { + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + if self.start_rebalance_inner().await? { + self.ctx.advance_data_movement_operation_epoch(); + } + Ok(()) + } + + pub(super) async fn start_rebalance_inner(self: &Arc) -> Result { info!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, @@ -55,6 +64,7 @@ impl ECStore { let cancel_tx = CancellationToken::new(); let rx = cancel_tx.clone(); let mut meta_to_save = None; + let mut movement_changed = false; { let mut rebalance_meta = self.rebalance_meta.write().await; @@ -72,14 +82,16 @@ impl ECStore { reason = "already_in_progress", "Skipped duplicate rebalance start" ); - return Ok(()); + return Ok(false); } let now = OffsetDateTime::now_utc(); if complete_rebalance_pools_at_goal(meta, now) { meta_to_save = Some(meta.clone()); + movement_changed = true; } if complete_rebalance_pools_with_empty_queue(meta, now) { meta_to_save = Some(meta.clone()); + movement_changed = true; } meta.cancel = Some(cancel_tx); @@ -118,7 +130,7 @@ impl ECStore { reason = "no_participants", "Skipped rebalance start because no pools are participating" ); - return Ok(()); + return Ok(movement_changed); } let mut workers_started = 0usize; @@ -186,7 +198,7 @@ impl ECStore { reason = "no_local_participants", "Skipped rebalance start because no local pools are participating" ); - return Ok(()); + return Ok(movement_changed); } info!( @@ -197,7 +209,7 @@ impl ECStore { worker_count = workers_started, "Rebalance started" ); - Ok(()) + Ok(true) } #[tracing::instrument(skip(self, rx))] @@ -214,53 +226,77 @@ impl ECStore { let mut quit = false; loop { + let mut terminal_state_saved = false; tokio::select! { result = done_rx.recv() => { quit = true; let now = OffsetDateTime::now_utc(); let terminal_event = classify_rebalance_terminal_event(result, now); msg = terminal_event.message().to_string(); - let mut rebalance_meta = store.rebalance_meta.write().await; - if let Some(meta) = rebalance_meta.as_mut() { - let meta_stopped = meta.stopped_at.is_some(); - if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { - if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. }) - && has_rebalance_cleanup_warnings(pool_stat) - { - pool_stat.info.stopping = false; - pool_stat.info.status = RebalStatus::Failed; - pool_stat.info.end_time = Some(now); - pool_stat.info.last_error = Some( - pool_stat - .cleanup_warnings - .last_message - .clone() - .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()), - ); - } else if should_preserve_rebalance_stopped_state( - meta_stopped, - pool_stat.info.status, - &terminal_event, - ) { - debug!( - event = EVENT_REBALANCE_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REBALANCE, - pool_index, - state = "stopped_preserved", - "Preserved stopped rebalance status" - ); + let movement_gate = store.ctx.data_movement_operation_gate(); + let movement_guard = movement_gate.write().await; + let previous_meta = store.rebalance_meta.read().await.clone(); + let terminal_state_present = { + let mut rebalance_meta = store.rebalance_meta.write().await; + if let Some(meta) = rebalance_meta.as_mut() { + let meta_stopped = meta.stopped_at.is_some(); + if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { + if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. }) + && has_rebalance_cleanup_warnings(pool_stat) + { + pool_stat.info.stopping = false; + pool_stat.info.status = RebalStatus::Failed; + pool_stat.info.end_time = Some(now); + pool_stat.info.last_error = Some( + pool_stat + .cleanup_warnings + .last_message + .clone() + .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()), + ); + } else if should_preserve_rebalance_stopped_state( + meta_stopped, + pool_stat.info.status, + &terminal_event, + ) { + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + state = "stopped_preserved", + "Preserved stopped rebalance status" + ); + } else { + pool_stat.info.stopping = false; + apply_rebalance_terminal_event( + &mut pool_stat.info.status, + &mut pool_stat.info.end_time, + &mut pool_stat.info.last_error, + terminal_event, + now, + ); + } + true } else { - pool_stat.info.stopping = false; - apply_rebalance_terminal_event( - &mut pool_stat.info.status, - &mut pool_stat.info.end_time, - &mut pool_stat.info.last_error, - terminal_event, - now, - ); + false } + } else { + false } + }; + + if terminal_state_present { + if let Err(err) = store.save_rebalance_stats_inner(pool_index, RebalSaveOpt::Stats).await { + let mut rebalance_meta = store.rebalance_meta.write().await; + *rebalance_meta = previous_meta; + drop(movement_guard); + return Err(Error::other(format!( + "rebalance terminal state save failed for pool {pool_index}: {err}" + ))); + } + store.ctx.advance_data_movement_operation_epoch(); + terminal_state_saved = true; } } _ = timer.tick() => { @@ -269,7 +305,7 @@ impl ECStore { } } - if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await { + if !terminal_state_saved && let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await { let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}")); error!("{} err: {:?}", msg, wrapped); if quit { @@ -590,16 +626,14 @@ impl ECStore { meta.percent_free_goal, ) { - pool_stat.info.status = RebalStatus::Completed; - pool_stat.info.end_time = Some(OffsetDateTime::now_utc()); info!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_REBALANCE, pool_index, - state = "completed", + state = "completion_ready", percent_free = pfi, - "Marked rebalance pool completed" + "Rebalance pool reached completion goal" ); return true; } @@ -612,6 +646,12 @@ impl ECStore { impl ECStore { #[tracing::instrument(skip(self))] pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> { + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + self.save_rebalance_stats_inner(pool_idx, opt).await + } + + pub(super) async fn save_rebalance_stats_inner(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> { let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; let Some(meta) = rebalance_meta.as_mut() else { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 0422ba94e..a70830f09 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -3614,6 +3614,26 @@ impl SetDisks { &self.ctx } + /// Admit one short scanner cache publication under this set's instance + /// movement fence. The caller must hold the returned guard through its + /// final conditional cache write; no scan-round work belongs under it. + pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> { + let operation_gate = self.ctx.data_movement_operation_gate(); + let operation_guard = operation_gate.read_owned().await; + if self.ctx.scanner_publication_state_allowed() { + let epoch = self.ctx.data_movement_operation_epoch(); + return Some((operation_guard, epoch)); + } + + // The owner deliberately marks the cached state UNKNOWN after every + // movement epoch advance. Do not strand remote scanner writers in that + // state: release this guard before asking the storage owner to refresh + // its durable movement snapshot, since the owner uses the same gate. + drop(operation_guard); + let owner = runtime_sources::object_store_handle().filter(|owner| Arc::ptr_eq(&owner.ctx, &self.ctx))?; + owner.scanner_data_usage_publication_admission_guard().await + } + /// Whether both sets' namespace-lock implementations cover the same object key. pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool { match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) { diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 77cdbc954..6ad051c0a 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -327,7 +327,7 @@ impl ECStore { async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> { run_bucket_usage_cleanup(guard, bucket, async { crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?; - crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await + crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await }) .await } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 1d143201c..5b9e67ab0 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -495,6 +495,11 @@ impl ECStore { ); } + // Initialize the storage-owned scanner publication state only after + // both movement metadata sources have been loaded. SetDisks cache + // writers remain fail-closed until this snapshot is available. + let _ = self.scanner_data_usage_publication_blocked().await; + let pools = installed_pool_meta.return_resumable_pools(); let mut pool_indices = Vec::with_capacity(pools.len()); diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index a40edd929..af7dcd44c 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -44,7 +44,7 @@ use crate::error::{ use crate::runtime::global::DISK_RESERVE_FRACTION; use crate::runtime::instance::InstanceContext; use crate::runtime::sources as runtime_sources; -use crate::services::rebalance::RebalanceMeta; +use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission}; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, @@ -349,16 +349,81 @@ impl ECStore { /// remain suspended until an operator clears or retries them, so they are /// a publication barrier even after the worker has stopped. pub async fn scanner_data_usage_publication_blocked(&self) -> bool { - if self.scanner_data_movement_active().await { + let operation_gate = self.ctx.data_movement_operation_gate(); + let _operation_guard = operation_gate.read_owned().await; + self.scanner_data_usage_publication_snapshot_blocked().await + } + + async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool { + if self.ctx.data_movement_operation_epoch_exhausted() { + self.ctx.set_scanner_publication_state(true); return true; } - + let decommission_cancelers = self.decommission_cancelers.read().await; + let decommission_active = decommission_cancelers + .iter() + .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active)); let pool_meta = self.pool_meta.read().await; - pool_meta.pools.iter().any(|pool| { + let decommission_active = decommission_active + || pool_meta.pools.iter().any(|pool| { + pool.decommission + .as_ref() + .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled) + }); + let decommission_terminal = pool_meta.pools.iter().any(|pool| { pool.decommission .as_ref() .is_some_and(|info| !info.queued && (info.failed || info.canceled)) - }) + }); + drop(pool_meta); + + let rebalance_active = self + .rebalance_meta + .read() + .await + .as_ref() + .is_some_and(is_rebalance_conflicting_with_decommission); + + let blocked = decommission_active || decommission_terminal || rebalance_active; + self.ctx.set_scanner_publication_state(blocked); + blocked + } + + /// Admit one short data-usage publication commit under the same + /// per-instance gate used by decommission side effects and transitions. + /// The epoch is sampled while the read guard is held, so a transition + /// cannot cross this admission without waiting for the commit to finish. + pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) { + let operation_gate = self.ctx.data_movement_operation_gate(); + let operation_guard = operation_gate.read_owned().await; + let epoch = self.ctx.data_movement_operation_epoch(); + (operation_guard, epoch) + } + + /// Acquire the movement gate and inspect the movement owner once. The + /// state inspection is performed after acquiring the read guard so a + /// transition cannot update its durable state between the check and the + /// publication commit. + pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> { + let operation_gate = self.ctx.data_movement_operation_gate(); + let operation_guard = operation_gate.read_owned().await; + if self.ctx.data_movement_operation_epoch_exhausted() { + return None; + } + if self.scanner_data_usage_publication_snapshot_blocked().await { + return None; + } + + Some((operation_guard, self.ctx.data_movement_operation_epoch())) + } + + /// Capture the current publication epoch without holding the movement + /// gate across backend I/O. Callers must re-admit the same epoch before a + /// mutation commits. + pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option { + let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?; + drop(operation_guard); + Some(epoch) } } @@ -995,6 +1060,85 @@ mod tests { } } + #[tokio::test] + async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + let operation_gate = store.ctx.data_movement_operation_gate(); + let movement_guard = operation_gate.write().await; + let pending = { + let store = store.clone(); + tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await }) + }; + tokio::task::yield_now().await; + assert!(!pending.is_finished(), "publication admission must wait for a movement writer"); + drop(movement_guard); + + let (_, epoch) = pending + .await + .expect("publication admission task should not panic") + .expect("idle store should admit publication"); + assert_eq!(epoch, 0); + assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1); + let (_, next_epoch) = store + .scanner_data_usage_publication_admission_guard() + .await + .expect("idle store should admit the next publication"); + assert_eq!(next_epoch, 1); + } + + #[tokio::test] + async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + let epoch = store + .scanner_data_usage_publication_epoch() + .await + .expect("idle store should expose a publication epoch"); + assert_eq!(epoch, 0); + + let operation_gate = store.ctx.data_movement_operation_gate(); + let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write()) + .await + .expect("epoch capture must not hold the movement gate across backend I/O"); + } + + #[tokio::test] + async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + *store.rebalance_meta.write().await = Some(RebalanceMeta { + pool_stats: vec![crate::services::rebalance::RebalanceStats { + participating: true, + info: crate::services::rebalance::RebalanceInfo { + status: crate::services::rebalance::RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }); + + assert!( + store.scanner_data_usage_publication_admission_guard().await.is_none(), + "active rebalance must fail closed at the storage-owned admission boundary" + ); + } + + #[tokio::test] + async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1); + + assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX); + assert!(store.ctx.data_movement_operation_epoch_exhausted()); + assert!( + store.scanner_data_usage_publication_admission_guard().await.is_none(), + "publication must fail closed at the reserved terminal epoch" + ); + + assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX); + assert!(store.ctx.data_movement_operation_epoch_exhausted()); + assert!(store.scanner_data_usage_publication_blocked().await); + } + // The object graph is the isolation carrier: two ECStore instances holding // distinct contexts report independent erasure state through their real // `&self` accessors — no cross-contamination. diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index cc9d123c7..550bd36e6 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -694,6 +694,11 @@ impl ECStore { /// callers only trigger missing-worker recovery after a real state change; /// delayed snapshots are merged monotonically and never blind-assigned. pub async fn reload_pool_meta(&self) -> Result { + // Serialize the durable reload with local movement transitions. Loading + // before acquiring this gate would allow a stale disk snapshot to + // overwrite a newer local transition after the writer commits. + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let mut reloaded = PoolMeta::default(); resolve_store_rebalance_pool_meta_reload_result( reloaded.load(self.pools[0].clone(), self.pools.clone()).await, @@ -701,15 +706,22 @@ impl ECStore { )?; // Lock order: release the decommission_cancelers guard before taking - // the pool_meta write guard; neither is held across the disk read. + // the pool_meta write guard; neither is held without the movement gate. let active_workers = { let cancelers = self.decommission_cancelers.read().await; - cancelers.iter().map(Option::is_some).collect::>() + cancelers + .iter() + .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active)) + .collect::>() }; let incoming_has_pools = !reloaded.pools.is_empty(); let mut pool_meta = self.pool_meta.write().await; + let movement_before = pool_meta.clone(); let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers); + if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) { + self.ctx.advance_data_movement_operation_epoch(); + } if !merged_newer && !incoming_has_pools { warn!( diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 295a1da85..3c69d2173 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -36,13 +36,13 @@ use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; use tracing::{debug, warn}; -use crate::ScannerObjectIO; use crate::storage_api::owner::HTTPPreconditions; use crate::{ BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig, - ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, TRANSITION_COMPLETE, save_config, - save_config_with_preconditions, storageclass, + SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, + TRANSITION_COMPLETE, save_config, save_config_with_preconditions, scanner_publication_admission_for_epoch, storageclass, }; +use crate::{ScannerConfigObjectDelete, ScannerObjectIO}; // Data usage constants pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; @@ -119,9 +119,13 @@ pub(crate) async fn read_config_with_revision( .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag")))?; Ok((Some(reader.read_all().await?), revision)) } - Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { - Ok((None, DataUsageCacheRevision::Missing)) - } + Err( + Error::ConfigNotFound + | Error::FileNotFound + | Error::VolumeNotFound + | Error::ObjectNotFound(_, _) + | Error::BucketNotFound(_), + ) => Ok((None, DataUsageCacheRevision::Missing)), Err(err) => Err(err), } } @@ -147,9 +151,13 @@ pub(crate) async fn read_config_revision(store: Arc, path .filter(|etag| !etag.is_empty()) .map(DataUsageCacheRevision::Etag) .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))), - Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { - Ok(DataUsageCacheRevision::Missing) - } + Err( + Error::ConfigNotFound + | Error::FileNotFound + | Error::VolumeNotFound + | Error::ObjectNotFound(_, _) + | Error::BucketNotFound(_), + ) => Ok(DataUsageCacheRevision::Missing), Err(err) => Err(err), } } diff --git a/crates/scanner/src/data_usage_define/persistence.rs b/crates/scanner/src/data_usage_define/persistence.rs index b0a28f504..18ac13881 100644 --- a/crates/scanner/src/data_usage_define/persistence.rs +++ b/crates/scanner/src/data_usage_define/persistence.rs @@ -15,6 +15,42 @@ use super::*; use std::sync::Arc; +#[derive(Debug)] +struct CachePublicationAdmissionUnavailable; + +impl std::fmt::Display for CachePublicationAdmissionUnavailable { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("scanner cache publication admission is unavailable") + } +} + +impl std::error::Error for CachePublicationAdmissionUnavailable {} + +fn cache_publication_admission_unavailable() -> Error { + Error::Io(std::io::Error::other(CachePublicationAdmissionUnavailable)) +} + +fn cache_publication_epoch_changed() -> Error { + Error::other(SCANNER_PUBLICATION_EPOCH_CHANGED) +} + +fn is_cache_publication_admission_unavailable(error: &StorageError) -> bool { + matches!( + error, + StorageError::Io(io_error) + if io_error + .get_ref() + .is_some_and(|source| source.downcast_ref::().is_some()) + ) +} + +fn is_cache_publication_epoch_changed(error: &StorageError) -> bool { + matches!( + error, + StorageError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED + ) +} + pub(super) enum DataUsageCacheLoadAttempt { Loaded { cache: Box, @@ -368,6 +404,12 @@ impl DataUsageCache { fn should_retry_save_error(err: &StorageError) -> bool { // Usage-cache files are best-effort scanner checkpoints. Retrying namespace // lock failures immediately only adds more lock traffic to the same hot object. + if is_cache_publication_admission_unavailable(err) { + return false; + } + if is_cache_publication_epoch_changed(err) { + return false; + } !matches!( err, StorageError::Lock(_) @@ -423,13 +465,14 @@ impl DataUsageCache { Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string()))) } - async fn save_path_with_retry( + async fn save_path_with_retry( store: Arc, path: &str, buf: &[u8], timeout_duration: Duration, max_retries: u32, revision: Option, + expected_epoch: Option, ) -> StorageResult<()> { Self::ensure_cache_save_metrics_registered(); let path_type = Self::cache_path_type(path); @@ -441,6 +484,17 @@ impl DataUsageCache { let buf_clone = buf.to_vec(); let revision = revision.clone(); async move { + let publication_admission = match expected_epoch { + Some(expected_epoch) => scanner_publication_admission_for_epoch(store_clone.clone(), expected_epoch).await, + None => store_clone.scanner_data_usage_publication_admission().await, + }; + let Some(_publication_admission) = publication_admission else { + return Err(if expected_epoch.is_some() { + cache_publication_epoch_changed() + } else { + cache_publication_admission_unavailable() + }); + }; if let Some(revision) = revision { save_config_with_preconditions(store_clone, &path_clone, buf_clone, revision.preconditions()).await?; } else { @@ -454,6 +508,14 @@ impl DataUsageCache { return Ok(()); }; + // An epoch-specific admission failure is authoritative: reconciling + // identical bytes cannot prove that this snapshot belongs to the + // captured movement epoch. Do not turn that fence failure into a + // successful stale publication. + if is_cache_publication_epoch_changed(&save_err) { + return Err(save_err); + } + for attempt in 0..=max_retries { let reconcile = timeout(timeout_duration, async { let mut reader = store @@ -486,24 +548,36 @@ impl DataUsageCache { Err(save_err) } - pub async fn save(&self, store: Arc, name: &str) -> StorageResult<()> { - self.save_inner(store, name, None).await + pub async fn save(&self, store: Arc, name: &str) -> StorageResult<()> { + self.save_inner(store, name, None, None).await } - pub(crate) async fn save_with_revisions( + #[cfg(test)] + pub(crate) async fn save_with_revisions( &self, store: Arc, name: &str, revisions: &DataUsageCacheRevisions, ) -> StorageResult<()> { - self.save_inner(store, name, Some(revisions)).await + self.save_inner(store, name, Some(revisions), None).await } - async fn save_inner( + pub(crate) async fn save_with_revisions_for_epoch( + &self, + store: Arc, + name: &str, + revisions: &DataUsageCacheRevisions, + expected_epoch: u64, + ) -> StorageResult<()> { + self.save_inner(store, name, Some(revisions), Some(expected_epoch)).await + } + + async fn save_inner( &self, store: Arc, name: &str, revisions: Option<&DataUsageCacheRevisions>, + expected_epoch: Option, ) -> StorageResult<()> { let mut buf = Vec::new(); self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?; @@ -517,6 +591,7 @@ impl DataUsageCache { timeout_duration, DATA_USAGE_CACHE_SAVE_RETRIES, revisions.map(|revisions| revisions.main.clone()), + expected_epoch, ) .await?; @@ -534,6 +609,7 @@ impl DataUsageCache { backup_timeout_duration, DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES, backup_revision, + expected_epoch, ) .await { @@ -548,6 +624,9 @@ impl DataUsageCache { error = %e, "Scanner cache backup save failed" ); + if is_cache_publication_admission_unavailable(&e) || is_cache_publication_epoch_changed(&e) { + return Err(e); + } } Ok(()) } diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index bdccb11c1..9a458d4b0 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -198,6 +198,22 @@ impl ObjectIO for CacheReadStore { } } +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for CacheReadStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _opts: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + #[async_trait::async_trait] impl ObjectIO for AmbiguousCacheCommitStore { type Error = Error; @@ -243,6 +259,22 @@ impl ObjectIO for AmbiguousCacheCommitStore { } } +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for AmbiguousCacheCommitStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _opts: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + #[async_trait::async_trait] impl ObjectIO for BackupFallbackStore { type Error = Error; @@ -314,6 +346,22 @@ impl ObjectIO for BackupFallbackStore { } } +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for BackupFallbackStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _opts: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + #[test] fn cache_revisions_map_to_compare_and_swap_preconditions() { let missing = DataUsageCacheRevision::Missing.preconditions(); diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index fa80ce49a..70fa9e530 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -513,6 +513,75 @@ where .await } +pub(crate) async fn save_config_with_publication_admission_for_epoch( + api: Arc, + file: &str, + data: Vec, + preconditions: HTTPPreconditions, + expected_epoch: u64, +) -> EcstoreResult +where + S: ScannerObjectIO + ScannerConfigObjectDelete, +{ + let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + save_config_with_preconditions(api, file, data, preconditions).await +} + +pub(crate) const SCANNER_PUBLICATION_EPOCH_CHANGED: &str = "scanner publication epoch changed before commit"; + +pub(crate) fn scanner_publication_epoch_changed(error: &EcstoreError) -> bool { + matches!( + error, + EcstoreError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED + ) +} + +pub(crate) async fn delete_config_with_publication_admission_for_epoch( + api: Arc, + bucket: &str, + object: &str, + opts: ScannerObjectOptions, + expected_epoch: u64, +) -> EcstoreResult +where + S: ScannerObjectIO + ScannerConfigObjectDelete, +{ + let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + api.delete_config_object(bucket, object, opts).await +} + +/// Capture the storage-owned publication epoch without retaining the read +/// guard across a potentially slow metadata read. Callers must compare this +/// token with a fresh admission immediately before their conditional write. +pub(crate) async fn scanner_publication_epoch(api: Arc) -> Option +where + S: ScannerConfigObjectDelete, +{ + let admission = api.scanner_data_usage_publication_admission().await?; + Some(admission.epoch()) +} + +/// Re-admit a publication only when the storage-owned movement epoch is still +/// the one observed before the caller's metadata read. The returned guard +/// remains held through the caller's short conditional commit. +pub(crate) async fn scanner_publication_admission_for_epoch( + api: Arc, + expected_epoch: u64, +) -> Option +where + S: ScannerConfigObjectDelete, +{ + let admission = api.scanner_data_usage_publication_admission().await?; + if admission.epoch() != expected_epoch { + return None; + } + Some(admission) +} + pub(crate) async fn save_config_shared_with_preconditions( api: Arc, file: &str, @@ -587,6 +656,39 @@ pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static { object: &str, opts: ScannerObjectOptions, ) -> EcstoreResult; + + /// Acquire storage-owned admission for one short data-usage publication + /// commit. Implementations without a storage-owned movement owner fail + /// closed; test fixtures opt into the explicit unfenced helper. + async fn scanner_data_usage_publication_admission(&self) -> Option { + None + } +} + +pub struct ScannerDataUsagePublicationAdmission { + epoch: u64, + _read_guard: Option>, +} + +impl ScannerDataUsagePublicationAdmission { + #[cfg(test)] + pub(crate) fn unfenced() -> Self { + Self { + epoch: 0, + _read_guard: None, + } + } + + fn fenced(read_guard: tokio::sync::OwnedRwLockReadGuard<()>, epoch: u64) -> Self { + Self { + epoch, + _read_guard: Some(read_guard), + } + } + + pub(crate) fn epoch(&self) -> u64 { + self.epoch + } } #[async_trait::async_trait] @@ -599,6 +701,28 @@ impl ScannerConfigObjectDelete for ECStore { ) -> EcstoreResult { ObjectOperations::delete_object(self, bucket, object, opts).await } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?; + Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch)) + } +} + +#[async_trait::async_trait] +impl ScannerConfigObjectDelete for SetDisks { + async fn delete_config_object( + &self, + bucket: &str, + object: &str, + opts: ScannerObjectOptions, + ) -> EcstoreResult { + ObjectOperations::delete_object(self, bucket, object, opts).await + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?; + Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch)) + } } #[cfg(test)] diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index 93ba33100..ef6418b9c 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -23,6 +23,7 @@ use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION; use crate::{ DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo, DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle, + scanner_publication_admission_for_epoch, scanner_publication_epoch, }; use hmac::{Hmac, KeyInit, Mac}; use rustfs_common::heal_channel::HealScanMode; @@ -686,6 +687,9 @@ async fn scan_and_persist_local_bucket( source.pool_index, source.set_index )) })?; + let expected_publication_epoch = scanner_publication_epoch(set.clone()).await.ok_or_else(|| { + RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement") + })?; let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]); let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source) .await @@ -708,6 +712,14 @@ async fn scan_and_persist_local_bucket( "remote namespace scanner cache lock was lost before reusing the current snapshot", )); } + if scanner_publication_admission_for_epoch(set.clone(), expected_publication_epoch) + .await + .is_none() + { + return Err(RemoteScannerServerError::retry_bucket( + "remote namespace scanner cache publication epoch changed before reusing the current snapshot", + )); + } return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { source, scan_plan_digest, @@ -796,9 +808,22 @@ async fn scan_and_persist_local_bucket( .await .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; let done_save = Metrics::time(Metric::SaveUsage); - let save_result = cache.save_with_revisions(set, &cache_name, &revisions).await; + // Each physical main/backup PUT must still prove the epoch captured before + // the scan. A movement transition that starts and ends during the scan + // therefore cannot admit the stale cache under the new epoch. + let save_result = cache + .save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch) + .await; done_save(); save_result.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache save failed: {err}")))?; + if scanner_publication_admission_for_epoch(set, expected_publication_epoch) + .await + .is_none() + { + return Err(RemoteScannerServerError::retry_bucket( + "remote namespace scanner cache publication epoch changed after persistence", + )); + } validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store) .await .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index f7a043308..c44ea8ae0 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -18,6 +18,7 @@ use std::future::Future; use std::sync::Mutex as StdMutex; use std::sync::{Arc, LazyLock, RwLock}; +use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch}; use crate::data_usage_define::{ BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision, @@ -30,8 +31,9 @@ use crate::runtime_config::{ use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason}; use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob}; use crate::scanner_io::{ - ScannerCycleDeferReason, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending, - dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation, + ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, + dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, + scanner_maintenance_generation, }; use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed}; use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard}; @@ -66,10 +68,12 @@ use crate::storage_api::scan::{ SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, }; use crate::{ - ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, - get_lifecycle_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache, - invalidate_data_usage_snapshot_cache, read_config, replace_bucket_usage_memory_from_info, save_config, - save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd, + ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _, + ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config, + get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config, + replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions, + save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch, + scanner_publication_epoch, scanner_publication_epoch_changed, }; const LOG_COMPONENT_SCANNER: &str = "scanner"; @@ -346,6 +350,23 @@ fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool { !info.is_complete_bucket_usage_snapshot() } +pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageInfo) -> bool { + if info.is_complete_bucket_usage_snapshot() { + return true; + } + + // Pre-marker snapshots remain readable only when their legacy identity is + // complete: a timestamp, a scanner cycle, and an exact bucket cardinality. + // A current snapshot with only scanner_epoch/scanner_cycle (or an explicit + // incomplete marker) is not evidence of a durable usage baseline. + !info.usage_snapshot_complete + && info.scanner_epoch.is_none() + && info.usage_snapshot_converged != Some(false) + && info.last_update.is_some() + && info.scanner_cycle.is_some() + && u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count) +} + fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool { data_usage_info_is_cold(authoritative) || observed.is_some_and(|observed| observed_data_usage_is_newer(observed, authoritative)) @@ -381,9 +402,18 @@ fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool { .is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0) } +#[cfg(test)] async fn sync_data_usage_backup_from_primary( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, +) -> Result<(), EcstoreError> { + sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await +} + +async fn sync_data_usage_backup_from_primary_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + expected_publication_epoch: Option, ) -> Result<(), EcstoreError> { let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { @@ -391,26 +421,62 @@ async fn sync_data_usage_backup_from_primary( return Ok(()); } + let read_epoch = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + } + expected_epoch + } + None => scanner_publication_epoch(storeapi.clone()) + .await + .ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?, + }; let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; - let primary = primary.ok_or_else(|| EcstoreError::other("authoritative data usage snapshot is missing"))?; - serde_json::from_slice::(&primary) + let primary = primary.ok_or(EcstoreError::ConfigNotFound)?; + let primary_info = serde_json::from_slice::(&primary) .map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?; + if !data_usage_info_has_persisted_baseline_identity(&primary_info) { + return Err(EcstoreError::other( + "authoritative data usage snapshot has no persisted baseline identity", + )); + } let primary = Bytes::from(primary); let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?; if backup.as_deref() == Some(primary.as_ref()) { - return Ok(()); + if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_some() + { + return Ok(()); + } + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); } let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower)); - let save_result = save_config_shared_with_preconditions( - storeapi.clone(), - &backup_path, - primary.clone(), - sha256hex, - revision.preconditions(), - ) - .await; + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + save_config_shared_with_preconditions( + storeapi.clone(), + &backup_path, + primary.clone(), + sha256hex, + revision.preconditions(), + ) + .await + }; match save_result { Ok(_) => {} @@ -427,9 +493,20 @@ async fn sync_data_usage_backup_from_primary( } let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; - if current_primary.as_deref() == Some(primary.as_ref()) { + if current_primary.as_deref() == Some(primary.as_ref()) + && scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_some() + { return Ok(()); } + if expected_publication_epoch.is_some() + && scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_none() + { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + } if retry < SCANNER_PERSIST_CAS_RETRIES { continue; } @@ -1052,7 +1129,7 @@ async fn fence_scanner_epoch_after_cycle_timeout( lock_lost: LockLost, ) -> bool where - Store: ScannerObjectIO, + Store: ScannerObjectIO + ScannerConfigObjectDelete, LockLost: Future, { let fence_ctx = ctx.child_token(); @@ -1089,7 +1166,7 @@ async fn handle_scanner_cycle_deadline( worker_stopped: bool, guard: &mut NamespaceLockGuard, ) where - Store: ScannerObjectIO, + Store: ScannerObjectIO + ScannerConfigObjectDelete, { let fenced = fence_scanner_epoch_after_cycle_timeout( ctx, @@ -1182,7 +1259,33 @@ async fn run_data_scanner_cycle_with_budget( let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - let mut background_heal_info = read_background_heal_info(storeapi.clone()).await; + // Refresh the storage-owned movement snapshot before reading background + // heal state. A missing heal object yields an in-memory default; do not + // let that default influence a cycle while publication is blocked. + if storeapi.scanner_data_usage_publication_blocked().await { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + let background_heal_read = read_background_heal_info_with_epoch(storeapi.clone()).await; + match background_heal_read.status { + BackgroundHealInfoReadStatus::Blocked => { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + BackgroundHealInfoReadStatus::Transient => { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable); + } + BackgroundHealInfoReadStatus::Failed => { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Failed; + } + BackgroundHealInfoReadStatus::ErasureSd + | BackgroundHealInfoReadStatus::Loaded + | BackgroundHealInfoReadStatus::Missing => {} + } + let mut background_heal_info = background_heal_read.info; + let background_heal_epoch = background_heal_read.expected_epoch; let scan_mode = get_cycle_scan_mode( cycle_info.current, @@ -1209,11 +1312,23 @@ async fn run_data_scanner_cycle_with_budget( configured_bitrot_cycle, ) { background_heal_info = new_heal_info.clone(); - save_background_heal_info(storeapi.clone(), new_heal_info).await; + save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await; } let cycle_start = std::time::Instant::now(); - let usage_persist_baseline = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { + // Baseline reads are part of the same publication proof as the eventual + // scanner aggregate. Hold only the short storage-owned admission guard + // across this metadata read; the full bucket scan runs after it is + // released and carries the captured epoch forward. + let Some((baseline_publication_guard, baseline_publication_epoch)) = + storeapi.scanner_data_usage_publication_admission_guard().await + else { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + }; + let usage_persist_baseline_result = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await; + drop(baseline_publication_guard); + let usage_persist_baseline = match usage_persist_baseline_result { Ok((data, revision)) => DataUsagePersistBaseline { data: data.map(Bytes::from), revision, @@ -1250,9 +1365,17 @@ async fn run_data_scanner_cycle_with_budget( ) .await; let publication_defer_reason = match &scan_result { + Ok(result) + if result + .publication_epoch() + .is_some_and(|publication_epoch| publication_epoch != baseline_publication_epoch) => + { + Some(ScannerCycleDeferReason::DataMovement) + } Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await, Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), }; + let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch); let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); let usage_persist_outcome = match publication_defer_reason { Some(reason) => { @@ -1267,12 +1390,13 @@ async fn run_data_scanner_cycle_with_budget( let ctx_clone = ctx.clone(); let route_probe_store = storeapi.clone(); let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move { - store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch( ctx_clone, storeapi_clone, receiver, Some(leader_epoch), Some(usage_persist_baseline), + publication_epoch, move || { let storeapi = route_probe_store.clone(); async move { storeapi.scanner_data_usage_publication_blocked().await } @@ -1344,7 +1468,7 @@ async fn run_data_scanner_cycle_with_budget( if !ctx.is_cancelled() && let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false) { - save_background_heal_info(storeapi.clone(), new_heal_info).await; + save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await; } mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; return ScannerCycleOutcome::Failed; @@ -1377,19 +1501,29 @@ async fn run_data_scanner_cycle_with_budget( "Scanner cycle is recovering to a newer durable cache generation" ); emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - let persisted = persist_required_scanner_cycle_floor( + let persisted = persist_required_scanner_cycle_floor_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, - required_cycle, &mut cycle_metrics_guard, + ScannerCycleFloorOptions { + required_cycle, + expected_publication_epoch: publication_epoch, + }, ) .await; return if persisted { cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial + } else if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement) } else { ScannerCycleOutcome::Failed }; @@ -1446,18 +1580,26 @@ async fn run_data_scanner_cycle_with_budget( scan_cycle_partial_reason(budget_reason), scan_cycle_partial_source(budget_reason), ); - let persisted = finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, &mut cycle_metrics_guard, + publication_epoch, ) .await; return if persisted { cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial + } else if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement) } else { ScannerCycleOutcome::Failed }; @@ -1531,18 +1673,26 @@ async fn run_data_scanner_cycle_with_budget( ); } emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - let persisted = finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, &mut cycle_metrics_guard, + publication_epoch, ) .await; return if persisted { cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial + } else if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement) } else { ScannerCycleOutcome::Failed }; @@ -1572,13 +1722,14 @@ async fn run_data_scanner_cycle_with_budget( state = "superseded", "Scanner cycle usage snapshot was superseded by concurrent namespace activity" ); - if finalize_partial_scan_cycle( + if finalize_partial_scan_cycle_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, &mut cycle_metrics_guard, + publication_epoch, ) .await { @@ -1586,11 +1737,29 @@ async fn run_data_scanner_cycle_with_budget( emit_scan_cycle_superseded(cycle_start.elapsed()); return ScannerCycleOutcome::Superseded; } + if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } emit_scan_cycle_complete(false, cycle_start.elapsed()); return ScannerCycleOutcome::Failed; } ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {} } + if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + let previous_cycle_info = cycle_info.clone(); if let Err(err) = advance_scanner_cycle(cycle_info) { error!( target: "rustfs::scanner", @@ -1610,7 +1779,19 @@ async fn run_data_scanner_cycle_with_budget( global_metrics().clear_current_scan_mode(); retain_recent_cycle_completions(&mut cycle_info.cycle_completed); - if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await { + if !persist_scanner_cycle_state_for_epoch(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, publication_epoch) + .await + { + if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + *cycle_info = previous_cycle_info; + emit_scan_cycle_deferred(cycle_start.elapsed()); + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } cycle_metrics_guard.finish(cycle_info.clone()).await; emit_scan_cycle_complete(false, cycle_start.elapsed()); return ScannerCycleOutcome::Failed; @@ -1620,7 +1801,7 @@ async fn run_data_scanner_cycle_with_budget( done_cycle(); emit_scan_cycle_complete(true, cycle_start.elapsed()); if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) { - save_background_heal_info(storeapi.clone(), new_heal_info).await; + save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await; } info!( diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index 6459ae6e5..db2e46ff8 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -162,6 +162,8 @@ pub(crate) enum ScannerCycleStateStartup { enum CycleRecoveryMarkerReadError { #[error("cycle recovery marker backend read failed: {0}")] Backend(#[source] EcstoreError), + #[error("cycle recovery marker publication is blocked by data movement")] + PublicationBlocked, #[error("invalid cycle recovery marker: {0}")] Invalid(&'static str), #[error("cycle recovery marker revision changed while publishing")] @@ -326,12 +328,13 @@ fn cycle_state_generation_and_epoch(buf: &[u8]) -> (u64, u64) { } async fn persist_cycle_recovery_marker( - storeapi: Arc, + storeapi: Arc, primary_revision: &DataUsageCacheRevision, generation: u64, leader_epoch: u64, classification: &'static str, reason: &'static str, + expected_epoch: u64, ) -> Result { let now = unix_now_secs(); let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { @@ -370,6 +373,9 @@ async fn persist_cycle_recovery_marker( state: "blocked".to_string(), }; let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?; + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else { + return Err(CycleRecoveryMarkerReadError::PublicationBlocked); + }; let save_result = save_config_with_preconditions( storeapi.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), @@ -478,7 +484,7 @@ async fn read_cycle_recovery_marker_revision( } async fn quarantine_invalid_cycle_state( - storeapi: Arc, + storeapi: Arc, revision: &DataUsageCacheRevision, buf: &[u8], ) -> ScannerCycleStateStartup { @@ -488,7 +494,7 @@ async fn quarantine_invalid_cycle_state( } async fn quarantine_invalid_cycle_state_with_reason( - storeapi: Arc, + storeapi: Arc, revision: &DataUsageCacheRevision, generation: u64, leader_epoch: u64, @@ -515,7 +521,19 @@ async fn quarantine_invalid_cycle_state_with_reason( reason: Some(reason.to_string()), }; set_scanner_cycle_recovery_status(base_status); - match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason).await { + let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker publication is blocked by data movement"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker publication is blocked by data movement".to_string(), + )); + }; + match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason, expected_epoch) + .await + { Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")), Err(CycleRecoveryMarkerReadError::Backend(_)) => { // Keep the poison object untouched and retry marker creation with the @@ -524,6 +542,16 @@ async fn quarantine_invalid_cycle_state_with_reason( "failed to persist scanner cycle recovery marker".to_string(), )); } + Err(CycleRecoveryMarkerReadError::PublicationBlocked) => { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker publication is blocked by data movement"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker publication is blocked by data movement".to_string(), + )); + } Err(CycleRecoveryMarkerReadError::Conflict) => { set_scanner_cycle_recovery_status(recovery_status( "transient", @@ -546,16 +574,18 @@ async fn mark_cycle_recovery_cleanup_pending( storeapi: Arc, mut marker: ScannerCycleRecoveryMarker, marker_revision: &DataUsageCacheRevision, + expected_epoch: u64, ) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> { marker.state = "cleanup-pending".to_string(); marker.last_attempt_at_unix_secs = unix_now_secs(); let bytes = serde_json::to_vec(&marker) .map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?; - let info = save_config_with_preconditions( + let info = save_config_with_publication_admission_for_epoch( storeapi.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), bytes, marker_revision.preconditions(), + expected_epoch, ) .await .map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?; @@ -567,7 +597,9 @@ async fn mark_cycle_recovery_cleanup_pending( Ok((marker, revision)) } -pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc) -> ScannerCycleStateStartup { +pub(crate) async fn load_scanner_cycle_state_for_startup( + storeapi: Arc, +) -> ScannerCycleStateStartup { let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { Ok((None, _)) => None, Ok((Some(data), marker_revision)) => match serde_json::from_slice::(&data) { @@ -594,6 +626,16 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker publication is blocked by data movement"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker publication is blocked by data movement".to_string(), + )); + } Err(CycleRecoveryMarkerReadError::Invalid(reason)) => { set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false)); return ScannerCycleStateStartup::Blocked; @@ -750,6 +792,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string())); } + let Some(reset_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return Err(ScannerError::Other("scanner recovery reset is blocked by data movement".to_string())); + }; + let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false), Err(CycleRecoveryMarkerReadError::Invalid(_)) => { @@ -835,7 +881,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< }; if let Some((primary_cycle, primary_epoch)) = primary_state { let (cleanup_marker, cleanup_marker_revision) = - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision).await?; + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?; set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending")); let usage_floor = persisted_usage_floor(storeapi.clone()).await?; let fence_epoch = primary_epoch @@ -855,14 +901,21 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "preserved scanner cycle state exceeds the bounded object size".to_string(), )); } - let preserved_info = save_config_with_preconditions( + let preserved_info = save_config_with_publication_admission_for_epoch( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), preserved_data, primary_revision.preconditions(), + reset_epoch, ) .await - .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")))?; + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")) + } + })?; let preserved_revision = preserved_info .etag .filter(|etag| !etag.is_empty()) @@ -873,7 +926,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after fencing newer cycle state".to_string(), )); } - fence_scanner_usage_epoch(&ctx, storeapi.clone(), fence_epoch) + fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch)) .await .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; if guard.is_lock_lost() { @@ -889,20 +942,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner cycle state changed before recovery marker cleanup".to_string(), )); } - storeapi - .delete_config_object( - RUSTFS_META_BUCKET, - DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), - ScannerObjectOptions { - // This is one exact metadata object. Prefix-delete mode - // bypasses HTTP preconditions in the ECStore path. - delete_prefix: false, - http_preconditions: Some(cleanup_marker_revision.preconditions()), - ..Default::default() - }, - ) - .await - .map_err(|err| ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")))?; + delete_config_with_publication_admission_for_epoch( + storeapi.clone(), + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + ScannerObjectOptions { + // This is one exact metadata object. Prefix-delete mode + // bypasses HTTP preconditions in the ECStore path. + delete_prefix: false, + http_preconditions: Some(cleanup_marker_revision.preconditions()), + ..Default::default() + }, + reset_epoch, + ) + .await + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")) + } + })?; set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); super::notify_scanner_cycle_recovery_wake(); return Ok(()); @@ -946,16 +1006,23 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< let (marker, marker_revision) = if marker.state == "cleanup-pending" { (marker, marker_revision) } else { - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision).await? + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await? }; - let rebuilt_info = save_config_with_preconditions( + let rebuilt_info = save_config_with_publication_admission_for_epoch( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), data, primary_revision.preconditions(), + reset_epoch, ) .await - .map_err(|err| ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")))?; + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")) + } + })?; let rebuilt_revision = rebuilt_info .etag .filter(|etag| !etag.is_empty()) @@ -965,7 +1032,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after rebuilding cycle state".to_string(), )); } - if let Err(err) = fence_scanner_usage_epoch(&ctx, storeapi.clone(), leader_epoch).await { + if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch)).await + { set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { path: DATA_USAGE_BLOOM_NAME_PATH.clone(), quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), @@ -1034,20 +1102,40 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< )); } - if let Err(err) = storeapi - .delete_config_object( - RUSTFS_META_BUCKET, - DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), - ScannerObjectOptions { - // This is one exact metadata object. Prefix-delete mode - // bypasses HTTP preconditions in the ECStore path. - delete_prefix: false, - http_preconditions: Some(marker_revision.preconditions()), - ..Default::default() - }, - ) - .await + if let Err(err) = delete_config_with_publication_admission_for_epoch( + storeapi.clone(), + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + ScannerObjectOptions { + // This is one exact metadata object. Prefix-delete mode + // bypasses HTTP preconditions in the ECStore path. + delete_prefix: false, + http_preconditions: Some(marker_revision.preconditions()), + ..Default::default() + }, + reset_epoch, + ) + .await { + if scanner_publication_epoch_changed(&err) { + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "cleanup-pending".to_string(), + classification: Some(marker.classification.clone()), + primary_revision: Some(rebuilt_revision.clone()), + generation: Some(next), + leader_epoch: Some(leader_epoch), + retry_count: marker.retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: true, + reason: Some("movement epoch changed before recovery marker cleanup".to_string()), + ..Default::default() + }); + return Err(ScannerError::Other( + "scanner recovery reset deferred by a movement epoch change".to_string(), + )); + } set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { path: DATA_USAGE_BLOOM_NAME_PATH.clone(), quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), @@ -1209,8 +1297,14 @@ pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(), Ok(()) } -pub(super) async fn persisted_usage_floor(storeapi: Arc) -> Result { +pub(super) async fn persisted_usage_floor( + storeapi: Arc, +) -> Result { + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string())); + }; let mut floor = PersistedUsageFloor::default(); + let mut found_any = false; let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> { floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); if let Some(completed_cycle) = usage.scanner_cycle { @@ -1229,6 +1323,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc) - let usage = serde_json::from_slice::(&data).map_err(|err| { ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}")) })?; + if !data_usage_info_has_persisted_baseline_identity(&usage) { + return Err(ScannerError::Other(format!( + "scanner usage floor from {primary_path} has no persisted baseline identity" + ))); + } let epoch = usage.scanner_epoch.unwrap_or_default(); update_floor(&mut floor, &usage, primary_path)?; Some(epoch) @@ -1247,6 +1346,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc) - let usage = serde_json::from_slice::(&data).map_err(|err| { ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}")) })?; + if !data_usage_info_has_persisted_baseline_identity(&usage) { + return Err(ScannerError::Other(format!( + "scanner usage floor from {backup_path} has no persisted baseline identity" + ))); + } let backup_epoch = usage.scanner_epoch.unwrap_or_default(); // A backup write from an older leader may complete after the // primary epoch has been fenced. It must not advance the startup @@ -1263,9 +1367,21 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc) - } } if any_found { + found_any = true; break; } } + + if !found_any { + return Err(ScannerError::Other( + "persisted scanner usage floor has no authoritative baseline".to_string(), + )); + } + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi, read_epoch).await else { + return Err(ScannerError::Other( + "scanner usage floor changed while its epoch proof was being confirmed".to_string(), + )); + }; Ok(floor) } @@ -1274,12 +1390,30 @@ pub(super) fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_ *leader_epoch = (*leader_epoch).max(floor.leader_epoch); } +#[derive(Clone, Copy)] +pub(super) struct ScannerCycleFloorOptions { + pub(super) required_cycle: u64, + pub(super) expected_publication_epoch: Option, +} + +#[cfg(test)] pub(super) async fn persist_scanner_cycle_state( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, leader_epoch: u64, +) -> bool { + persist_scanner_cycle_state_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, None).await +} + +pub(super) async fn persist_scanner_cycle_state_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + expected_publication_epoch: Option, ) -> bool { let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) { Ok(buf) => buf, @@ -1315,9 +1449,29 @@ pub(super) async fn persist_scanner_cycle_state( #[cfg(test)] notify_scanner_cycle_state_persist_test_hook(leader_epoch); - match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) - .await - { + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return false; + }; + if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) { + return false; + } + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "publication_admission_unavailable", + "Scanner state persistence skipped without movement admission" + ); + return false; + }; + save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) + .await + }; + match save_result { Ok(object_info) => { let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else { error!( @@ -1345,6 +1499,13 @@ pub(super) async fn persist_scanner_cycle_state( ); return false; } + if let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return false; + } debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, @@ -1436,6 +1597,13 @@ pub(super) async fn persist_scanner_cycle_state( } if persisted_cycle.next >= cycle_info.next { + if let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return false; + } *cycle_info = persisted_cycle; debug!( target: "rustfs::scanner", @@ -1496,19 +1664,33 @@ pub(super) async fn persist_scanner_cycle_state( false } +#[cfg(test)] pub(super) async fn finalize_partial_scan_cycle( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, leader_epoch: u64, cycle_metrics_guard: &mut ScannerCycleMetricsGuard, +) -> bool { + finalize_partial_scan_cycle_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, cycle_metrics_guard, None).await +} + +pub(super) async fn finalize_partial_scan_cycle_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + cycle_metrics_guard: &mut ScannerCycleMetricsGuard, + expected_publication_epoch: Option, ) -> bool { // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter // must still advance (and persist) because per-bucket next_cycle is stamped from // it and compacted folders are only rescanned when their hash matches // next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle // expiry and usage refresh on every folder outside the stuck window. + let previous_cycle_info = cycle_info.clone(); if let Err(err) = advance_scanner_cycle(cycle_info) { error!( target: "rustfs::scanner", @@ -1524,28 +1706,69 @@ pub(super) async fn finalize_partial_scan_cycle( } cycle_info.current = 0; global_metrics().clear_current_scan_mode(); - let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; + let persisted = persist_scanner_cycle_state_for_epoch( + ctx, + storeapi.clone(), + cycle_info, + revision, + leader_epoch, + expected_publication_epoch, + ) + .await; + if !persisted + && let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi, expected_epoch) + .await + .is_none() + { + *cycle_info = previous_cycle_info; + } cycle_metrics_guard.finish(cycle_info.clone()).await; persisted } +#[cfg(test)] pub(super) async fn persist_required_scanner_cycle_floor( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, leader_epoch: u64, required_cycle: u64, cycle_metrics_guard: &mut ScannerCycleMetricsGuard, ) -> bool { - if required_cycle <= cycle_info.current || required_cycle == u64::MAX { + persist_required_scanner_cycle_floor_for_epoch( + ctx, + storeapi, + cycle_info, + revision, + leader_epoch, + cycle_metrics_guard, + ScannerCycleFloorOptions { + required_cycle, + expected_publication_epoch: None, + }, + ) + .await +} + +pub(super) async fn persist_required_scanner_cycle_floor_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + cycle_metrics_guard: &mut ScannerCycleMetricsGuard, + options: ScannerCycleFloorOptions, +) -> bool { + if options.required_cycle <= cycle_info.current || options.required_cycle == u64::MAX { error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, current_cycle = cycle_info.current, - required_cycle, + required_cycle = options.required_cycle, state = "invalid_cache_cycle_floor", "Scanner cache cycle floor is invalid" ); @@ -1553,10 +1776,27 @@ pub(super) async fn persist_required_scanner_cycle_floor( return false; } - cycle_info.next = cycle_info.next.max(required_cycle); + let previous_cycle_info = cycle_info.clone(); + cycle_info.next = cycle_info.next.max(options.required_cycle); cycle_info.current = 0; global_metrics().clear_current_scan_mode(); - let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; + let persisted = persist_scanner_cycle_state_for_epoch( + ctx, + storeapi.clone(), + cycle_info, + revision, + leader_epoch, + options.expected_publication_epoch, + ) + .await; + if !persisted + && let Some(expected_epoch) = options.expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi, expected_epoch) + .await + .is_none() + { + *cycle_info = previous_cycle_info; + } cycle_metrics_guard.finish(cycle_info.clone()).await; persisted } diff --git a/crates/scanner/src/scanner/heal_info.rs b/crates/scanner/src/scanner/heal_info.rs index 18d0511f4..55e2a0dba 100644 --- a/crates/scanner/src/scanner/heal_info.rs +++ b/crates/scanner/src/scanner/heal_info.rs @@ -26,31 +26,90 @@ pub struct BackgroundHealInfo { pub current_scan_mode: HealScanMode, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum BackgroundHealInfoReadStatus { + ErasureSd, + Loaded, + Missing, + Blocked, + Transient, + Failed, +} + +pub(super) struct BackgroundHealInfoRead { + pub(super) info: BackgroundHealInfo, + pub(super) expected_epoch: Option, + pub(super) status: BackgroundHealInfoReadStatus, +} + +pub(super) fn classify_background_heal_read_error(error: &EcstoreError) -> BackgroundHealInfoReadStatus { + if matches!(error, EcstoreError::ConfigNotFound) { + BackgroundHealInfoReadStatus::Missing + } else { + BackgroundHealInfoReadStatus::Transient + } +} + +pub(super) fn decode_background_heal_info(data: &[u8]) -> Result { + serde_json::from_slice(data) +} + /// Read background healing information from storage pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHealInfo { + read_background_heal_info_with_epoch(storeapi).await.info +} + +/// Read background healing information together with the movement epoch that +/// fenced the read. The epoch must be reused by the matching cycle update so a +/// missing-object default cannot be committed across a movement transition. +pub(super) async fn read_background_heal_info_with_epoch(storeapi: Arc) -> BackgroundHealInfoRead { // Skip for ErasureSD setup if scanner_is_erasure_sd().await { - return BackgroundHealInfo::default(); + return BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch: None, + status: BackgroundHealInfoReadStatus::ErasureSd, + }; + } + + let expected_epoch = scanner_publication_epoch(storeapi.clone()).await; + if expected_epoch.is_none() { + return BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch, + status: BackgroundHealInfoReadStatus::Blocked, + }; } // Get last healing information match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await { - Ok(buf) => serde_json::from_slice::(&buf).unwrap_or_else(|e| { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, - path = %&*BACKGROUND_HEAL_INFO_PATH, - state = "decode_failed", - error = %e, - "Scanner background heal decode failed" - ); - BackgroundHealInfo::default() - }), + Ok(buf) => match decode_background_heal_info(&buf) { + Ok(info) => BackgroundHealInfoRead { + info, + expected_epoch, + status: BackgroundHealInfoReadStatus::Loaded, + }, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "decode_failed", + error = %e, + "Scanner background heal decode failed" + ); + BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch, + status: BackgroundHealInfoReadStatus::Failed, + } + } + }, Err(e) => { - // Only log if it's not a ConfigNotFound error - if e != EcstoreError::ConfigNotFound { + let status = classify_background_heal_read_error(&e); + if status == BackgroundHealInfoReadStatus::Transient { warn!( target: "rustfs::scanner", event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, @@ -62,7 +121,11 @@ pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHeal "Scanner background heal read failed" ); } - BackgroundHealInfo::default() + BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch, + status, + } } } } @@ -70,6 +133,14 @@ pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHeal /// Save background healing information to storage #[instrument(skip(storeapi))] pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundHealInfo) { + save_background_heal_info_for_epoch(storeapi, info, None).await; +} + +pub(super) async fn save_background_heal_info_for_epoch( + storeapi: Arc, + info: BackgroundHealInfo, + expected_epoch: Option, +) { // Skip for ErasureSD setup if scanner_is_erasure_sd().await { return; @@ -93,7 +164,25 @@ pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundH } }; - // Save configuration + // Save configuration only after storage-owned movement admission. The + // read path may return an in-memory default for a missing object, but a + // movement transition must not let that default become durable state. + let publication_admission = match expected_epoch { + Some(expected_epoch) => scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await, + None => storeapi.scanner_data_usage_publication_admission().await, + }; + let Some(_publication_admission) = publication_admission else { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "publication_admission_unavailable", + "Scanner background heal save skipped without movement admission" + ); + return; + }; if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await { warn!( target: "rustfs::scanner", diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index ab22f56d9..8183b7602 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -61,16 +61,22 @@ pub(super) async fn reconcile_scanner_leadership_claim( } pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result { - serde_json::from_slice(data) - .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}"))) + let usage: DataUsageInfo = serde_json::from_slice(data) + .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?; + if !data_usage_info_has_persisted_baseline_identity(&usage) { + return Err(ScannerError::Other(format!( + "scanner usage epoch fence from {path} has no persisted baseline identity" + ))); + } + Ok(usage) } pub(super) async fn usage_snapshot_for_epoch_fence( storeapi: Arc, primary: Option<&[u8]>, -) -> Result { +) -> Result, ScannerError> { if let Some(primary) = primary { - return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()); + return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()).map(Some); } let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -78,7 +84,7 @@ pub(super) async fn usage_snapshot_for_epoch_fence( .await .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?; if let Some(backup) = backup.as_deref() { - return decode_usage_snapshot_for_epoch_fence(backup, &backup_path); + return decode_usage_snapshot_for_epoch_fence(backup, &backup_path).map(Some); } for path in [ @@ -89,26 +95,53 @@ pub(super) async fn usage_snapshot_for_epoch_fence( .await .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; if let Some(legacy) = legacy.as_deref() { - return decode_usage_snapshot_for_epoch_fence(legacy, &path); + return decode_usage_snapshot_for_epoch_fence(legacy, &path).map(Some); } } - Ok(DataUsageInfo::default()) + // A missing usage snapshot is an uninitialized state, not an empty + // snapshot. Leadership fencing may proceed without creating a plausible + // default; the first authoritative scanner publication will create it. + Ok(None) } -pub(super) async fn fence_scanner_usage_epoch( +pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, claimed_epoch: u64, + expected_publication_epoch: Option, ) -> Result<(), ScannerError> { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { if ctx.is_cancelled() { return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); } + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return Err(ScannerError::Other( + "scanner usage epoch fence publication is blocked by data movement".to_string(), + )); + }; + if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(ScannerError::Other( + "scanner usage epoch fence changed while recovery reset was in progress".to_string(), + )); + } let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) .await .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?; - let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?; + let Some(mut usage) = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await? else { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(ScannerError::Other( + "scanner usage epoch fence changed while confirming a missing usage baseline".to_string(), + )); + }; + return Err(ScannerError::Other("authoritative scanner usage baseline is missing".to_string())); + }; match usage.scanner_epoch { Some(epoch) if epoch > claimed_epoch => { return Err(ScannerError::Other(format!( @@ -122,9 +155,18 @@ pub(super) async fn fence_scanner_usage_epoch( let data = serde_json::to_vec(&usage) .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; - let save_result = + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(ScannerError::Other( + "scanner usage epoch fence changed while preparing its conditional write".to_string(), + )); + }; save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) - .await; + .await + }; if save_result .as_ref() .ok() @@ -165,10 +207,13 @@ pub(super) async fn fence_scanner_usage_epoch( pub(super) async fn complete_scanner_leadership_claim( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, claimed_epoch: u64, + expected_publication_epoch: Option, ) -> bool { - if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await { + if let Err(err) = + fence_scanner_usage_epoch_with_expected_epoch(ctx, storeapi, claimed_epoch, expected_publication_epoch).await + { error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, @@ -187,7 +232,7 @@ pub(super) async fn complete_scanner_leadership_claim( pub(super) async fn claim_scanner_leadership( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, persisted_epoch: &mut u64, @@ -226,15 +271,69 @@ pub(super) async fn claim_scanner_leadership( }; let previous_revision = revision.clone(); - let save_result = + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return false; + }; + let (usage_primary, _) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { + Ok(result) => result, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "leader_usage_baseline_read_failed", + error = %err, + "Scanner leadership claim deferred because the usage baseline could not be read" + ); + return false; + } + }; + match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref()).await { + Ok(Some(_)) => {} + Ok(None) => { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "leader_usage_baseline_missing", + "Scanner leadership claim deferred until a usage baseline is published" + ); + return false; + } + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "leader_usage_baseline_invalid", + error = %err, + "Scanner leadership claim deferred because the usage baseline is invalid" + ); + return false; + } + } + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return false; + }; save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions()) - .await; + .await + }; match save_result { Ok(object_info) => { if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) { *revision = DataUsageCacheRevision::Etag(etag); *persisted_epoch = claimed_epoch; - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await; } match reconcile_scanner_leadership_claim( @@ -249,7 +348,7 @@ pub(super) async fn claim_scanner_leadership( .await { Ok(ScannerLeadershipClaimReconcile::Durable) => { - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await; } Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue, Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { @@ -293,7 +392,7 @@ pub(super) async fn claim_scanner_leadership( .await { Ok(ScannerLeadershipClaimReconcile::Durable) => { - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await; } Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index d73d63cde..9e1d7cef8 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info}; use super::*; use crate::EcstoreResult; use crate::{ @@ -22,6 +23,7 @@ use crate::{ }; use std::collections::{HashMap, HashSet}; use std::io::Cursor; +use std::sync::atomic::{AtomicBool, Ordering}; use std::task::Poll; use temp_env::{with_var, with_var_unset}; use tokio::io::AsyncReadExt; @@ -343,6 +345,7 @@ struct MemoryConfigStore { cancel_after_successful_puts: Mutex>, replace_after_successful_puts: Mutex)>>, put_counts: Mutex>, + publication_admission_blocked: AtomicBool, } fn memory_config_key(bucket: &str, object: &str) -> String { @@ -1350,7 +1353,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() { let old_usage = DataUsageInfo { scanner_epoch: Some(7), scanner_cycle: Some(41), - ..Default::default() + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) }; let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode"); save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone()) @@ -1424,7 +1427,7 @@ async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() { let usage = DataUsageInfo { scanner_epoch: Some(7), scanner_cycle: Some(41), - ..Default::default() + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) }; save_config( store.clone(), @@ -1695,7 +1698,7 @@ async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() { DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&DataUsageInfo { scanner_epoch: Some(u64::MAX - 1), - ..Default::default() + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) }) .expect("usage floor should encode"), ) @@ -1847,14 +1850,12 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() { let store = Arc::new(MemoryConfigStore::default()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] { + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(epoch); + usage.scanner_cycle = Some(cycle); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, path), - serde_json::to_vec(&DataUsageInfo { - scanner_epoch: Some(epoch), - scanner_cycle: Some(cycle), - ..Default::default() - }) - .expect("usage snapshot should encode"), + serde_json::to_vec(&usage).expect("usage snapshot should encode"), ); } @@ -1879,14 +1880,12 @@ async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() { let store = Arc::new(MemoryConfigStore::default()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] { + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(epoch); + usage.scanner_cycle = Some(cycle); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, path), - serde_json::to_vec(&DataUsageInfo { - scanner_epoch: Some(epoch), - scanner_cycle: Some(cycle), - ..Default::default() - }) - .expect("usage snapshot should encode"), + serde_json::to_vec(&usage).expect("usage snapshot should encode"), ); } @@ -1916,6 +1915,23 @@ fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() { })); } +#[test] +fn scanner_baseline_identity_requires_complete_or_strict_legacy_shape() { + assert!(!data_usage_info_has_persisted_baseline_identity(&DataUsageInfo { + scanner_epoch: Some(3), + scanner_cycle: Some(7), + ..Default::default() + })); + + let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + legacy.usage_snapshot_complete = false; + legacy.scanner_cycle = Some(7); + assert!(data_usage_info_has_persisted_baseline_identity(&legacy)); + + legacy.scanner_epoch = Some(3); + assert!(!data_usage_info_has_persisted_baseline_identity(&legacy)); +} + #[test] fn scanner_startup_prompts_only_for_a_newer_valid_observation() { let authoritative = DataUsageInfo { @@ -1948,12 +1964,9 @@ fn scanner_startup_prompts_only_for_a_newer_valid_observation() { #[tokio::test] async fn scanner_startup_prefers_v2_over_legacy_usage() { let store = Arc::new(MemoryConfigStore::default()); - let legacy = DataUsageInfo { - scanner_epoch: Some(19), - scanner_cycle: Some(41), - last_update: Some(std::time::SystemTime::now()), - ..Default::default() - }; + let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + legacy.scanner_epoch = Some(19); + legacy.scanner_cycle = Some(41); let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), @@ -1976,13 +1989,9 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() { } ); - let authoritative = DataUsageInfo { - scanner_epoch: Some(23), - scanner_cycle: Some(51), - last_update: Some(std::time::SystemTime::now()), - usage_snapshot_complete: true, - ..Default::default() - }; + let mut authoritative = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + authoritative.scanner_epoch = Some(23); + authoritative.scanner_cycle = Some(51); let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode"); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), @@ -2024,6 +2033,8 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() { #[tokio::test] async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() { let store = Arc::new(MemoryConfigStore::default()); + assert!(persisted_usage_floor(store.clone()).await.is_err()); + store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), b"not-json".to_vec(), @@ -2087,6 +2098,39 @@ async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() { } } +#[tokio::test] +async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_primary() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + store.objects.lock().await.insert( + primary_key.clone(), + serde_json::to_vec(&primary).expect("primary usage snapshot should encode"), + ); + store.revisions.lock().await.insert(primary_key.clone(), 1); + + store.publication_admission_blocked.store(true, Ordering::Release); + let movement_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()) + .await + .expect_err("movement admission loss should fail backup synchronization"); + assert!(scanner_publication_epoch_changed(&movement_error)); + + store.publication_admission_blocked.store(false, Ordering::Release); + store.objects.lock().await.remove(&primary_key); + store.revisions.lock().await.remove(&primary_key); + assert!(matches!( + sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()).await, + Err(EcstoreError::ConfigNotFound) + )); + + store.objects.lock().await.insert(primary_key.clone(), b"not-json".to_vec()); + store.revisions.lock().await.insert(primary_key, 1); + let corrupt_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store) + .await + .expect_err("corrupt primary should fail backup synchronization"); + assert!(!scanner_publication_epoch_changed(&corrupt_error)); +} + #[async_trait::async_trait] impl crate::ScannerConfigObjectDelete for MemoryConfigStore { async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult { @@ -2110,6 +2154,10 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore { revisions.remove(&key); Ok(ObjectInfo::default()) } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + (!self.publication_admission_blocked.load(Ordering::Acquire)).then(crate::ScannerDataUsagePublicationAdmission::unfenced) + } } #[test] @@ -2270,6 +2318,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf started: Utc::now(), }; assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + seed_usage_snapshot_for_leadership_claim(&store).await; let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); let old_epoch_commit = CurrentCycle { @@ -2313,6 +2362,61 @@ async fn test_leadership_claim_rejects_terminal_epoch() { assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); } +#[tokio::test] +async fn leadership_claim_defers_without_usage_baseline_before_bloom_write() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + let mut persisted_epoch = 0; + + assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); + assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err()); +} + +#[tokio::test] +async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() { + let store = Arc::new(MemoryConfigStore::default()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store.objects.lock().await.insert(usage_key.clone(), b"not-json".to_vec()); + store.revisions.lock().await.insert(usage_key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + let mut persisted_epoch = 0; + + assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); +} + +#[tokio::test] +async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_write() { + let store = Arc::new(MemoryConfigStore::default()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let data = serde_json::to_vec(&DataUsageInfo::default()).expect("default usage should encode"); + store.objects.lock().await.insert(usage_key.clone(), data); + store.revisions.lock().await.insert(usage_key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + let mut persisted_epoch = 0; + + assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); +} + #[tokio::test] async fn test_leadership_claim_confirms_commit_after_returned_error() { let store = Arc::new(MemoryConfigStore::default()); @@ -2329,6 +2433,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() { started: Utc::now(), }; let mut persisted_epoch = 0; + seed_usage_snapshot_for_leadership_claim(&store).await; assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); @@ -2373,6 +2478,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { ); old_usage.buckets_count = 1; old_usage.calculate_totals(); + old_usage.usage_snapshot_complete = true; let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode"); store.objects.lock().await.insert(usage_key.clone(), old_data.clone()); store.revisions.lock().await.insert(usage_key, 1); @@ -2419,6 +2525,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() { started: Utc::now(), }; assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + seed_usage_snapshot_for_leadership_claim(&store).await; let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); store @@ -2590,6 +2697,35 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() { } } +#[tokio::test] +async fn test_observational_usage_defers_when_authoritative_baseline_is_missing() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1); + observation.usage_snapshot_converged = Some(false); + sender.send(observation).await.expect("observation should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + CancellationToken::new(), + store.clone(), + receiver, + None, + None, + || async { false }, + ) + .await; + + assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)); + assert!( + !store + .objects + .lock() + .await + .contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())) + ); +} + #[tokio::test] async fn test_usage_route_barrier_precedes_durable_reconciliation() { let store = Arc::new(MemoryConfigStore::default()); @@ -3425,6 +3561,14 @@ fn complete_usage_with_bucket_count(last_update: Option, info } +async fn seed_usage_snapshot_for_leadership_claim(store: &Arc) { + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let data = serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)) + .expect("leadership usage baseline should encode"); + store.objects.lock().await.insert(key.clone(), data); + store.revisions.lock().await.insert(key, 1); +} + fn usage_with_last_update(last_update: Option) -> DataUsageInfo { complete_usage_with_bucket_count(last_update, 0) } @@ -5129,6 +5273,19 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() { assert_eq!(info.bitrot_start_time, Some(now)); } +#[test] +fn background_heal_read_failures_never_become_initializable_defaults() { + assert_eq!( + classify_background_heal_read_error(&EcstoreError::ConfigNotFound), + BackgroundHealInfoReadStatus::Missing + ); + assert_eq!( + classify_background_heal_read_error(&EcstoreError::SlowDown), + BackgroundHealInfoReadStatus::Transient + ); + assert!(decode_background_heal_info(b"not-json").is_err()); +} + #[test] fn test_background_heal_info_for_scan_start_keeps_deep_window_start() { with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || { diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index 4f291cd02..9b211c1f5 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -112,11 +112,39 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel } pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + ctx: CancellationToken, + storeapi: Arc, + receiver: mpsc::Receiver, + leader_epoch: Option, + initial_baseline: Option, + route_probe: F, +) -> DataUsagePersistOutcome +where + F: Fn() -> Fut + Send + Sync, + Fut: Future + Send, +{ + store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch( + ctx, + storeapi, + receiver, + leader_epoch, + initial_baseline, + None, + route_probe, + ) + .await +} + +pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch< + F, + Fut, +>( ctx: CancellationToken, storeapi: Arc, mut receiver: mpsc::Receiver, leader_epoch: Option, initial_baseline: Option, + expected_publication_epoch: Option, route_probe: F, ) -> DataUsagePersistOutcome where @@ -134,6 +162,14 @@ where if let Some(leader_epoch) = leader_epoch { data_usage_info.scanner_epoch = Some(leader_epoch); } + if let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } let observational = data_usage_info.usage_snapshot_converged == Some(false); let target_path = if observational { DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() @@ -154,7 +190,28 @@ where break; } + let mut publication_epoch = expected_publication_epoch; if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() { + let read_epoch = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } + expected_epoch + } + None => { + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + }; + read_epoch + } + }; + publication_epoch = Some(read_epoch); let authoritative_data = match next_baseline.as_ref() { Some(baseline) => baseline.data.clone(), None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { @@ -175,25 +232,48 @@ where } }, }; - let authoritative = match authoritative_data.as_deref() { - Some(data) => match serde_json::from_slice::(data) { - Ok(info) => info, - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), - state = "observed_baseline_decode_failed", - error = %err, - "Scanner refused to publish an observation from an invalid authoritative baseline" - ); - outcome = DataUsagePersistOutcome::Failed; - continue; - } - }, - None => DataUsageInfo::default(), + let Some(authoritative_data) = authoritative_data else { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_missing", + "Scanner deferred observational publication until an authoritative usage baseline exists" + ); + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + }; + let authoritative = match serde_json::from_slice::(&authoritative_data) { + Ok(info) if data_usage_info_has_persisted_baseline_identity(&info) => info, + Ok(_) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_identity_missing", + "Scanner refused to publish an observation without authoritative baseline identity" + ); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_decode_failed", + error = %err, + "Scanner refused to publish an observation from an invalid authoritative baseline" + ); + outcome = DataUsagePersistOutcome::Failed; + continue; + } }; data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity()); } @@ -240,6 +320,26 @@ where break 'updates; } + let publication_epoch_for_save = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + expected_epoch + } + None => match publication_epoch.take() { + Some(epoch) => epoch, + None => { + let Some(epoch) = scanner_publication_epoch(storeapi.clone()).await else { + break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + }; + epoch + } + }, + }; let baseline = if !observational && cas_retry == 0 { next_baseline.take() } else { @@ -329,14 +429,22 @@ where } let done_save = Metrics::time(Metric::SaveUsage); - let save_result = save_config_shared_with_preconditions( - storeapi.clone(), - target_path, - data.clone(), - sha256hex.clone(), - revision.preconditions(), - ) - .await; + let save_result = { + let Some(_publication_admission) = + scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await + else { + done_save(); + break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + }; + save_config_shared_with_preconditions( + storeapi.clone(), + target_path, + data.clone(), + sha256hex.clone(), + revision.preconditions(), + ) + .await + }; done_save(); match save_result { @@ -427,7 +535,16 @@ where if observational { invalidate_admin_data_usage_snapshot_cache().await; } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch( + storeapi.clone(), + &data_usage_info, + expected_publication_epoch, + ) + .await; + if expected_publication_epoch.is_some() && !cleanup_ok { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } invalidate_data_usage_snapshot_cache().await; replace_bucket_usage_memory_from_info(&data_usage_info).await; } @@ -438,7 +555,16 @@ where if observational { invalidate_admin_data_usage_snapshot_cache().await; } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch( + storeapi.clone(), + &data_usage_info, + expected_publication_epoch, + ) + .await; + if expected_publication_epoch.is_some() && !cleanup_ok { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } invalidate_data_usage_snapshot_cache().await; } global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); @@ -460,7 +586,16 @@ where if observational { invalidate_admin_data_usage_snapshot_cache().await; } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch( + storeapi.clone(), + &data_usage_info, + expected_publication_epoch, + ) + .await; + if expected_publication_epoch.is_some() && !cleanup_ok { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } invalidate_data_usage_snapshot_cache().await; replace_bucket_usage_memory_from_info(&data_usage_info).await; } @@ -471,7 +606,10 @@ where if backup_due { let done_save = Metrics::time(Metric::SaveUsage); - if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await { + let backup_result = + sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await; + done_save(); + if let Err(e) = backup_result { warn!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, @@ -482,22 +620,50 @@ where error = %e, "Scanner data usage backup save failed" ); + if scanner_publication_epoch_changed(&e) { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } + outcome = DataUsagePersistOutcome::Failed; + break 'updates; } - done_save(); } } outcome } -pub(super) async fn cleanup_observed_data_usage_snapshot( +pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch( storeapi: Arc, authoritative: &DataUsageInfo, -) { + expected_publication_epoch: Option, +) -> bool { + let read_epoch = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return false; + } + expected_epoch + } + None => match scanner_publication_epoch(storeapi.clone()).await { + Some(read_epoch) => read_epoch, + None => return false, + }, + }; + if expected_publication_epoch.is_some() + && scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_none() + { + return false; + } let (observed_data, revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { Ok((Some(data), revision)) => (data, revision), - Ok((None, _)) => return, + Ok((None, _)) => return true, Err(err) => { error!( target: "rustfs::scanner", @@ -509,7 +675,7 @@ pub(super) async fn cleanup_observed_data_usage_snapshot( error = %err, "Scanner could not inspect observational data usage snapshot before authoritative cleanup" ); - return; + return true; } }; let observed = match serde_json::from_slice::(&observed_data) { @@ -525,25 +691,26 @@ pub(super) async fn cleanup_observed_data_usage_snapshot( error = %err, "Scanner refused to remove an invalid observational data usage snapshot after authoritative save" ); - return; + return true; } }; if observed_data_usage_is_newer(&observed, authoritative) { - return; + return true; } - let result = storeapi - .delete_config_object( - RUSTFS_META_BUCKET, - DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), - ScannerObjectOptions { - delete_prefix: true, - delete_prefix_object: true, - http_preconditions: Some(revision.preconditions()), - ..Default::default() - }, - ) - .await; + let result = delete_config_with_publication_admission_for_epoch( + storeapi, + RUSTFS_META_BUCKET, + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + ScannerObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + http_preconditions: Some(revision.preconditions()), + ..Default::default() + }, + read_epoch, + ) + .await; match result { Ok(_) @@ -564,6 +731,10 @@ pub(super) async fn cleanup_observed_data_usage_snapshot( error = %err, "Scanner could not remove stale observational data usage snapshot after authoritative save" ); + if scanner_publication_epoch_changed(&err) { + return false; + } } } + true } diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 030668f26..0a5ecdf12 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -56,9 +56,10 @@ use crate::storage_api::scan::NamespaceLocking as _; use crate::storage_api::scanner_io::{BucketInfo, BucketOptions}; use crate::{ BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, - RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, - ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version, - get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, storageclass, + RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerConfigObjectDelete as _, ScannerDiskExt as _, + ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, + enqueue_runtime_free_version, get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, + scanner_publication_admission_for_epoch, scanner_publication_epoch, storageclass, }; pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file"; @@ -143,6 +144,10 @@ pub struct ScannerBucketScanPlan { all_buckets: Arc>, digest: DataUsageScanPlanDigest, leader_epoch: u64, + /// Epoch captured once for the whole scanner cycle. `None` is retained + /// for unfenced test implementations; production plans always carry the + /// admission token captured before bucket enumeration. + publication_epoch: Option, dirty_usage_buckets: Arc, bucket_failures: ScannerBucketFailureState, pending_maintenance_work: Arc, @@ -578,6 +583,7 @@ fn scanner_activity_preflight( #[derive(Debug)] pub(crate) struct ScannerCycleResult { pub(crate) status: ScannerCycleStatus, + publication_epoch: Option, dirty_usage_clear: Option, remote_dirty_usage_acknowledgements: Vec, failed_dirty_usage: bool, @@ -589,6 +595,7 @@ impl ScannerCycleResult { pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option) -> Self { Self { status, + publication_epoch: None, dirty_usage_clear, remote_dirty_usage_acknowledgements: Vec::new(), failed_dirty_usage: false, @@ -597,6 +604,15 @@ impl ScannerCycleResult { } } + pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option) -> Self { + self.publication_epoch = publication_epoch; + self + } + + pub(crate) fn publication_epoch(&self) -> Option { + self.publication_epoch + } + fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self { self.failed_dirty_usage = failed_dirty_usage; self diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 001901187..070a89deb 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -395,6 +395,7 @@ pub(super) async fn persist_and_publish_cache_snapshot( updates: &mpsc::Sender, mut cache_snapshot: DataUsageCache, cache_cycle_floor: &AtomicU64, + expected_publication_epoch: u64, ) -> Option { let source = cache_snapshot.info.source?; let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await { @@ -489,7 +490,7 @@ pub(super) async fn persist_and_publish_cache_snapshot( let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache_snapshot - .save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions) + .save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch) .await { error!( @@ -519,6 +520,24 @@ pub(super) async fn persist_and_publish_cache_snapshot( ); return None; } + // The persisted-root fast path performs no PUT, so it also needs the + // cycle token re-admission before forwarding the root to the aggregate. + // This final check covers both the fast path and a successful save. + if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch) + .await + .is_none() + { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "publication_epoch_changed_before_publish", + "Scanner cache root publish skipped after movement epoch change" + ); + return None; + } drop(guard); let last_update = cache_snapshot.info.last_update; diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index db52cb559..5b8246bc8 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks { all_buckets, digest: scan_plan_digest, leader_epoch, + publication_epoch, dirty_usage_buckets, bucket_failures, pending_maintenance_work, @@ -40,6 +41,12 @@ impl ScannerIOCache for SetDisks { let set_label = self.set_index.to_string(); let source = DataUsageCacheSource::new(self.pool_index, self.set_index); + let expected_publication_epoch = match publication_epoch { + Some(epoch) => epoch, + None => scanner_publication_epoch(self.clone()) + .await + .ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?, + }; let mut old_cache = DataUsageCache::default(); if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await { warn!( @@ -76,10 +83,16 @@ impl ScannerIOCache for SetDisks { cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); } reset_disk_bucket_scan_gauges(&pool_label, &set_label); - return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref()) - .await - .map(|_| ()) - .ok_or_else(|| StorageError::other("failed to persist empty scanner set scope")); + return persist_and_publish_cache_snapshot( + self, + &updates, + cache, + cache_cycle_floor.as_ref(), + expected_publication_epoch, + ) + .await + .map(|_| ()) + .ok_or_else(|| StorageError::other("failed to persist empty scanner set scope")); } let (disks, healing) = self.get_online_disks_with_healing(false).await; @@ -414,6 +427,7 @@ impl ScannerIOCache for SetDisks { let pending_maintenance_work_clone = pending_maintenance_work.clone(); let dirty_usage_buckets_clone = dirty_usage_buckets.clone(); let cache_cycle_floor_clone = cache_cycle_floor.clone(); + let expected_publication_epoch_clone = expected_publication_epoch; let remote_server_epoch = match worker_mode { NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch), NamespaceScannerWorkerMode::Coordinator => None, @@ -753,6 +767,23 @@ impl ScannerIOCache for SetDisks { ); continue; } + if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch) + .await + .is_none() + { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "publication_epoch_changed_before_reuse", + "Current scanner bucket cache root publish skipped after movement epoch change" + ); + continue; + } if let Err(e) = send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone) .await @@ -901,7 +932,12 @@ impl ScannerIOCache for SetDisks { { let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache - .save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) + .save_with_revisions_for_epoch( + store_clone_clone.clone(), + cache_name.as_str(), + &revisions, + expected_publication_epoch_clone, + ) .await { error!( @@ -958,7 +994,12 @@ impl ScannerIOCache for SetDisks { false } else { match partial_cache - .save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) + .save_with_revisions_for_epoch( + store_clone_clone.clone(), + cache_name.as_str(), + &revisions, + expected_publication_epoch_clone, + ) .await { Ok(()) => true, @@ -1029,7 +1070,12 @@ impl ScannerIOCache for SetDisks { let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache - .save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions) + .save_with_revisions_for_epoch( + store_clone_clone.clone(), + &cache_name, + &revisions, + expected_publication_epoch_clone, + ) .await { done_save(); @@ -1064,6 +1110,24 @@ impl ScannerIOCache for SetDisks { continue; } + if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch_clone) + .await + .is_none() + { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "publication_epoch_changed_after_save", + "Scanner bucket cache root publish skipped after movement epoch change" + ); + continue; + } + debug!( target: "rustfs::scanner::io", event = EVENT_SCANNER_DATA_USAGE_STREAM, @@ -1159,7 +1223,14 @@ impl ScannerIOCache for SetDisks { cache.info.lkg_scan_plan_digest = None; cache.clone() }; - let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await; + let _ = persist_and_publish_cache_snapshot( + self.clone(), + &updates, + cache_snapshot, + cache_cycle_floor.as_ref(), + expected_publication_epoch, + ) + .await; } else { let mut incomplete_scope = cache_mutex.lock().await.clone(); incomplete_scope.info.name = DATA_USAGE_ROOT.to_string(); diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index defa8a47c..7a74e95f8 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -68,6 +68,19 @@ impl ScannerIOCycle for ECStore { )); } + // Capture one storage-owned movement epoch for the entire cycle. Set + // workers must not each observe a fresh epoch: a movement transition + // between sets would otherwise allow a mixed-generation aggregate. + let publication_epoch = match self.scanner_data_usage_publication_admission().await { + Some(admission) => Some(admission.epoch()), + None => { + return Ok(ScannerCycleResult::new( + ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), + None, + )); + } + }; + let distributed = self.setup_is_dist_erasure().await; let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) { ScannerActivityPreflight::Ready(snapshot) => snapshot, @@ -131,7 +144,9 @@ impl ScannerIOCycle for ECStore { if all_buckets.is_empty() { reset_set_scan_gauges(); if !bucket_plan_complete { - return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); + return Ok( + ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch) + ); } let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await; let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot); @@ -155,7 +170,7 @@ impl ScannerIOCycle for ECStore { ) .await? { - return Ok(ScannerCycleResult::new(status, None)); + return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch)); } let dirty_usage_clear = (status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone()); @@ -165,6 +180,7 @@ impl ScannerIOCycle for ECStore { Vec::new() }; return Ok(ScannerCycleResult::new(status, dirty_usage_clear) + .with_publication_epoch(publication_epoch) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)); } @@ -180,7 +196,7 @@ impl ScannerIOCycle for ECStore { "Scanner set state update detected missing disk sets" ); reset_set_scan_gauges(); - return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); + return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)); } let set_scan_limit = scanner_budgeted_concurrency_limit( @@ -250,6 +266,7 @@ impl ScannerIOCycle for ECStore { all_buckets: Arc::clone(&all_buckets), digest: scan_plan_digest, leader_epoch, + publication_epoch, dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(), bucket_failures: bucket_failures.clone(), pending_maintenance_work: pending_maintenance_work.clone(), @@ -430,6 +447,7 @@ impl ScannerIOCycle for ECStore { Vec::new() }; Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear) + .with_publication_epoch(publication_epoch) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) .with_failed_dirty_usage(!failed_buckets.is_empty()) .with_pending_maintenance_work(pending_maintenance_work) diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 9d2c1586d..7931a44f9 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -145,6 +145,50 @@ async fn scanner_cache_locks_allow_cross_source_workers() { assert!(!second.is_lock_lost()); } +#[tokio::test] +async fn scanner_set_cache_admission_tracks_owner_snapshot_and_fails_closed() { + let (_temp_dir, store) = setup_two_pool_scanner_store().await; + let set = store.pools[0].disk_set[0].clone(); + + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_none(), + "a set must not publish before the owner has refreshed its movement snapshot" + ); + assert!(!store.scanner_data_usage_publication_blocked().await); + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_some(), + "an idle owner snapshot should admit the set cache" + ); + + let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()]; + pool_stats[0] = EcstoreRebalanceStats { + participating: true, + info: EcstoreRebalanceInfo { + start_time: Some(OffsetDateTime::now_utc()), + status: EcstoreRebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }; + *store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta { + id: Uuid::new_v4().to_string(), + pool_stats, + ..Default::default() + }); + assert!(store.scanner_data_usage_publication_blocked().await); + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_none(), + "active movement must keep set cache publication blocked" + ); + + *store.rebalance_meta.write().await = None; + assert!(!store.scanner_data_usage_publication_blocked().await); + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_some(), + "an idle owner refresh must make set cache publication live again" + ); +} + #[tokio::test] async fn scanner_cycle_is_deferred_while_rebalance_is_active() { let (_temp_dir, store) = setup_two_pool_scanner_store().await;