fix(scanner): fence system metadata publication (#6444)

* feat(scanner): fence usage publication during data movement

* fix(scanner): detect movement refresh state changes

* fix(scanner): fence publication during data movement

* fix(scanner): close movement epoch publication races

* fix(scanner): fence movement-sensitive publication paths

* fix(scanner): fence cache and heal recovery paths

* fix(scanner): carry publication epoch through scan cycle

* fix(scanner): recheck remote cache epoch after save

* fix(scanner): recheck local cache epoch before publish

* fix(scanner): fence data usage writers and baseline

* fix(scanner): expose decommission activity to publication fence

* fix(scanner): release publication gate before reads

* fix(scanner): complete publication fence integration

* fix(scanner): avoid empty usage baseline publication

* chore(scanner): gate test-only helpers

* fix: use decommission canceler in reload test

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-08-23 17:28:21 +08:00
committed by GitHub
parent 9cda615519
commit e196a134cc
26 changed files with 2465 additions and 370 deletions
+155 -33
View File
@@ -135,7 +135,7 @@ struct DecommissionOperation {
} }
impl DecommissionCanceler { impl DecommissionCanceler {
fn new(token: CancellationToken) -> Self { pub(crate) fn new(token: CancellationToken) -> Self {
Self { Self {
operation: Arc::new(DecommissionOperation { operation: Arc::new(DecommissionOperation {
token, token,
@@ -153,7 +153,7 @@ impl DecommissionCanceler {
&self.operation.token &self.operation.token
} }
fn is_active(&self) -> bool { pub(crate) fn is_active(&self) -> bool {
self.operation.active.load(Ordering::Acquire) self.operation.active.load(Ordering::Acquire)
} }
@@ -1444,6 +1444,30 @@ fn should_replace_pool_status_for_status_refresh(
!has_active_worker && persisted.last_update > current.last_update !has_active_worker && persisted.last_update > current.last_update
} }
fn pool_decommission_movement_snapshot(
info: Option<&PoolDecommissionInfo>,
) -> (bool, bool, bool, bool, bool, Option<OffsetDateTime>) {
info.map(|info| {
(
info.has_decommission_state(),
info.complete,
info.failed,
info.canceled,
info.queued,
info.start_time,
)
})
.unwrap_or_default()
}
pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool {
before.pools.len() != after.pools.len()
|| before.pools.iter().zip(after.pools.iter()).any(|(before, after)| {
pool_decommission_movement_snapshot(before.decommission.as_ref())
!= pool_decommission_movement_snapshot(after.decommission.as_ref())
})
}
/// Merges a persisted pool metadata snapshot into `current` monotonically: /// Merges a persisted pool metadata snapshot into `current` monotonically:
/// a pool entry is replaced only when no active worker covers it and the /// a pool entry is replaced only when no active worker covers it and the
/// snapshot is strictly newer, so delayed snapshots never roll back local /// snapshot is strictly newer, so delayed snapshots never roll back local
@@ -3358,6 +3382,8 @@ impl ECStore {
.first() .first()
.cloned() .cloned()
.ok_or_else(|| Error::other("refresh_pool_status_meta: no pools available"))?; .ok_or_else(|| Error::other("refresh_pool_status_meta: no pools available"))?;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut persisted = PoolMeta::default(); let mut persisted = PoolMeta::default();
persisted.load(pool, self.pools.clone()).await?; persisted.load(pool, self.pools.clone()).await?;
@@ -3370,7 +3396,9 @@ impl ECStore {
}; };
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers); if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(()) Ok(())
} }
@@ -3489,11 +3517,19 @@ impl ECStore {
async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?;
let _start_guard = self.start_gate.lock().await; let _start_guard = self.start_gate.lock().await;
let operation_gate = self.ctx.decommission_operation_gate(); // Signal cancellation before waiting for the movement writer. A worker
let operation_guard = operation_gate.write().await; // may still hold a publication read guard while it observes this
// signal; waiting for the writer first would deadlock that handoff.
// Lock order: decommission_cancelers before pool_meta. Holding both makes if let Some(owner) = owner {
// owner validation and the terminal transition one atomic operation. owner.cancel();
} else if let Some(canceler) = self.decommission_cancelers.read().await.get(idx).and_then(Option::as_ref) {
canceler.cancel();
}
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta, terminal_canceler) = { let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await; let cancelers = self.decommission_cancelers.read().await;
let mut lock = self.pool_meta.write().await; let mut lock = self.pool_meta.write().await;
@@ -3562,9 +3598,26 @@ impl ECStore {
self.release_decommission_canceler_slot(idx, canceler).await; self.release_decommission_canceler_slot(idx, canceler).await;
} }
if should_save_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("decommission_cancel for pool {idx}"); let stage = format!("decommission_cancel for pool {idx}");
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; if let Err(err) =
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
{
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "terminal_reload_failed",
error = %err,
"Decommission cancel saved locally but pool meta reload failed"
);
}
} }
Ok(()) Ok(())
@@ -3589,7 +3642,11 @@ impl ECStore {
.unwrap_or((false, false, false, false)); .unwrap_or((false, false, false, false));
ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?; ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?;
} }
self.cancel_decommission_routines_and_wait(&[idx]).await; // Cancel workers before waiting for the movement writer so active
// object operations can observe the signal and release read guards.
self.cancel_decommission_routines(&[idx]).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let (should_reload_pool_meta, previous_pool_meta) = { let (should_reload_pool_meta, previous_pool_meta) = {
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
@@ -3606,17 +3663,37 @@ impl ECStore {
return Err(err); return Err(err);
} }
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("clear_decommission for pool {idx}"); let stage = format!("clear_decommission for pool {idx}");
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; if let Err(err) =
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
{
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "terminal_reload_failed",
error = %err,
"Decommission clear saved locally but pool meta reload failed"
);
}
} }
Ok(()) Ok(())
} }
async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> { async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> {
// Serialize promotion and generation capture with clear/restart transitions.
let (changed, generation, save_error) = { let (changed, generation, save_error) = {
let _start_guard = self.start_gate.lock().await; let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
if pool_meta.pools.get(idx).is_none() { if pool_meta.pools.get(idx).is_none() {
return Err(Error::other("failed to start decommission: target pool was not found")); return Err(Error::other("failed to start decommission: target pool was not found"));
@@ -3644,6 +3721,9 @@ impl ECStore {
return Err(err); return Err(err);
} }
if changed {
self.ctx.advance_data_movement_operation_epoch();
}
if changed && let Some(notification_sys) = runtime_sources::notification_sys() { if changed && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("promote_queued_decommission for pool {idx}"); let stage = format!("promote_queued_decommission for pool {idx}");
if let Err(err) = if let Err(err) =
@@ -3668,6 +3748,8 @@ impl ECStore {
} }
async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> { async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let changed = { let changed = {
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())? pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())?
@@ -3708,19 +3790,20 @@ impl ECStore {
is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx)) is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx))
} }
#[cfg(test)]
async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) { async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) {
self.cancel_decommission_routines(indices).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
}
async fn cancel_decommission_routines(&self, indices: &[usize]) {
{ {
let mut cancelers = self.decommission_cancelers.write().await; let mut cancelers = self.decommission_cancelers.write().await;
for idx in indices { for idx in indices {
take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx); take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx);
} }
} }
self.wait_for_decommission_side_effects().await;
}
async fn wait_for_decommission_side_effects(&self) {
let operation_gate = self.ctx.decommission_operation_gate();
let _operation_guard = operation_gate.write().await;
} }
async fn reserve_decommission_routines( async fn reserve_decommission_routines(
@@ -4218,7 +4301,7 @@ impl ECStore {
} }
decommission_cancel_signal_result(rx.is_cancelled())?; decommission_cancel_signal_result(rx.is_cancelled())?;
self.ensure_decommission_generation_current(idx, generation).await?; self.ensure_decommission_generation_current(idx, generation).await?;
let operation_gate = self.ctx.decommission_operation_gate(); let operation_gate = self.ctx.data_movement_operation_gate();
let bucket_incarnation_fence = match expected_bucket_incarnation_id { let bucket_incarnation_fence = match expected_bucket_incarnation_id {
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?), Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
@@ -5113,9 +5196,12 @@ impl ECStore {
{ {
ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?; ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?;
let _start_guard = self.start_gate.lock().await; let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: decommission_cancelers before pool_meta. Holding both makes // Lock order: movement gate, then decommission_cancelers, then pool_meta.
// owner validation and the terminal transition one atomic operation. // Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await; let cancelers = self.decommission_cancelers.read().await;
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
@@ -5151,6 +5237,12 @@ impl ECStore {
if let Some(canceler) = terminal_canceler.as_ref() { if let Some(canceler) = terminal_canceler.as_ref() {
self.release_decommission_canceler_slot(idx, canceler).await; self.release_decommission_canceler_slot(idx, canceler).await;
} }
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("decommission_failed for pool {idx}"); let stage = format!("decommission_failed for pool {idx}");
if let Some(err) = observe_decommission_terminal_reload_result( if let Some(err) = observe_decommission_terminal_reload_result(
@@ -5208,7 +5300,12 @@ impl ECStore {
} }
self.verify_decommission_durable_ilm_receipts(idx).await?; self.verify_decommission_durable_ilm_receipts(idx).await?;
let _start_guard = self.start_gate.lock().await; let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = { let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await; let cancelers = self.decommission_cancelers.read().await;
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
@@ -5249,6 +5346,12 @@ impl ECStore {
if let Some(canceler) = terminal_canceler.as_ref() { if let Some(canceler) = terminal_canceler.as_ref() {
self.release_decommission_canceler_slot(idx, canceler).await; self.release_decommission_canceler_slot(idx, canceler).await;
} }
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("complete_decommission for pool {idx}"); let stage = format!("complete_decommission for pool {idx}");
if let Some(err) = observe_decommission_terminal_reload_result( if let Some(err) = observe_decommission_terminal_reload_result(
@@ -5479,11 +5582,16 @@ impl ECStore {
self.ensure_decommission_rebalance_idle_after_refresh().await?; self.ensure_decommission_rebalance_idle_after_refresh().await?;
let all_space_infos = self.get_decommission_all_pool_space_infos().await?; let all_space_infos = self.get_decommission_all_pool_space_infos().await?;
self.cancel_decommission_routines_and_wait(&indices).await; // Signal cancellation before waiting for the movement writer so active
// object operations can observe the signal and release read guards.
self.cancel_decommission_routines(&indices).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let index_cancelers = if let Some((rx, local_indices)) = reservation { let index_cancelers = if let Some((rx, local_indices)) = reservation {
// Lock order matches terminal transitions: decommission_cancelers // Lock order matches terminal transitions: movement gate, then
// before pool_meta while start_gate excludes another start. // decommission_cancelers, then pool_meta while start_gate excludes
// another start.
let mut cancelers = self.decommission_cancelers.write().await; let mut cancelers = self.decommission_cancelers.write().await;
let pool_meta = self.pool_meta.read().await; let pool_meta = self.pool_meta.read().await;
ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?; ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?;
@@ -5505,6 +5613,10 @@ impl ECStore {
let previous_pool_meta = self let previous_pool_meta = self
.save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets) .save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets)
.await?; .await?;
self.ctx.advance_data_movement_operation_epoch();
// The local durable transition is now fenced. Release the writer
// before any peer RPC; remote reload must not block scanner admission.
drop(_movement_guard);
if let Some(notification_sys) = runtime_sources::notification_sys() if let Some(notification_sys) = runtime_sources::notification_sys()
&& let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await) && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await)
@@ -5519,11 +5631,20 @@ impl ECStore {
"Decommission start failed after pool metadata save" "Decommission start failed after pool metadata save"
); );
{ let rollback_result = {
let mut pool_meta = self.pool_meta.write().await; let movement_guard = movement_gate.write().await;
rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone()); {
} let mut pool_meta = self.pool_meta.write().await;
if let Err(rollback_save_err) = self.save_current_pool_meta().await { rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone());
}
let rollback_result = self.save_current_pool_meta().await;
if rollback_result.is_ok() {
self.ctx.advance_data_movement_operation_epoch();
}
drop(movement_guard);
rollback_result
};
if let Err(rollback_save_err) = rollback_result {
error!( error!(
event = EVENT_DECOMMISSION_STATE, event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
@@ -6527,7 +6648,7 @@ impl ECStore {
generation: OffsetDateTime, generation: OffsetDateTime,
) -> Result<()> { ) -> Result<()> {
self.ensure_decommission_generation_current(idx, generation).await?; self.ensure_decommission_generation_current(idx, generation).await?;
let operation_gate = self.ctx.decommission_operation_gate(); let operation_gate = self.ctx.data_movement_operation_gate();
run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await
} }
@@ -8077,7 +8198,7 @@ mod pools_tests {
..Default::default() ..Default::default()
}; };
merge_pool_status_refresh(&mut current, persisted, &[false]); assert!(merge_pool_status_refresh(&mut current, persisted, &[false]));
let info = current.pools[0] let info = current.pools[0]
.decommission .decommission
@@ -8121,7 +8242,7 @@ mod pools_tests {
..Default::default() ..Default::default()
}; };
merge_pool_status_refresh(&mut current, persisted, &[true]); assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
let info = current.pools[0] let info = current.pools[0]
.decommission .decommission
@@ -8162,7 +8283,7 @@ mod pools_tests {
..Default::default() ..Default::default()
}; };
merge_pool_status_refresh(&mut current, persisted, &[true]); assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
let info = current.pools[0] let info = current.pools[0]
.decommission .decommission
@@ -8591,7 +8712,7 @@ mod pools_tests {
#[tokio::test] #[tokio::test]
async fn test_decommission_transition_waits_without_registered_canceler() { async fn test_decommission_transition_waits_without_registered_canceler() {
let store = decommission_worker_test_store(PoolMeta::default(), vec![None]); let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
let operation_gate = store.ctx.decommission_operation_gate(); let operation_gate = store.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read().await; let operation_guard = operation_gate.read().await;
let transition = tokio::spawn({ let transition = tokio::spawn({
let store = store.clone(); let store = store.clone();
@@ -11338,6 +11459,7 @@ mod pools_tests {
assert!(store.decommission_cancelers.read().await[0].is_none()); assert!(store.decommission_cancelers.read().await[0].is_none());
assert!(!canceler.is_active()); assert!(!canceler.is_active());
assert!(canceler.is_cancelled()); assert!(canceler.is_cancelled());
assert_eq!(store.ctx.data_movement_operation_epoch(), 1);
} }
#[test] #[test]
+260 -30
View File
@@ -388,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
"nonconverged data usage observations cannot replace the quota-authoritative snapshot", "nonconverged data usage observations cannot replace the quota-authoritative snapshot",
)); ));
} }
let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
// Prevent older data from overwriting newer persisted stats // Prevent older data from overwriting newer persisted stats
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
if let Ok((existing, source)) = existing_snapshot
&& source.is_authoritative() && source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now()) && let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{ {
@@ -400,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
return Ok(()); return Ok(());
} }
save_data_usage_in_backend(data_usage_info, store).await save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
} }
async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> { async fn save_data_usage_in_backend(
data_usage_info: DataUsageInfo,
store: Arc<ECStore>,
expected_publication_epoch: u64,
) -> Result<(), Error> {
let data = let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?; serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
// Save to backend using the same mechanism as original code // Save to backend using the same mechanism as original code
let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if publication_epoch != expected_publication_epoch {
return Err(Error::other("data usage publication epoch changed before save"));
}
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data) crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
.await .await
.map_err(Error::other)?; .map_err(Error::other)?;
drop(publication_guard);
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await; cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
.await;
// Invalidate the cached snapshot so readers observe the new save on their // Invalidate the cached snapshot so readers observe the new save on their
// next request instead of waiting out the remaining TTL. The next cached // next request instead of waiting out the remaining TTL. The next cached
@@ -449,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
} }
} }
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo) async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
where store: &S,
authoritative: &DataUsageInfo,
publication_store: Option<&ECStore>,
) where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized, S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{ {
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { let observed_read_epoch = match publication_store {
Some(publication_store) => {
let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
return;
};
Some(epoch)
}
None => None,
};
let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
let (observed, revision) = match observed_snapshot {
Ok(Some(snapshot)) => snapshot, Ok(Some(snapshot)) => snapshot,
Ok(None) => return, Ok(None) => return,
Err(err) => { Err(err) => {
@@ -469,6 +498,19 @@ where
return; return;
} }
let publication_guard = match publication_store {
Some(publication_store) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return;
};
if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
return;
}
Some(guard)
}
None => None,
};
match store.delete_observed_data_usage_snapshot(&revision).await { match store.delete_observed_data_usage_snapshot(&revision).await {
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {} Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
Err(err) => { Err(err) => {
@@ -479,6 +521,15 @@ where
); );
} }
} }
drop(publication_guard);
}
#[cfg(test)]
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
} }
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) { fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -519,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> { pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
prepare_bucket_usage_for_namespace_change(bucket, None).await?; prepare_bucket_usage_for_namespace_change(bucket, None).await?;
remove_bucket_usage_from_backend_with_guard(store, bucket, None).await remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
} }
pub(crate) async fn prepare_bucket_usage_for_namespace_change( pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -542,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
Ok(()) Ok(())
} }
#[cfg(test)]
pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>( pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
store: &S, store: &S,
bucket: &str, bucket: &str,
@@ -551,6 +603,24 @@ where
S: EcstoreObjectIO + ?Sized, S: EcstoreObjectIO + ?Sized,
{ {
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await; let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
store: &ECStore,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
async fn invalidate_bucket_usage_snapshot_caches(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
) -> Result<(), Error> {
let mut snapshot_cache = data_usage_snapshot_cache().write().await; let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?; ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache); clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -558,7 +628,7 @@ where
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await; let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?; ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache); clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
result Ok(())
} }
async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error> async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -617,48 +687,83 @@ fn ensure_bucket_namespace_guard(
Ok(()) Ok(())
} }
#[cfg(test)]
async fn remove_bucket_usage_from_backend_with_store_and_guard<S>( async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
store: &S, store: &S,
bucket: &str, bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>, guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> ) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
}
async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication_store: Option<&ECStore>,
) -> Result<(), Error>
where where
S: EcstoreObjectIO + ?Sized, S: EcstoreObjectIO + ?Sized,
{ {
ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?; ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
let primary_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let primary_seed = load_data_usage_seed_for_missing_primary(store).await?; let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
remove_bucket_usage_from_object_with_retries( remove_bucket_usage_from_object_with_retries_and_publication(
store, store,
DATA_USAGE_OBJ_NAME_PATH.as_str(), DATA_USAGE_OBJ_NAME_PATH.as_str(),
bucket, bucket,
DATA_USAGE_REMOVE_CAS_RETRIES, DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&primary_seed), primary_seed.as_ref(),
guard, guard,
publication_store.map(|store| (store, primary_seed_epoch)),
) )
.await?; .await?;
ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?; ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
let backup_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await? .await?
.map_or(primary_seed, |(data_usage_info, _)| data_usage_info); .map(|(data_usage_info, _)| data_usage_info)
remove_bucket_usage_from_object_with_retries( .or_else(|| primary_seed.clone());
remove_bucket_usage_from_object_with_retries_and_publication(
store, store,
DATA_USAGE_OBJ_BACKUP_PATH.as_str(), DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
bucket, bucket,
DATA_USAGE_REMOVE_CAS_RETRIES, DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&backup_seed), backup_seed.as_ref(),
guard, guard,
publication_store.map(|store| (store, backup_seed_epoch)),
) )
.await?; .await?;
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?; ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
if let Err(err) = remove_bucket_usage_from_object_with_retries( if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
store, store,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
bucket, bucket,
DATA_USAGE_REMOVE_CAS_RETRIES, DATA_USAGE_REMOVE_CAS_RETRIES,
None, None,
guard, guard,
publication_store.map(|store| (store, None)),
) )
.await .await
{ {
@@ -672,12 +777,21 @@ where
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
] { ] {
remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?; remove_bucket_usage_from_object_with_retries_and_publication(
store,
object,
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await?;
} }
Ok(()) Ok(())
} }
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error> async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
where where
S: EcstoreObjectIO + ?Sized, S: EcstoreObjectIO + ?Sized,
{ {
@@ -690,12 +804,13 @@ where
if !authoritative { if !authoritative {
data_usage_info.usage_snapshot_complete = false; data_usage_info.usage_snapshot_complete = false;
} }
return Ok(data_usage_info); return Ok(Some(data_usage_info));
} }
} }
Ok(DataUsageInfo::default()) Ok(None)
} }
#[cfg(test)]
async fn remove_bucket_usage_from_object_with_retries<S>( async fn remove_bucket_usage_from_object_with_retries<S>(
store: &S, store: &S,
object: &str, object: &str,
@@ -704,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
missing_seed: Option<&DataUsageInfo>, missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>, guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> ) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
.await
}
async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
store: &S,
object: &str,
bucket: &str,
cas_retries: usize,
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication: Option<(&ECStore, Option<u64>)>,
) -> Result<(), Error>
where where
S: EcstoreObjectIO + ?Sized, S: EcstoreObjectIO + ?Sized,
{ {
for attempt in 0..=cas_retries { for attempt in 0..=cas_retries {
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?; ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? { let read_epoch = match publication {
Some((publication_store, expected_publication_epoch)) => {
let epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
return Err(Error::other("data usage publication epoch changed before snapshot read"));
}
Some(epoch)
}
None => None,
};
let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
let (mut data_usage_info, revision) = match loaded_snapshot {
Some((data_usage_info, revision)) => (data_usage_info, Some(revision)), Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
None => match missing_seed { None => match missing_seed {
Some(data_usage_info) => (data_usage_info.clone(), None), Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -733,6 +878,22 @@ where
}, },
}; };
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?; ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
let publication_guard = match publication {
Some((publication_store, expected_publication_epoch)) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if expected_publication_epoch
.or(read_epoch)
.is_some_and(|expected| expected != publication_epoch)
{
return Err(Error::other("data usage publication epoch changed before snapshot commit"));
}
Some(guard)
}
None => None,
};
let save_result = store let save_result = store
.put_object( .put_object(
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
@@ -745,6 +906,7 @@ where
}, },
) )
.await; .await;
drop(publication_guard);
match save_result { match save_result {
Ok(_) => return Ok(()), Ok(_) => return Ok(()),
Err(err) => { Err(err) => {
@@ -2286,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::layout::endpoints::EndpointServerPools;
use crate::runtime::instance::InstanceContext;
use crate::storage_api_contracts::object::ObjectIO as _; use crate::storage_api_contracts::object::ObjectIO as _;
use rustfs_data_usage::BucketUsageInfo; use rustfs_data_usage::BucketUsageInfo;
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey}; use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2308,6 +2472,7 @@ mod tests {
error_after_commit_put: Option<usize>, error_after_commit_put: Option<usize>,
advance_time_on_put: Option<Duration>, advance_time_on_put: Option<Duration>,
advance_time_after_get: Option<(UsageObjectSlot, Duration)>, advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
advance_time_before_put: Option<(usize, Duration)>, advance_time_before_put: Option<(usize, Duration)>,
advance_time_after_put: Option<(usize, Duration)>, advance_time_after_put: Option<(usize, Duration)>,
put_count: usize, put_count: usize,
@@ -2385,10 +2550,21 @@ mod tests {
} }
_ => None, _ => None,
}; };
let advance_publication_epoch = match state.advance_publication_epoch_after_get {
Some((expected_slot, ref ctx)) if expected_slot == slot => {
let ctx = Arc::clone(ctx);
state.advance_publication_epoch_after_get = None;
Some(ctx)
}
_ => None,
};
drop(state); drop(state);
if let Some(duration) = advance { if let Some(duration) = advance {
tokio::time::advance(duration).await; tokio::time::advance(duration).await;
} }
if let Some(ctx) = advance_publication_epoch {
ctx.advance_data_movement_operation_epoch();
}
Ok(crate::object_api::GetObjectReader { Ok(crate::object_api::GetObjectReader {
stream: Box::new(Cursor::new(data)), stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo { object_info: ObjectInfo {
@@ -2589,6 +2765,23 @@ mod tests {
.to_string() .to_string()
} }
fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: TokioMutex::new(()),
pool_meta_save_gate: TokioMutex::new(()),
ctx,
bucket_fence_registry: Arc::default(),
})
}
#[test] #[test]
fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() { fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
// `to_object_err` rewrites the raw storage variants before they reach // `to_object_err` rewrites the raw storage variants before they reach
@@ -4333,7 +4526,15 @@ mod tests {
.expect("namespace lock acquisition should not fail") .expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired"), .expect("namespace lock should be acquired"),
); );
let store = Arc::new(UsageCasStore::default()); let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((encoded.clone(), 1)),
backup_object: Some((encoded, 1)),
..Default::default()
}),
});
let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now()); let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
let mut snapshot_cache = data_usage_snapshot_cache().write().await; let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot { *snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4432,21 +4633,17 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() { async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
let store = Arc::new(UsageCasStore::default()); let store = Arc::new(UsageCasStore::default());
remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
.await .await
.expect("bucket removal should create both usage fences"); .expect("bucket removal should remain a no-op without a usage baseline");
let state = store.state.lock().await; let state = store.state.lock().await;
assert_eq!(state.put_count, 2); assert_eq!(state.put_count, 0);
for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() { assert!(state.object.is_none());
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode"); assert!(state.backup_object.is_none());
assert_eq!(*revision, 1);
assert!(saved.last_update.is_some());
assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
}
} }
#[tokio::test] #[tokio::test]
@@ -4598,6 +4795,39 @@ mod tests {
assert_eq!(backup_err, Error::PreconditionFailed); assert_eq!(backup_err, Error::PreconditionFailed);
} }
#[tokio::test]
async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
let ctx = Arc::new(InstanceContext::new());
let publication_store = build_publication_store(ctx.clone());
let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
..Default::default()
}),
});
let expected_epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.expect("idle publication store should admit the initial read");
let err = remove_bucket_usage_from_object_with_retries_and_publication(
store.as_ref(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
"bucket-a",
0,
None,
None,
Some((publication_store.as_ref(), Some(expected_epoch))),
)
.await
.expect_err("a movement epoch flip during the read must fence the commit");
assert!(err.to_string().contains("epoch changed"));
assert_eq!(store.state.lock().await.put_count, 0);
}
#[tokio::test] #[tokio::test]
async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() { async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
+76 -8
View File
@@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager}; use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use s3s::region::Region; use s3s::region::Region;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::sync::{Arc, OnceLock}; use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{OnceCell, RwLock}; use tokio::sync::{OnceCell, RwLock};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use uuid::Uuid; use uuid::Uuid;
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
/// Runtime state owned by a single `ECStore` instance. /// Runtime state owned by a single `ECStore` instance.
/// ///
/// This is intentionally minimal in the first migration slice; subsequent /// This is intentionally minimal in the first migration slice; subsequent
@@ -160,10 +167,22 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance. /// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static. /// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>, background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and /// Serializes data-movement transitions with scanner publication commits.
/// a subsequent restart. Readers are held across one object side effect; /// Readers are held across one publication commit; movement transitions
/// the transition path takes the writer after cancelling the routine. /// take the writer at their durable state commit boundary.
decommission_operation_gate: Arc<RwLock<()>>, data_movement_operation_gate: Arc<RwLock<()>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
data_movement_operation_epoch: AtomicU64,
/// Once the admission epoch reaches its reserved terminal value, no new
/// publication may be admitted. Keeping this state separate from the
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
/// Resolves object-encryption material at the application boundary. /// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>, object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>, tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -204,7 +223,10 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())), local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None), bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(), background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())), data_movement_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
object_encryption_resolver: OnceLock::new(), object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -223,8 +245,54 @@ impl InstanceContext {
self.lock_manager.clone() self.lock_manager.clone()
} }
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> { pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate) Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
SCANNER_PUBLICATION_STATE_BLOCKED
} else {
SCANNER_PUBLICATION_STATE_ALLOWED
},
Ordering::Release,
);
}
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
result
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
self.data_movement_operation_epoch_exhausted
.store(epoch == u64::MAX, Ordering::Release);
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
} }
/// Install the application-owned object-encryption resolver once. /// Install the application-owned object-encryption resolver once.
@@ -48,12 +48,12 @@ fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usi
.unwrap_or_default() .unwrap_or_default()
} }
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) { fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
if persisted.id.is_empty() && persisted.pool_stats.is_empty() { if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
clear_rebalance_status_refresh(current); return clear_rebalance_status_refresh(current);
return;
} }
let before = current.clone();
match current.as_mut() { match current.as_mut() {
Some(current_meta) => { Some(current_meta) => {
if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
@@ -66,14 +66,41 @@ fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted
*current = Some(persisted); *current = Some(persisted);
} }
} }
match (before.as_ref(), current.as_ref()) {
(None, None) => false,
(None, Some(_)) | (Some(_), None) => true,
(Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
}
} }
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) { fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) { if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
*current = None; current.take().is_some()
} else {
false
} }
} }
fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
let Some(current) = current else {
return true;
};
current.id != persisted.id
|| current.stopped_at != persisted.stopped_at
|| current.pool_stats.len() != persisted.pool_stats.len()
|| current
.pool_stats
.iter()
.zip(persisted.pool_stats.iter())
.any(|(current, persisted)| {
current.participating != persisted.participating
|| current.info.status != persisted.info.status
|| current.info.stopping != persisted.info.stopping
})
}
impl ECStore { impl ECStore {
pub(super) async fn save_rebalance_meta_with_merge<S>( pub(super) async fn save_rebalance_meta_with_merge<S>(
&self, &self,
@@ -121,7 +148,10 @@ impl ECStore {
"Loading rebalance metadata" "Loading rebalance metadata"
); );
let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?; let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if resolve_rebalance_meta_load_result(meta.load(pool).await)? { if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
{ {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -130,6 +160,10 @@ impl ECStore {
drop(rebalance_meta); drop(rebalance_meta);
} }
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?; resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
debug!( debug!(
event = EVENT_REBALANCE_STATE, event = EVENT_REBALANCE_STATE,
@@ -139,10 +173,15 @@ impl ECStore {
"Loaded rebalance metadata" "Loaded rebalance metadata"
); );
} else { } else {
let movement_changed = self.rebalance_meta.read().await.is_some();
{ {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
*rebalance_meta = None; *rebalance_meta = None;
} }
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
debug!( debug!(
event = EVENT_REBALANCE_STATE, event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
@@ -160,14 +199,20 @@ impl ECStore {
pub async fn refresh_rebalance_status_meta(&self) -> Result<()> { pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?; let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
let mut persisted = RebalanceMeta::new(); let mut persisted = RebalanceMeta::new();
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
match persisted.load(pool).await { match persisted.load(pool).await {
Ok(()) => { Ok(()) => {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
merge_rebalance_status_refresh(&mut rebalance_meta, persisted); if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
self.ctx.advance_data_movement_operation_epoch();
}
} }
Err(Error::ConfigNotFound) => { Err(Error::ConfigNotFound) => {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
clear_rebalance_status_refresh(&mut rebalance_meta); if clear_rebalance_status_refresh(&mut rebalance_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
} }
Err(err) => { Err(err) => {
return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}"))); return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
@@ -349,6 +394,8 @@ impl ECStore {
#[tracing::instrument(skip(self, bucktes))] #[tracing::instrument(skip(self, bucktes))]
pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> { pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
let _start_guard = self.start_gate.lock().await; let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let decommission_running = self.is_decommission_running().await; let decommission_running = self.is_decommission_running().await;
{ {
@@ -356,12 +403,16 @@ impl ECStore {
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?; validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
} }
self.init_rebalance_meta(bucktes).await let id = self.init_rebalance_meta(bucktes).await?;
self.ctx.advance_data_movement_operation_epoch();
Ok(id)
} }
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> { pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
let _start_guard = self.start_gate.lock().await; let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
{ {
let rebalance_meta = self.rebalance_meta.read().await; let rebalance_meta = self.rebalance_meta.read().await;
@@ -379,7 +430,10 @@ impl ECStore {
} }
} }
self.start_rebalance().await if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
} }
pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> { pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
@@ -548,6 +602,9 @@ impl ECStore {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> { pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = { let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id) stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
@@ -560,6 +617,7 @@ impl ECStore {
.await, .await,
"stop_rebalance", "stop_rebalance",
)?; )?;
self.ctx.advance_data_movement_operation_epoch();
} }
Ok(()) Ok(())
@@ -570,6 +628,9 @@ impl ECStore {
expected_id: Option<&str>, expected_id: Option<&str>,
start_error: String, start_error: String,
) -> Result<()> { ) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = { let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
rollback_rebalance_start_meta_snapshot_for_id( rollback_rebalance_start_meta_snapshot_for_id(
@@ -587,6 +648,7 @@ impl ECStore {
.await, .await,
"rollback_rebalance_start", "rollback_rebalance_start",
)?; )?;
self.ctx.advance_data_movement_operation_epoch();
} }
Ok(()) Ok(())
@@ -597,6 +659,8 @@ impl ECStore {
return Ok(()); return Ok(());
} }
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let encoded_error = encode_rebalance_stop_propagation_record(&record); let encoded_error = encode_rebalance_stop_propagation_record(&record);
let meta_to_save = { let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -610,6 +674,7 @@ impl ECStore {
.await, .await,
"record_rebalance_stop_propagation", "record_rebalance_stop_propagation",
)?; )?;
self.ctx.advance_data_movement_operation_epoch();
} }
Ok(()) Ok(())
@@ -682,7 +747,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
merge_rebalance_status_refresh(&mut current, persisted); assert!(merge_rebalance_status_refresh(&mut current, persisted));
let refreshed = current.as_ref().expect("refresh should keep rebalance metadata"); let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed); assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
@@ -721,7 +786,7 @@ mod tests {
..Default::default() ..Default::default()
}; };
merge_rebalance_status_refresh(&mut current, persisted); assert!(!merge_rebalance_status_refresh(&mut current, persisted));
assert!( assert!(
current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(), current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
@@ -42,6 +42,15 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
impl ECStore { impl ECStore {
#[tracing::instrument(skip_all)] #[tracing::instrument(skip_all)]
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> { pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
pub(super) async fn start_rebalance_inner(self: &Arc<Self>) -> Result<bool> {
info!( info!(
event = EVENT_REBALANCE_STATE, event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
@@ -55,6 +64,7 @@ impl ECStore {
let cancel_tx = CancellationToken::new(); let cancel_tx = CancellationToken::new();
let rx = cancel_tx.clone(); let rx = cancel_tx.clone();
let mut meta_to_save = None; let mut meta_to_save = None;
let mut movement_changed = false;
{ {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -72,14 +82,16 @@ impl ECStore {
reason = "already_in_progress", reason = "already_in_progress",
"Skipped duplicate rebalance start" "Skipped duplicate rebalance start"
); );
return Ok(()); return Ok(false);
} }
let now = OffsetDateTime::now_utc(); let now = OffsetDateTime::now_utc();
if complete_rebalance_pools_at_goal(meta, now) { if complete_rebalance_pools_at_goal(meta, now) {
meta_to_save = Some(meta.clone()); meta_to_save = Some(meta.clone());
movement_changed = true;
} }
if complete_rebalance_pools_with_empty_queue(meta, now) { if complete_rebalance_pools_with_empty_queue(meta, now) {
meta_to_save = Some(meta.clone()); meta_to_save = Some(meta.clone());
movement_changed = true;
} }
meta.cancel = Some(cancel_tx); meta.cancel = Some(cancel_tx);
@@ -118,7 +130,7 @@ impl ECStore {
reason = "no_participants", reason = "no_participants",
"Skipped rebalance start because no pools are participating" "Skipped rebalance start because no pools are participating"
); );
return Ok(()); return Ok(movement_changed);
} }
let mut workers_started = 0usize; let mut workers_started = 0usize;
@@ -186,7 +198,7 @@ impl ECStore {
reason = "no_local_participants", reason = "no_local_participants",
"Skipped rebalance start because no local pools are participating" "Skipped rebalance start because no local pools are participating"
); );
return Ok(()); return Ok(movement_changed);
} }
info!( info!(
@@ -197,7 +209,7 @@ impl ECStore {
worker_count = workers_started, worker_count = workers_started,
"Rebalance started" "Rebalance started"
); );
Ok(()) Ok(true)
} }
#[tracing::instrument(skip(self, rx))] #[tracing::instrument(skip(self, rx))]
@@ -214,53 +226,77 @@ impl ECStore {
let mut quit = false; let mut quit = false;
loop { loop {
let mut terminal_state_saved = false;
tokio::select! { tokio::select! {
result = done_rx.recv() => { result = done_rx.recv() => {
quit = true; quit = true;
let now = OffsetDateTime::now_utc(); let now = OffsetDateTime::now_utc();
let terminal_event = classify_rebalance_terminal_event(result, now); let terminal_event = classify_rebalance_terminal_event(result, now);
msg = terminal_event.message().to_string(); msg = terminal_event.message().to_string();
let mut rebalance_meta = store.rebalance_meta.write().await; let movement_gate = store.ctx.data_movement_operation_gate();
if let Some(meta) = rebalance_meta.as_mut() { let movement_guard = movement_gate.write().await;
let meta_stopped = meta.stopped_at.is_some(); let previous_meta = store.rebalance_meta.read().await.clone();
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { let terminal_state_present = {
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. }) let mut rebalance_meta = store.rebalance_meta.write().await;
&& has_rebalance_cleanup_warnings(pool_stat) if let Some(meta) = rebalance_meta.as_mut() {
{ let meta_stopped = meta.stopped_at.is_some();
pool_stat.info.stopping = false; if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
pool_stat.info.status = RebalStatus::Failed; if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
pool_stat.info.end_time = Some(now); && has_rebalance_cleanup_warnings(pool_stat)
pool_stat.info.last_error = Some( {
pool_stat pool_stat.info.stopping = false;
.cleanup_warnings pool_stat.info.status = RebalStatus::Failed;
.last_message pool_stat.info.end_time = Some(now);
.clone() pool_stat.info.last_error = Some(
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()), pool_stat
); .cleanup_warnings
} else if should_preserve_rebalance_stopped_state( .last_message
meta_stopped, .clone()
pool_stat.info.status, .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
&terminal_event, );
) { } else if should_preserve_rebalance_stopped_state(
debug!( meta_stopped,
event = EVENT_REBALANCE_STATE, pool_stat.info.status,
component = LOG_COMPONENT_ECSTORE, &terminal_event,
subsystem = LOG_SUBSYSTEM_REBALANCE, ) {
pool_index, debug!(
state = "stopped_preserved", event = EVENT_REBALANCE_STATE,
"Preserved stopped rebalance status" component = LOG_COMPONENT_ECSTORE,
); subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
}
true
} else { } else {
pool_stat.info.stopping = false; false
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
} }
} else {
false
} }
};
if terminal_state_present {
if let Err(err) = store.save_rebalance_stats_inner(pool_index, RebalSaveOpt::Stats).await {
let mut rebalance_meta = store.rebalance_meta.write().await;
*rebalance_meta = previous_meta;
drop(movement_guard);
return Err(Error::other(format!(
"rebalance terminal state save failed for pool {pool_index}: {err}"
)));
}
store.ctx.advance_data_movement_operation_epoch();
terminal_state_saved = true;
} }
} }
_ = timer.tick() => { _ = timer.tick() => {
@@ -269,7 +305,7 @@ impl ECStore {
} }
} }
if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await { if !terminal_state_saved && let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}")); let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
error!("{} err: {:?}", msg, wrapped); error!("{} err: {:?}", msg, wrapped);
if quit { if quit {
@@ -590,16 +626,14 @@ impl ECStore {
meta.percent_free_goal, meta.percent_free_goal,
) )
{ {
pool_stat.info.status = RebalStatus::Completed;
pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
info!( info!(
event = EVENT_REBALANCE_STATE, event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE, component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE, subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index, pool_index,
state = "completed", state = "completion_ready",
percent_free = pfi, percent_free = pfi,
"Marked rebalance pool completed" "Rebalance pool reached completion goal"
); );
return true; return true;
} }
@@ -612,6 +646,12 @@ impl ECStore {
impl ECStore { impl ECStore {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> { pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
self.save_rebalance_stats_inner(pool_idx, opt).await
}
pub(super) async fn save_rebalance_stats_inner(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let meta_to_save = { let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
let Some(meta) = rebalance_meta.as_mut() else { let Some(meta) = rebalance_meta.as_mut() else {
+20
View File
@@ -3614,6 +3614,26 @@ impl SetDisks {
&self.ctx &self.ctx
} }
/// Admit one short scanner cache publication under this set's instance
/// movement fence. The caller must hold the returned guard through its
/// final conditional cache write; no scan-round work belongs under it.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.scanner_publication_state_allowed() {
let epoch = self.ctx.data_movement_operation_epoch();
return Some((operation_guard, epoch));
}
// The owner deliberately marks the cached state UNKNOWN after every
// movement epoch advance. Do not strand remote scanner writers in that
// state: release this guard before asking the storage owner to refresh
// its durable movement snapshot, since the owner uses the same gate.
drop(operation_guard);
let owner = runtime_sources::object_store_handle().filter(|owner| Arc::ptr_eq(&owner.ctx, &self.ctx))?;
owner.scanner_data_usage_publication_admission_guard().await
}
/// Whether both sets' namespace-lock implementations cover the same object key. /// Whether both sets' namespace-lock implementations cover the same object key.
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool { pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) { match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
+1 -1
View File
@@ -327,7 +327,7 @@ impl ECStore {
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> { async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
run_bucket_usage_cleanup(guard, bucket, async { run_bucket_usage_cleanup(guard, bucket, async {
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?; crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
}) })
.await .await
} }
+5
View File
@@ -495,6 +495,11 @@ impl ECStore {
); );
} }
// Initialize the storage-owned scanner publication state only after
// both movement metadata sources have been loaded. SetDisks cache
// writers remain fail-closed until this snapshot is available.
let _ = self.scanner_data_usage_publication_blocked().await;
let pools = installed_pool_meta.return_resumable_pools(); let pools = installed_pool_meta.return_resumable_pools();
let mut pool_indices = Vec::with_capacity(pools.len()); let mut pool_indices = Vec::with_capacity(pools.len());
+149 -5
View File
@@ -44,7 +44,7 @@ use crate::error::{
use crate::runtime::global::DISK_RESERVE_FRACTION; use crate::runtime::global::DISK_RESERVE_FRACTION;
use crate::runtime::instance::InstanceContext; use crate::runtime::instance::InstanceContext;
use crate::runtime::sources as runtime_sources; use crate::runtime::sources as runtime_sources;
use crate::services::rebalance::RebalanceMeta; use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::storage_api_contracts::{ use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -349,16 +349,81 @@ impl ECStore {
/// remain suspended until an operator clears or retries them, so they are /// remain suspended until an operator clears or retries them, so they are
/// a publication barrier even after the worker has stopped. /// a publication barrier even after the worker has stopped.
pub async fn scanner_data_usage_publication_blocked(&self) -> bool { pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
if self.scanner_data_movement_active().await { let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await;
self.scanner_data_usage_publication_snapshot_blocked().await
}
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
if self.ctx.data_movement_operation_epoch_exhausted() {
self.ctx.set_scanner_publication_state(true);
return true; return true;
} }
let decommission_cancelers = self.decommission_cancelers.read().await;
let decommission_active = decommission_cancelers
.iter()
.any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
let pool_meta = self.pool_meta.read().await; let pool_meta = self.pool_meta.read().await;
pool_meta.pools.iter().any(|pool| { let decommission_active = decommission_active
|| pool_meta.pools.iter().any(|pool| {
pool.decommission
.as_ref()
.is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
});
let decommission_terminal = pool_meta.pools.iter().any(|pool| {
pool.decommission pool.decommission
.as_ref() .as_ref()
.is_some_and(|info| !info.queued && (info.failed || info.canceled)) .is_some_and(|info| !info.queued && (info.failed || info.canceled))
}) });
drop(pool_meta);
let rebalance_active = self
.rebalance_meta
.read()
.await
.as_ref()
.is_some_and(is_rebalance_conflicting_with_decommission);
let blocked = decommission_active || decommission_terminal || rebalance_active;
self.ctx.set_scanner_publication_state(blocked);
blocked
}
/// Admit one short data-usage publication commit under the same
/// per-instance gate used by decommission side effects and transitions.
/// The epoch is sampled while the read guard is held, so a transition
/// cannot cross this admission without waiting for the commit to finish.
pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
let epoch = self.ctx.data_movement_operation_epoch();
(operation_guard, epoch)
}
/// Acquire the movement gate and inspect the movement owner once. The
/// state inspection is performed after acquiring the read guard so a
/// transition cannot update its durable state between the check and the
/// publication commit.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_operation_epoch_exhausted() {
return None;
}
if self.scanner_data_usage_publication_snapshot_blocked().await {
return None;
}
Some((operation_guard, self.ctx.data_movement_operation_epoch()))
}
/// Capture the current publication epoch without holding the movement
/// gate across backend I/O. Callers must re-admit the same epoch before a
/// mutation commits.
pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
drop(operation_guard);
Some(epoch)
} }
} }
@@ -995,6 +1060,85 @@ mod tests {
} }
} }
#[tokio::test]
async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let operation_gate = store.ctx.data_movement_operation_gate();
let movement_guard = operation_gate.write().await;
let pending = {
let store = store.clone();
tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
};
tokio::task::yield_now().await;
assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
drop(movement_guard);
let (_, epoch) = pending
.await
.expect("publication admission task should not panic")
.expect("idle store should admit publication");
assert_eq!(epoch, 0);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
let (_, next_epoch) = store
.scanner_data_usage_publication_admission_guard()
.await
.expect("idle store should admit the next publication");
assert_eq!(next_epoch, 1);
}
#[tokio::test]
async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let epoch = store
.scanner_data_usage_publication_epoch()
.await
.expect("idle store should expose a publication epoch");
assert_eq!(epoch, 0);
let operation_gate = store.ctx.data_movement_operation_gate();
let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
.await
.expect("epoch capture must not hold the movement gate across backend I/O");
}
#[tokio::test]
async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
*store.rebalance_meta.write().await = Some(RebalanceMeta {
pool_stats: vec![crate::services::rebalance::RebalanceStats {
participating: true,
info: crate::services::rebalance::RebalanceInfo {
status: crate::services::rebalance::RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
});
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"active rebalance must fail closed at the storage-owned admission boundary"
);
}
#[tokio::test]
async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"publication must fail closed at the reserved terminal epoch"
);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(store.scanner_data_usage_publication_blocked().await);
}
// The object graph is the isolation carrier: two ECStore instances holding // The object graph is the isolation carrier: two ECStore instances holding
// distinct contexts report independent erasure state through their real // distinct contexts report independent erasure state through their real
// `&self` accessors — no cross-contamination. // `&self` accessors — no cross-contamination.
+14 -2
View File
@@ -694,6 +694,11 @@ impl ECStore {
/// callers only trigger missing-worker recovery after a real state change; /// callers only trigger missing-worker recovery after a real state change;
/// delayed snapshots are merged monotonically and never blind-assigned. /// delayed snapshots are merged monotonically and never blind-assigned.
pub async fn reload_pool_meta(&self) -> Result<bool> { pub async fn reload_pool_meta(&self) -> Result<bool> {
// Serialize the durable reload with local movement transitions. Loading
// before acquiring this gate would allow a stale disk snapshot to
// overwrite a newer local transition after the writer commits.
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut reloaded = PoolMeta::default(); let mut reloaded = PoolMeta::default();
resolve_store_rebalance_pool_meta_reload_result( resolve_store_rebalance_pool_meta_reload_result(
reloaded.load(self.pools[0].clone(), self.pools.clone()).await, reloaded.load(self.pools[0].clone(), self.pools.clone()).await,
@@ -701,15 +706,22 @@ impl ECStore {
)?; )?;
// Lock order: release the decommission_cancelers guard before taking // Lock order: release the decommission_cancelers guard before taking
// the pool_meta write guard; neither is held across the disk read. // the pool_meta write guard; neither is held without the movement gate.
let active_workers = { let active_workers = {
let cancelers = self.decommission_cancelers.read().await; let cancelers = self.decommission_cancelers.read().await;
cancelers.iter().map(Option::is_some).collect::<Vec<_>>() cancelers
.iter()
.map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
.collect::<Vec<_>>()
}; };
let incoming_has_pools = !reloaded.pools.is_empty(); let incoming_has_pools = !reloaded.pools.is_empty();
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
let movement_before = pool_meta.clone();
let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers); let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
if !merged_newer && !incoming_has_pools { if !merged_newer && !incoming_has_pools {
warn!( warn!(
+17 -9
View File
@@ -36,13 +36,13 @@ use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use tokio::time::{Duration, Instant, sleep, timeout}; use tokio::time::{Duration, Instant, sleep, timeout};
use tracing::{debug, warn}; use tracing::{debug, warn};
use crate::ScannerObjectIO;
use crate::storage_api::owner::HTTPPreconditions; use crate::storage_api::owner::HTTPPreconditions;
use crate::{ use crate::{
BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig, BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig,
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, TRANSITION_COMPLETE, save_config, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError,
save_config_with_preconditions, storageclass, TRANSITION_COMPLETE, save_config, save_config_with_preconditions, scanner_publication_admission_for_epoch, storageclass,
}; };
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
// Data usage constants // Data usage constants
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
@@ -119,9 +119,13 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag")))?; .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag")))?;
Ok((Some(reader.read_all().await?), revision)) Ok((Some(reader.read_all().await?), revision))
} }
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { Err(
Ok((None, DataUsageCacheRevision::Missing)) Error::ConfigNotFound
} | Error::FileNotFound
| Error::VolumeNotFound
| Error::ObjectNotFound(_, _)
| Error::BucketNotFound(_),
) => Ok((None, DataUsageCacheRevision::Missing)),
Err(err) => Err(err), Err(err) => Err(err),
} }
} }
@@ -147,9 +151,13 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
.filter(|etag| !etag.is_empty()) .filter(|etag| !etag.is_empty())
.map(DataUsageCacheRevision::Etag) .map(DataUsageCacheRevision::Etag)
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))), .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { Err(
Ok(DataUsageCacheRevision::Missing) Error::ConfigNotFound
} | Error::FileNotFound
| Error::VolumeNotFound
| Error::ObjectNotFound(_, _)
| Error::BucketNotFound(_),
) => Ok(DataUsageCacheRevision::Missing),
Err(err) => Err(err), Err(err) => Err(err),
} }
} }
@@ -15,6 +15,42 @@
use super::*; use super::*;
use std::sync::Arc; use std::sync::Arc;
#[derive(Debug)]
struct CachePublicationAdmissionUnavailable;
impl std::fmt::Display for CachePublicationAdmissionUnavailable {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("scanner cache publication admission is unavailable")
}
}
impl std::error::Error for CachePublicationAdmissionUnavailable {}
fn cache_publication_admission_unavailable() -> Error {
Error::Io(std::io::Error::other(CachePublicationAdmissionUnavailable))
}
fn cache_publication_epoch_changed() -> Error {
Error::other(SCANNER_PUBLICATION_EPOCH_CHANGED)
}
fn is_cache_publication_admission_unavailable(error: &StorageError) -> bool {
matches!(
error,
StorageError::Io(io_error)
if io_error
.get_ref()
.is_some_and(|source| source.downcast_ref::<CachePublicationAdmissionUnavailable>().is_some())
)
}
fn is_cache_publication_epoch_changed(error: &StorageError) -> bool {
matches!(
error,
StorageError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED
)
}
pub(super) enum DataUsageCacheLoadAttempt { pub(super) enum DataUsageCacheLoadAttempt {
Loaded { Loaded {
cache: Box<DataUsageCache>, cache: Box<DataUsageCache>,
@@ -368,6 +404,12 @@ impl DataUsageCache {
fn should_retry_save_error(err: &StorageError) -> bool { fn should_retry_save_error(err: &StorageError) -> bool {
// Usage-cache files are best-effort scanner checkpoints. Retrying namespace // Usage-cache files are best-effort scanner checkpoints. Retrying namespace
// lock failures immediately only adds more lock traffic to the same hot object. // lock failures immediately only adds more lock traffic to the same hot object.
if is_cache_publication_admission_unavailable(err) {
return false;
}
if is_cache_publication_epoch_changed(err) {
return false;
}
!matches!( !matches!(
err, err,
StorageError::Lock(_) StorageError::Lock(_)
@@ -423,13 +465,14 @@ impl DataUsageCache {
Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string()))) Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string())))
} }
async fn save_path_with_retry<S: ScannerObjectIO>( async fn save_path_with_retry<S: ScannerObjectIO + ScannerConfigObjectDelete>(
store: Arc<S>, store: Arc<S>,
path: &str, path: &str,
buf: &[u8], buf: &[u8],
timeout_duration: Duration, timeout_duration: Duration,
max_retries: u32, max_retries: u32,
revision: Option<DataUsageCacheRevision>, revision: Option<DataUsageCacheRevision>,
expected_epoch: Option<u64>,
) -> StorageResult<()> { ) -> StorageResult<()> {
Self::ensure_cache_save_metrics_registered(); Self::ensure_cache_save_metrics_registered();
let path_type = Self::cache_path_type(path); let path_type = Self::cache_path_type(path);
@@ -441,6 +484,17 @@ impl DataUsageCache {
let buf_clone = buf.to_vec(); let buf_clone = buf.to_vec();
let revision = revision.clone(); let revision = revision.clone();
async move { async move {
let publication_admission = match expected_epoch {
Some(expected_epoch) => scanner_publication_admission_for_epoch(store_clone.clone(), expected_epoch).await,
None => store_clone.scanner_data_usage_publication_admission().await,
};
let Some(_publication_admission) = publication_admission else {
return Err(if expected_epoch.is_some() {
cache_publication_epoch_changed()
} else {
cache_publication_admission_unavailable()
});
};
if let Some(revision) = revision { if let Some(revision) = revision {
save_config_with_preconditions(store_clone, &path_clone, buf_clone, revision.preconditions()).await?; save_config_with_preconditions(store_clone, &path_clone, buf_clone, revision.preconditions()).await?;
} else { } else {
@@ -454,6 +508,14 @@ impl DataUsageCache {
return Ok(()); return Ok(());
}; };
// An epoch-specific admission failure is authoritative: reconciling
// identical bytes cannot prove that this snapshot belongs to the
// captured movement epoch. Do not turn that fence failure into a
// successful stale publication.
if is_cache_publication_epoch_changed(&save_err) {
return Err(save_err);
}
for attempt in 0..=max_retries { for attempt in 0..=max_retries {
let reconcile = timeout(timeout_duration, async { let reconcile = timeout(timeout_duration, async {
let mut reader = store let mut reader = store
@@ -486,24 +548,36 @@ impl DataUsageCache {
Err(save_err) Err(save_err)
} }
pub async fn save<S: ScannerObjectIO>(&self, store: Arc<S>, name: &str) -> StorageResult<()> { pub async fn save<S: ScannerObjectIO + ScannerConfigObjectDelete>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
self.save_inner(store, name, None).await self.save_inner(store, name, None, None).await
} }
pub(crate) async fn save_with_revisions<S: ScannerObjectIO>( #[cfg(test)]
pub(crate) async fn save_with_revisions<S: ScannerObjectIO + ScannerConfigObjectDelete>(
&self, &self,
store: Arc<S>, store: Arc<S>,
name: &str, name: &str,
revisions: &DataUsageCacheRevisions, revisions: &DataUsageCacheRevisions,
) -> StorageResult<()> { ) -> StorageResult<()> {
self.save_inner(store, name, Some(revisions)).await self.save_inner(store, name, Some(revisions), None).await
} }
async fn save_inner<S: ScannerObjectIO>( pub(crate) async fn save_with_revisions_for_epoch<S: ScannerObjectIO + ScannerConfigObjectDelete>(
&self,
store: Arc<S>,
name: &str,
revisions: &DataUsageCacheRevisions,
expected_epoch: u64,
) -> StorageResult<()> {
self.save_inner(store, name, Some(revisions), Some(expected_epoch)).await
}
async fn save_inner<S: ScannerObjectIO + ScannerConfigObjectDelete>(
&self, &self,
store: Arc<S>, store: Arc<S>,
name: &str, name: &str,
revisions: Option<&DataUsageCacheRevisions>, revisions: Option<&DataUsageCacheRevisions>,
expected_epoch: Option<u64>,
) -> StorageResult<()> { ) -> StorageResult<()> {
let mut buf = Vec::new(); let mut buf = Vec::new();
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?; self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
@@ -517,6 +591,7 @@ impl DataUsageCache {
timeout_duration, timeout_duration,
DATA_USAGE_CACHE_SAVE_RETRIES, DATA_USAGE_CACHE_SAVE_RETRIES,
revisions.map(|revisions| revisions.main.clone()), revisions.map(|revisions| revisions.main.clone()),
expected_epoch,
) )
.await?; .await?;
@@ -534,6 +609,7 @@ impl DataUsageCache {
backup_timeout_duration, backup_timeout_duration,
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES, DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
backup_revision, backup_revision,
expected_epoch,
) )
.await .await
{ {
@@ -548,6 +624,9 @@ impl DataUsageCache {
error = %e, error = %e,
"Scanner cache backup save failed" "Scanner cache backup save failed"
); );
if is_cache_publication_admission_unavailable(&e) || is_cache_publication_epoch_changed(&e) {
return Err(e);
}
} }
Ok(()) Ok(())
} }
@@ -198,6 +198,22 @@ impl ObjectIO for CacheReadStore {
} }
} }
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for CacheReadStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_opts: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl ObjectIO for AmbiguousCacheCommitStore { impl ObjectIO for AmbiguousCacheCommitStore {
type Error = Error; type Error = Error;
@@ -243,6 +259,22 @@ impl ObjectIO for AmbiguousCacheCommitStore {
} }
} }
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for AmbiguousCacheCommitStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_opts: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl ObjectIO for BackupFallbackStore { impl ObjectIO for BackupFallbackStore {
type Error = Error; type Error = Error;
@@ -314,6 +346,22 @@ impl ObjectIO for BackupFallbackStore {
} }
} }
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for BackupFallbackStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_opts: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
#[test] #[test]
fn cache_revisions_map_to_compare_and_swap_preconditions() { fn cache_revisions_map_to_compare_and_swap_preconditions() {
let missing = DataUsageCacheRevision::Missing.preconditions(); let missing = DataUsageCacheRevision::Missing.preconditions();
+124
View File
@@ -513,6 +513,75 @@ where
.await .await
} }
pub(crate) async fn save_config_with_publication_admission_for_epoch<S>(
api: Arc<S>,
file: &str,
data: Vec<u8>,
preconditions: HTTPPreconditions,
expected_epoch: u64,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO + ScannerConfigObjectDelete,
{
let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
save_config_with_preconditions(api, file, data, preconditions).await
}
pub(crate) const SCANNER_PUBLICATION_EPOCH_CHANGED: &str = "scanner publication epoch changed before commit";
pub(crate) fn scanner_publication_epoch_changed(error: &EcstoreError) -> bool {
matches!(
error,
EcstoreError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED
)
}
pub(crate) async fn delete_config_with_publication_admission_for_epoch<S>(
api: Arc<S>,
bucket: &str,
object: &str,
opts: ScannerObjectOptions,
expected_epoch: u64,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO + ScannerConfigObjectDelete,
{
let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
api.delete_config_object(bucket, object, opts).await
}
/// Capture the storage-owned publication epoch without retaining the read
/// guard across a potentially slow metadata read. Callers must compare this
/// token with a fresh admission immediately before their conditional write.
pub(crate) async fn scanner_publication_epoch<S>(api: Arc<S>) -> Option<u64>
where
S: ScannerConfigObjectDelete,
{
let admission = api.scanner_data_usage_publication_admission().await?;
Some(admission.epoch())
}
/// Re-admit a publication only when the storage-owned movement epoch is still
/// the one observed before the caller's metadata read. The returned guard
/// remains held through the caller's short conditional commit.
pub(crate) async fn scanner_publication_admission_for_epoch<S>(
api: Arc<S>,
expected_epoch: u64,
) -> Option<ScannerDataUsagePublicationAdmission>
where
S: ScannerConfigObjectDelete,
{
let admission = api.scanner_data_usage_publication_admission().await?;
if admission.epoch() != expected_epoch {
return None;
}
Some(admission)
}
pub(crate) async fn save_config_shared_with_preconditions<S>( pub(crate) async fn save_config_shared_with_preconditions<S>(
api: Arc<S>, api: Arc<S>,
file: &str, file: &str,
@@ -587,6 +656,39 @@ pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static {
object: &str, object: &str,
opts: ScannerObjectOptions, opts: ScannerObjectOptions,
) -> EcstoreResult<ScannerObjectInfo>; ) -> EcstoreResult<ScannerObjectInfo>;
/// Acquire storage-owned admission for one short data-usage publication
/// commit. Implementations without a storage-owned movement owner fail
/// closed; test fixtures opt into the explicit unfenced helper.
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
None
}
}
pub struct ScannerDataUsagePublicationAdmission {
epoch: u64,
_read_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
}
impl ScannerDataUsagePublicationAdmission {
#[cfg(test)]
pub(crate) fn unfenced() -> Self {
Self {
epoch: 0,
_read_guard: None,
}
}
fn fenced(read_guard: tokio::sync::OwnedRwLockReadGuard<()>, epoch: u64) -> Self {
Self {
epoch,
_read_guard: Some(read_guard),
}
}
pub(crate) fn epoch(&self) -> u64 {
self.epoch
}
} }
#[async_trait::async_trait] #[async_trait::async_trait]
@@ -599,6 +701,28 @@ impl ScannerConfigObjectDelete for ECStore {
) -> EcstoreResult<ScannerObjectInfo> { ) -> EcstoreResult<ScannerObjectInfo> {
ObjectOperations::delete_object(self, bucket, object, opts).await ObjectOperations::delete_object(self, bucket, object, opts).await
} }
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
}
}
#[async_trait::async_trait]
impl ScannerConfigObjectDelete for SetDisks {
async fn delete_config_object(
&self,
bucket: &str,
object: &str,
opts: ScannerObjectOptions,
) -> EcstoreResult<ScannerObjectInfo> {
ObjectOperations::delete_object(self, bucket, object, opts).await
}
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
}
} }
#[cfg(test)] #[cfg(test)]
+26 -1
View File
@@ -23,6 +23,7 @@ use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
use crate::{ use crate::{
DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo, DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo,
DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle, DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle,
scanner_publication_admission_for_epoch, scanner_publication_epoch,
}; };
use hmac::{Hmac, KeyInit, Mac}; use hmac::{Hmac, KeyInit, Mac};
use rustfs_common::heal_channel::HealScanMode; use rustfs_common::heal_channel::HealScanMode;
@@ -686,6 +687,9 @@ async fn scan_and_persist_local_bucket(
source.pool_index, source.set_index source.pool_index, source.set_index
)) ))
})?; })?;
let expected_publication_epoch = scanner_publication_epoch(set.clone()).await.ok_or_else(|| {
RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement")
})?;
let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]); let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source) let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
.await .await
@@ -708,6 +712,14 @@ async fn scan_and_persist_local_bucket(
"remote namespace scanner cache lock was lost before reusing the current snapshot", "remote namespace scanner cache lock was lost before reusing the current snapshot",
)); ));
} }
if scanner_publication_admission_for_epoch(set.clone(), expected_publication_epoch)
.await
.is_none()
{
return Err(RemoteScannerServerError::retry_bucket(
"remote namespace scanner cache publication epoch changed before reusing the current snapshot",
));
}
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
source, source,
scan_plan_digest, scan_plan_digest,
@@ -796,9 +808,22 @@ async fn scan_and_persist_local_bucket(
.await .await
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?;
let done_save = Metrics::time(Metric::SaveUsage); let done_save = Metrics::time(Metric::SaveUsage);
let save_result = cache.save_with_revisions(set, &cache_name, &revisions).await; // Each physical main/backup PUT must still prove the epoch captured before
// the scan. A movement transition that starts and ends during the scan
// therefore cannot admit the stale cache under the new epoch.
let save_result = cache
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch)
.await;
done_save(); done_save();
save_result.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache save failed: {err}")))?; save_result.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache save failed: {err}")))?;
if scanner_publication_admission_for_epoch(set, expected_publication_epoch)
.await
.is_none()
{
return Err(RemoteScannerServerError::retry_bucket(
"remote namespace scanner cache publication epoch changed after persistence",
));
}
validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store) validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store)
.await .await
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?;
+214 -33
View File
@@ -18,6 +18,7 @@ use std::future::Future;
use std::sync::Mutex as StdMutex; use std::sync::Mutex as StdMutex;
use std::sync::{Arc, LazyLock, RwLock}; use std::sync::{Arc, LazyLock, RwLock};
use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch};
use crate::data_usage_define::{ use crate::data_usage_define::{
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision, DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
@@ -30,8 +31,9 @@ use crate::runtime_config::{
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason}; use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob}; use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
use crate::scanner_io::{ use crate::scanner_io::{
ScannerCycleDeferReason, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending, ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified,
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation, dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed,
scanner_maintenance_generation,
}; };
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed}; use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard}; use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
@@ -66,10 +68,12 @@ use crate::storage_api::scan::{
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
}; };
use crate::{ use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
get_lifecycle_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache, ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
invalidate_data_usage_snapshot_cache, read_config, replace_bucket_usage_memory_from_info, save_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd, replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch,
scanner_publication_epoch, scanner_publication_epoch_changed,
}; };
const LOG_COMPONENT_SCANNER: &str = "scanner"; const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -346,6 +350,23 @@ fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
!info.is_complete_bucket_usage_snapshot() !info.is_complete_bucket_usage_snapshot()
} }
pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageInfo) -> bool {
if info.is_complete_bucket_usage_snapshot() {
return true;
}
// Pre-marker snapshots remain readable only when their legacy identity is
// complete: a timestamp, a scanner cycle, and an exact bucket cardinality.
// A current snapshot with only scanner_epoch/scanner_cycle (or an explicit
// incomplete marker) is not evidence of a durable usage baseline.
!info.usage_snapshot_complete
&& info.scanner_epoch.is_none()
&& info.usage_snapshot_converged != Some(false)
&& info.last_update.is_some()
&& info.scanner_cycle.is_some()
&& u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count)
}
fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool { fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool {
data_usage_info_is_cold(authoritative) data_usage_info_is_cold(authoritative)
|| observed.is_some_and(|observed| observed_data_usage_is_newer(observed, authoritative)) || observed.is_some_and(|observed| observed_data_usage_is_newer(observed, authoritative))
@@ -381,9 +402,18 @@ fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
.is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0) .is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0)
} }
#[cfg(test)]
async fn sync_data_usage_backup_from_primary( async fn sync_data_usage_backup_from_primary(
ctx: &CancellationToken, ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<(), EcstoreError> {
sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await
}
async fn sync_data_usage_backup_from_primary_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_publication_epoch: Option<u64>,
) -> Result<(), EcstoreError> { ) -> Result<(), EcstoreError> {
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
@@ -391,26 +421,62 @@ async fn sync_data_usage_backup_from_primary(
return Ok(()); return Ok(());
} }
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
expected_epoch
}
None => scanner_publication_epoch(storeapi.clone())
.await
.ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?,
};
let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
let primary = primary.ok_or_else(|| EcstoreError::other("authoritative data usage snapshot is missing"))?; let primary = primary.ok_or(EcstoreError::ConfigNotFound)?;
serde_json::from_slice::<DataUsageInfo>(&primary) let primary_info = serde_json::from_slice::<DataUsageInfo>(&primary)
.map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?; .map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?;
if !data_usage_info_has_persisted_baseline_identity(&primary_info) {
return Err(EcstoreError::other(
"authoritative data usage snapshot has no persisted baseline identity",
));
}
let primary = Bytes::from(primary); let primary = Bytes::from(primary);
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?; let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
if backup.as_deref() == Some(primary.as_ref()) { if backup.as_deref() == Some(primary.as_ref()) {
return Ok(()); if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_some()
{
return Ok(());
}
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
} }
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower)); let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
let save_result = save_config_shared_with_preconditions( let save_result = {
storeapi.clone(), let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
&backup_path, if retry < SCANNER_PERSIST_CAS_RETRIES {
primary.clone(), continue;
sha256hex, }
revision.preconditions(), return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
) };
.await; save_config_shared_with_preconditions(
storeapi.clone(),
&backup_path,
primary.clone(),
sha256hex,
revision.preconditions(),
)
.await
};
match save_result { match save_result {
Ok(_) => {} Ok(_) => {}
@@ -427,9 +493,20 @@ async fn sync_data_usage_backup_from_primary(
} }
let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
if current_primary.as_deref() == Some(primary.as_ref()) { if current_primary.as_deref() == Some(primary.as_ref())
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_some()
{
return Ok(()); return Ok(());
} }
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
if retry < SCANNER_PERSIST_CAS_RETRIES { if retry < SCANNER_PERSIST_CAS_RETRIES {
continue; continue;
} }
@@ -1052,7 +1129,7 @@ async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
lock_lost: LockLost, lock_lost: LockLost,
) -> bool ) -> bool
where where
Store: ScannerObjectIO, Store: ScannerObjectIO + ScannerConfigObjectDelete,
LockLost: Future<Output = ()>, LockLost: Future<Output = ()>,
{ {
let fence_ctx = ctx.child_token(); let fence_ctx = ctx.child_token();
@@ -1089,7 +1166,7 @@ async fn handle_scanner_cycle_deadline<Store>(
worker_stopped: bool, worker_stopped: bool,
guard: &mut NamespaceLockGuard, guard: &mut NamespaceLockGuard,
) where ) where
Store: ScannerObjectIO, Store: ScannerObjectIO + ScannerConfigObjectDelete,
{ {
let fenced = fence_scanner_epoch_after_cycle_timeout( let fenced = fence_scanner_epoch_after_cycle_timeout(
ctx, ctx,
@@ -1182,7 +1259,33 @@ async fn run_data_scanner_cycle_with_budget(
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
let mut background_heal_info = read_background_heal_info(storeapi.clone()).await; // Refresh the storage-owned movement snapshot before reading background
// heal state. A missing heal object yields an in-memory default; do not
// let that default influence a cycle while publication is blocked.
if storeapi.scanner_data_usage_publication_blocked().await {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
let background_heal_read = read_background_heal_info_with_epoch(storeapi.clone()).await;
match background_heal_read.status {
BackgroundHealInfoReadStatus::Blocked => {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
BackgroundHealInfoReadStatus::Transient => {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
BackgroundHealInfoReadStatus::Failed => {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
}
BackgroundHealInfoReadStatus::ErasureSd
| BackgroundHealInfoReadStatus::Loaded
| BackgroundHealInfoReadStatus::Missing => {}
}
let mut background_heal_info = background_heal_read.info;
let background_heal_epoch = background_heal_read.expected_epoch;
let scan_mode = get_cycle_scan_mode( let scan_mode = get_cycle_scan_mode(
cycle_info.current, cycle_info.current,
@@ -1209,11 +1312,23 @@ async fn run_data_scanner_cycle_with_budget(
configured_bitrot_cycle, configured_bitrot_cycle,
) { ) {
background_heal_info = new_heal_info.clone(); background_heal_info = new_heal_info.clone();
save_background_heal_info(storeapi.clone(), new_heal_info).await; save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
} }
let cycle_start = std::time::Instant::now(); let cycle_start = std::time::Instant::now();
let usage_persist_baseline = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { // Baseline reads are part of the same publication proof as the eventual
// scanner aggregate. Hold only the short storage-owned admission guard
// across this metadata read; the full bucket scan runs after it is
// released and carries the captured epoch forward.
let Some((baseline_publication_guard, baseline_publication_epoch)) =
storeapi.scanner_data_usage_publication_admission_guard().await
else {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
let usage_persist_baseline_result = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await;
drop(baseline_publication_guard);
let usage_persist_baseline = match usage_persist_baseline_result {
Ok((data, revision)) => DataUsagePersistBaseline { Ok((data, revision)) => DataUsagePersistBaseline {
data: data.map(Bytes::from), data: data.map(Bytes::from),
revision, revision,
@@ -1250,9 +1365,17 @@ async fn run_data_scanner_cycle_with_budget(
) )
.await; .await;
let publication_defer_reason = match &scan_result { let publication_defer_reason = match &scan_result {
Ok(result)
if result
.publication_epoch()
.is_some_and(|publication_epoch| publication_epoch != baseline_publication_epoch) =>
{
Some(ScannerCycleDeferReason::DataMovement)
}
Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await, Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
}; };
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
let usage_persist_outcome = match publication_defer_reason { let usage_persist_outcome = match publication_defer_reason {
Some(reason) => { Some(reason) => {
@@ -1267,12 +1390,13 @@ async fn run_data_scanner_cycle_with_budget(
let ctx_clone = ctx.clone(); let ctx_clone = ctx.clone();
let route_probe_store = storeapi.clone(); let route_probe_store = storeapi.clone();
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move { let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
ctx_clone, ctx_clone,
storeapi_clone, storeapi_clone,
receiver, receiver,
Some(leader_epoch), Some(leader_epoch),
Some(usage_persist_baseline), Some(usage_persist_baseline),
publication_epoch,
move || { move || {
let storeapi = route_probe_store.clone(); let storeapi = route_probe_store.clone();
async move { storeapi.scanner_data_usage_publication_blocked().await } async move { storeapi.scanner_data_usage_publication_blocked().await }
@@ -1344,7 +1468,7 @@ async fn run_data_scanner_cycle_with_budget(
if !ctx.is_cancelled() if !ctx.is_cancelled()
&& let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false) && let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false)
{ {
save_background_heal_info(storeapi.clone(), new_heal_info).await; save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
} }
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed; return ScannerCycleOutcome::Failed;
@@ -1377,19 +1501,29 @@ async fn run_data_scanner_cycle_with_budget(
"Scanner cycle is recovering to a newer durable cache generation" "Scanner cycle is recovering to a newer durable cache generation"
); );
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
let persisted = persist_required_scanner_cycle_floor( let persisted = persist_required_scanner_cycle_floor_for_epoch(
ctx, ctx,
storeapi.clone(), storeapi.clone(),
cycle_info, cycle_info,
cycle_revision, cycle_revision,
leader_epoch, leader_epoch,
required_cycle,
&mut cycle_metrics_guard, &mut cycle_metrics_guard,
ScannerCycleFloorOptions {
required_cycle,
expected_publication_epoch: publication_epoch,
},
) )
.await; .await;
return if persisted { return if persisted {
cycle_budget.mark_cycle_state_persisted(); cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial ScannerCycleOutcome::Partial
} else if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
} else { } else {
ScannerCycleOutcome::Failed ScannerCycleOutcome::Failed
}; };
@@ -1446,18 +1580,26 @@ async fn run_data_scanner_cycle_with_budget(
scan_cycle_partial_reason(budget_reason), scan_cycle_partial_reason(budget_reason),
scan_cycle_partial_source(budget_reason), scan_cycle_partial_source(budget_reason),
); );
let persisted = finalize_partial_scan_cycle( let persisted = finalize_partial_scan_cycle_for_epoch(
ctx, ctx,
storeapi.clone(), storeapi.clone(),
cycle_info, cycle_info,
cycle_revision, cycle_revision,
leader_epoch, leader_epoch,
&mut cycle_metrics_guard, &mut cycle_metrics_guard,
publication_epoch,
) )
.await; .await;
return if persisted { return if persisted {
cycle_budget.mark_cycle_state_persisted(); cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial ScannerCycleOutcome::Partial
} else if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
} else { } else {
ScannerCycleOutcome::Failed ScannerCycleOutcome::Failed
}; };
@@ -1531,18 +1673,26 @@ async fn run_data_scanner_cycle_with_budget(
); );
} }
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
let persisted = finalize_partial_scan_cycle( let persisted = finalize_partial_scan_cycle_for_epoch(
ctx, ctx,
storeapi.clone(), storeapi.clone(),
cycle_info, cycle_info,
cycle_revision, cycle_revision,
leader_epoch, leader_epoch,
&mut cycle_metrics_guard, &mut cycle_metrics_guard,
publication_epoch,
) )
.await; .await;
return if persisted { return if persisted {
cycle_budget.mark_cycle_state_persisted(); cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial ScannerCycleOutcome::Partial
} else if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
} else { } else {
ScannerCycleOutcome::Failed ScannerCycleOutcome::Failed
}; };
@@ -1572,13 +1722,14 @@ async fn run_data_scanner_cycle_with_budget(
state = "superseded", state = "superseded",
"Scanner cycle usage snapshot was superseded by concurrent namespace activity" "Scanner cycle usage snapshot was superseded by concurrent namespace activity"
); );
if finalize_partial_scan_cycle( if finalize_partial_scan_cycle_for_epoch(
ctx, ctx,
storeapi.clone(), storeapi.clone(),
cycle_info, cycle_info,
cycle_revision, cycle_revision,
leader_epoch, leader_epoch,
&mut cycle_metrics_guard, &mut cycle_metrics_guard,
publication_epoch,
) )
.await .await
{ {
@@ -1586,11 +1737,29 @@ async fn run_data_scanner_cycle_with_budget(
emit_scan_cycle_superseded(cycle_start.elapsed()); emit_scan_cycle_superseded(cycle_start.elapsed());
return ScannerCycleOutcome::Superseded; return ScannerCycleOutcome::Superseded;
} }
if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
emit_scan_cycle_complete(false, cycle_start.elapsed()); emit_scan_cycle_complete(false, cycle_start.elapsed());
return ScannerCycleOutcome::Failed; return ScannerCycleOutcome::Failed;
} }
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {} ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {}
} }
if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
let previous_cycle_info = cycle_info.clone();
if let Err(err) = advance_scanner_cycle(cycle_info) { if let Err(err) = advance_scanner_cycle(cycle_info) {
error!( error!(
target: "rustfs::scanner", target: "rustfs::scanner",
@@ -1610,7 +1779,19 @@ async fn run_data_scanner_cycle_with_budget(
global_metrics().clear_current_scan_mode(); global_metrics().clear_current_scan_mode();
retain_recent_cycle_completions(&mut cycle_info.cycle_completed); retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await { if !persist_scanner_cycle_state_for_epoch(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, publication_epoch)
.await
{
if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
emit_scan_cycle_deferred(cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
cycle_metrics_guard.finish(cycle_info.clone()).await; cycle_metrics_guard.finish(cycle_info.clone()).await;
emit_scan_cycle_complete(false, cycle_start.elapsed()); emit_scan_cycle_complete(false, cycle_start.elapsed());
return ScannerCycleOutcome::Failed; return ScannerCycleOutcome::Failed;
@@ -1620,7 +1801,7 @@ async fn run_data_scanner_cycle_with_budget(
done_cycle(); done_cycle();
emit_scan_cycle_complete(true, cycle_start.elapsed()); emit_scan_cycle_complete(true, cycle_start.elapsed());
if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) { if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) {
save_background_heal_info(storeapi.clone(), new_heal_info).await; save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
} }
info!( info!(
+293 -53
View File
@@ -162,6 +162,8 @@ pub(crate) enum ScannerCycleStateStartup {
enum CycleRecoveryMarkerReadError { enum CycleRecoveryMarkerReadError {
#[error("cycle recovery marker backend read failed: {0}")] #[error("cycle recovery marker backend read failed: {0}")]
Backend(#[source] EcstoreError), Backend(#[source] EcstoreError),
#[error("cycle recovery marker publication is blocked by data movement")]
PublicationBlocked,
#[error("invalid cycle recovery marker: {0}")] #[error("invalid cycle recovery marker: {0}")]
Invalid(&'static str), Invalid(&'static str),
#[error("cycle recovery marker revision changed while publishing")] #[error("cycle recovery marker revision changed while publishing")]
@@ -326,12 +328,13 @@ fn cycle_state_generation_and_epoch(buf: &[u8]) -> (u64, u64) {
} }
async fn persist_cycle_recovery_marker( async fn persist_cycle_recovery_marker(
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
primary_revision: &DataUsageCacheRevision, primary_revision: &DataUsageCacheRevision,
generation: u64, generation: u64,
leader_epoch: u64, leader_epoch: u64,
classification: &'static str, classification: &'static str,
reason: &'static str, reason: &'static str,
expected_epoch: u64,
) -> Result<ScannerCycleRecoveryMarker, CycleRecoveryMarkerReadError> { ) -> Result<ScannerCycleRecoveryMarker, CycleRecoveryMarkerReadError> {
let now = unix_now_secs(); let now = unix_now_secs();
let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
@@ -370,6 +373,9 @@ async fn persist_cycle_recovery_marker(
state: "blocked".to_string(), state: "blocked".to_string(),
}; };
let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?; let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?;
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(CycleRecoveryMarkerReadError::PublicationBlocked);
};
let save_result = save_config_with_preconditions( let save_result = save_config_with_preconditions(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -478,7 +484,7 @@ async fn read_cycle_recovery_marker_revision(
} }
async fn quarantine_invalid_cycle_state( async fn quarantine_invalid_cycle_state(
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
revision: &DataUsageCacheRevision, revision: &DataUsageCacheRevision,
buf: &[u8], buf: &[u8],
) -> ScannerCycleStateStartup { ) -> ScannerCycleStateStartup {
@@ -488,7 +494,7 @@ async fn quarantine_invalid_cycle_state(
} }
async fn quarantine_invalid_cycle_state_with_reason( async fn quarantine_invalid_cycle_state_with_reason(
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
revision: &DataUsageCacheRevision, revision: &DataUsageCacheRevision,
generation: u64, generation: u64,
leader_epoch: u64, leader_epoch: u64,
@@ -515,7 +521,19 @@ async fn quarantine_invalid_cycle_state_with_reason(
reason: Some(reason.to_string()), reason: Some(reason.to_string()),
}; };
set_scanner_cycle_recovery_status(base_status); set_scanner_cycle_recovery_status(base_status);
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason).await { let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
};
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason, expected_epoch)
.await
{
Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")), Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")),
Err(CycleRecoveryMarkerReadError::Backend(_)) => { Err(CycleRecoveryMarkerReadError::Backend(_)) => {
// Keep the poison object untouched and retry marker creation with the // Keep the poison object untouched and retry marker creation with the
@@ -524,6 +542,16 @@ async fn quarantine_invalid_cycle_state_with_reason(
"failed to persist scanner cycle recovery marker".to_string(), "failed to persist scanner cycle recovery marker".to_string(),
)); ));
} }
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::Conflict) => { Err(CycleRecoveryMarkerReadError::Conflict) => {
set_scanner_cycle_recovery_status(recovery_status( set_scanner_cycle_recovery_status(recovery_status(
"transient", "transient",
@@ -546,16 +574,18 @@ async fn mark_cycle_recovery_cleanup_pending(
storeapi: Arc<ECStore>, storeapi: Arc<ECStore>,
mut marker: ScannerCycleRecoveryMarker, mut marker: ScannerCycleRecoveryMarker,
marker_revision: &DataUsageCacheRevision, marker_revision: &DataUsageCacheRevision,
expected_epoch: u64,
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> { ) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
marker.state = "cleanup-pending".to_string(); marker.state = "cleanup-pending".to_string();
marker.last_attempt_at_unix_secs = unix_now_secs(); marker.last_attempt_at_unix_secs = unix_now_secs();
let bytes = serde_json::to_vec(&marker) let bytes = serde_json::to_vec(&marker)
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
let info = save_config_with_preconditions( let info = save_config_with_publication_admission_for_epoch(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
bytes, bytes,
marker_revision.preconditions(), marker_revision.preconditions(),
expected_epoch,
) )
.await .await
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
@@ -567,7 +597,9 @@ async fn mark_cycle_recovery_cleanup_pending(
Ok((marker, revision)) Ok((marker, revision))
} }
pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl ScannerObjectIO>) -> ScannerCycleStateStartup { pub(crate) async fn load_scanner_cycle_state_for_startup(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> ScannerCycleStateStartup {
let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
Ok((None, _)) => None, Ok((None, _)) => None,
Ok((Some(data), marker_revision)) => match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&data) { Ok((Some(data), marker_revision)) => match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&data) {
@@ -594,6 +626,16 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl Scan
"failed to read scanner cycle recovery marker: {err}" "failed to read scanner cycle recovery marker: {err}"
))); )));
} }
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::Invalid(reason)) => { Err(CycleRecoveryMarkerReadError::Invalid(reason)) => {
set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false)); set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false));
return ScannerCycleStateStartup::Blocked; return ScannerCycleStateStartup::Blocked;
@@ -750,6 +792,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string())); return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
} }
let Some(reset_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other("scanner recovery reset is blocked by data movement".to_string()));
};
let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false), Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false),
Err(CycleRecoveryMarkerReadError::Invalid(_)) => { Err(CycleRecoveryMarkerReadError::Invalid(_)) => {
@@ -835,7 +881,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
}; };
if let Some((primary_cycle, primary_epoch)) = primary_state { if let Some((primary_cycle, primary_epoch)) = primary_state {
let (cleanup_marker, cleanup_marker_revision) = let (cleanup_marker, cleanup_marker_revision) =
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision).await?; mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending")); set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
let usage_floor = persisted_usage_floor(storeapi.clone()).await?; let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
let fence_epoch = primary_epoch let fence_epoch = primary_epoch
@@ -855,14 +901,21 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"preserved scanner cycle state exceeds the bounded object size".to_string(), "preserved scanner cycle state exceeds the bounded object size".to_string(),
)); ));
} }
let preserved_info = save_config_with_preconditions( let preserved_info = save_config_with_publication_admission_for_epoch(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(), DATA_USAGE_BLOOM_NAME_PATH.as_str(),
preserved_data, preserved_data,
primary_revision.preconditions(), primary_revision.preconditions(),
reset_epoch,
) )
.await .await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")))?; .map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}"))
}
})?;
let preserved_revision = preserved_info let preserved_revision = preserved_info
.etag .etag
.filter(|etag| !etag.is_empty()) .filter(|etag| !etag.is_empty())
@@ -873,7 +926,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after fencing newer cycle state".to_string(), "scanner leader lock was lost after fencing newer cycle state".to_string(),
)); ));
} }
fence_scanner_usage_epoch(&ctx, storeapi.clone(), fence_epoch) fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch))
.await .await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
if guard.is_lock_lost() { if guard.is_lock_lost() {
@@ -889,20 +942,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner cycle state changed before recovery marker cleanup".to_string(), "scanner cycle state changed before recovery marker cleanup".to_string(),
)); ));
} }
storeapi delete_config_with_publication_admission_for_epoch(
.delete_config_object( storeapi.clone(),
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions { ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode // This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path. // bypasses HTTP preconditions in the ECStore path.
delete_prefix: false, delete_prefix: false,
http_preconditions: Some(cleanup_marker_revision.preconditions()), http_preconditions: Some(cleanup_marker_revision.preconditions()),
..Default::default() ..Default::default()
}, },
) reset_epoch,
.await )
.map_err(|err| ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")))?; .await
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}"))
}
})?;
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
super::notify_scanner_cycle_recovery_wake(); super::notify_scanner_cycle_recovery_wake();
return Ok(()); return Ok(());
@@ -946,16 +1006,23 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
let (marker, marker_revision) = if marker.state == "cleanup-pending" { let (marker, marker_revision) = if marker.state == "cleanup-pending" {
(marker, marker_revision) (marker, marker_revision)
} else { } else {
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision).await? mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
}; };
let rebuilt_info = save_config_with_preconditions( let rebuilt_info = save_config_with_publication_admission_for_epoch(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(), DATA_USAGE_BLOOM_NAME_PATH.as_str(),
data, data,
primary_revision.preconditions(), primary_revision.preconditions(),
reset_epoch,
) )
.await .await
.map_err(|err| ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")))?; .map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}"))
}
})?;
let rebuilt_revision = rebuilt_info let rebuilt_revision = rebuilt_info
.etag .etag
.filter(|etag| !etag.is_empty()) .filter(|etag| !etag.is_empty())
@@ -965,7 +1032,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after rebuilding cycle state".to_string(), "scanner leader lock was lost after rebuilding cycle state".to_string(),
)); ));
} }
if let Err(err) = fence_scanner_usage_epoch(&ctx, storeapi.clone(), leader_epoch).await { if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch)).await
{
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(), path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
@@ -1034,20 +1102,40 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
)); ));
} }
if let Err(err) = storeapi if let Err(err) = delete_config_with_publication_admission_for_epoch(
.delete_config_object( storeapi.clone(),
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions { ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode // This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path. // bypasses HTTP preconditions in the ECStore path.
delete_prefix: false, delete_prefix: false,
http_preconditions: Some(marker_revision.preconditions()), http_preconditions: Some(marker_revision.preconditions()),
..Default::default() ..Default::default()
}, },
) reset_epoch,
.await )
.await
{ {
if scanner_publication_epoch_changed(&err) {
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
state: "cleanup-pending".to_string(),
classification: Some(marker.classification.clone()),
primary_revision: Some(rebuilt_revision.clone()),
generation: Some(next),
leader_epoch: Some(leader_epoch),
retry_count: marker.retry_count,
max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES,
retryable: true,
reason: Some("movement epoch changed before recovery marker cleanup".to_string()),
..Default::default()
});
return Err(ScannerError::Other(
"scanner recovery reset deferred by a movement epoch change".to_string(),
));
}
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(), path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
@@ -1209,8 +1297,14 @@ pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(),
Ok(()) Ok(())
} }
pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -> Result<PersistedUsageFloor, ScannerError> { pub(super) async fn persisted_usage_floor(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<PersistedUsageFloor, ScannerError> {
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string()));
};
let mut floor = PersistedUsageFloor::default(); let mut floor = PersistedUsageFloor::default();
let mut found_any = false;
let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> { let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> {
floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default());
if let Some(completed_cycle) = usage.scanner_cycle { if let Some(completed_cycle) = usage.scanner_cycle {
@@ -1229,6 +1323,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| { let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}")) ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}"))
})?; })?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {primary_path} has no persisted baseline identity"
)));
}
let epoch = usage.scanner_epoch.unwrap_or_default(); let epoch = usage.scanner_epoch.unwrap_or_default();
update_floor(&mut floor, &usage, primary_path)?; update_floor(&mut floor, &usage, primary_path)?;
Some(epoch) Some(epoch)
@@ -1247,6 +1346,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| { let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}")) ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}"))
})?; })?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {backup_path} has no persisted baseline identity"
)));
}
let backup_epoch = usage.scanner_epoch.unwrap_or_default(); let backup_epoch = usage.scanner_epoch.unwrap_or_default();
// A backup write from an older leader may complete after the // A backup write from an older leader may complete after the
// primary epoch has been fenced. It must not advance the startup // primary epoch has been fenced. It must not advance the startup
@@ -1263,9 +1367,21 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
} }
} }
if any_found { if any_found {
found_any = true;
break; break;
} }
} }
if !found_any {
return Err(ScannerError::Other(
"persisted scanner usage floor has no authoritative baseline".to_string(),
));
}
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi, read_epoch).await else {
return Err(ScannerError::Other(
"scanner usage floor changed while its epoch proof was being confirmed".to_string(),
));
};
Ok(floor) Ok(floor)
} }
@@ -1274,12 +1390,30 @@ pub(super) fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_
*leader_epoch = (*leader_epoch).max(floor.leader_epoch); *leader_epoch = (*leader_epoch).max(floor.leader_epoch);
} }
#[derive(Clone, Copy)]
pub(super) struct ScannerCycleFloorOptions {
pub(super) required_cycle: u64,
pub(super) expected_publication_epoch: Option<u64>,
}
#[cfg(test)]
pub(super) async fn persist_scanner_cycle_state( pub(super) async fn persist_scanner_cycle_state(
ctx: &CancellationToken, ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle, cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision, revision: &mut DataUsageCacheRevision,
leader_epoch: u64, leader_epoch: u64,
) -> bool {
persist_scanner_cycle_state_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, None).await
}
pub(super) async fn persist_scanner_cycle_state_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> bool { ) -> bool {
let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) { let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) {
Ok(buf) => buf, Ok(buf) => buf,
@@ -1315,9 +1449,29 @@ pub(super) async fn persist_scanner_cycle_state(
#[cfg(test)] #[cfg(test)]
notify_scanner_cycle_state_persist_test_hook(leader_epoch); notify_scanner_cycle_state_persist_test_hook(leader_epoch);
match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
.await return false;
{ };
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
return false;
}
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "publication_admission_unavailable",
"Scanner state persistence skipped without movement admission"
);
return false;
};
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
.await
};
match save_result {
Ok(object_info) => { Ok(object_info) => {
let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else { let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else {
error!( error!(
@@ -1345,6 +1499,13 @@ pub(super) async fn persist_scanner_cycle_state(
); );
return false; return false;
} }
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
debug!( debug!(
target: "rustfs::scanner", target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE, event = EVENT_SCANNER_PERSIST_STATE,
@@ -1436,6 +1597,13 @@ pub(super) async fn persist_scanner_cycle_state(
} }
if persisted_cycle.next >= cycle_info.next { if persisted_cycle.next >= cycle_info.next {
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
*cycle_info = persisted_cycle; *cycle_info = persisted_cycle;
debug!( debug!(
target: "rustfs::scanner", target: "rustfs::scanner",
@@ -1496,19 +1664,33 @@ pub(super) async fn persist_scanner_cycle_state(
false false
} }
#[cfg(test)]
pub(super) async fn finalize_partial_scan_cycle( pub(super) async fn finalize_partial_scan_cycle(
ctx: &CancellationToken, ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle, cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision, revision: &mut DataUsageCacheRevision,
leader_epoch: u64, leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard, cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool {
finalize_partial_scan_cycle_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, cycle_metrics_guard, None).await
}
pub(super) async fn finalize_partial_scan_cycle_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
expected_publication_epoch: Option<u64>,
) -> bool { ) -> bool {
// A budget-limited cycle is deliberate pacing, not a failure. The cycle counter // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter
// must still advance (and persist) because per-bucket next_cycle is stamped from // must still advance (and persist) because per-bucket next_cycle is stamped from
// it and compacted folders are only rescanned when their hash matches // it and compacted folders are only rescanned when their hash matches
// next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle // next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle
// expiry and usage refresh on every folder outside the stuck window. // expiry and usage refresh on every folder outside the stuck window.
let previous_cycle_info = cycle_info.clone();
if let Err(err) = advance_scanner_cycle(cycle_info) { if let Err(err) = advance_scanner_cycle(cycle_info) {
error!( error!(
target: "rustfs::scanner", target: "rustfs::scanner",
@@ -1524,28 +1706,69 @@ pub(super) async fn finalize_partial_scan_cycle(
} }
cycle_info.current = 0; cycle_info.current = 0;
global_metrics().clear_current_scan_mode(); global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; let persisted = persist_scanner_cycle_state_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
revision,
leader_epoch,
expected_publication_epoch,
)
.await;
if !persisted
&& let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
}
cycle_metrics_guard.finish(cycle_info.clone()).await; cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted persisted
} }
#[cfg(test)]
pub(super) async fn persist_required_scanner_cycle_floor( pub(super) async fn persist_required_scanner_cycle_floor(
ctx: &CancellationToken, ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle, cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision, revision: &mut DataUsageCacheRevision,
leader_epoch: u64, leader_epoch: u64,
required_cycle: u64, required_cycle: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard, cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool { ) -> bool {
if required_cycle <= cycle_info.current || required_cycle == u64::MAX { persist_required_scanner_cycle_floor_for_epoch(
ctx,
storeapi,
cycle_info,
revision,
leader_epoch,
cycle_metrics_guard,
ScannerCycleFloorOptions {
required_cycle,
expected_publication_epoch: None,
},
)
.await
}
pub(super) async fn persist_required_scanner_cycle_floor_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
options: ScannerCycleFloorOptions,
) -> bool {
if options.required_cycle <= cycle_info.current || options.required_cycle == u64::MAX {
error!( error!(
target: "rustfs::scanner", target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE, event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER, component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME, subsystem = LOG_SUBSYSTEM_RUNTIME,
current_cycle = cycle_info.current, current_cycle = cycle_info.current,
required_cycle, required_cycle = options.required_cycle,
state = "invalid_cache_cycle_floor", state = "invalid_cache_cycle_floor",
"Scanner cache cycle floor is invalid" "Scanner cache cycle floor is invalid"
); );
@@ -1553,10 +1776,27 @@ pub(super) async fn persist_required_scanner_cycle_floor(
return false; return false;
} }
cycle_info.next = cycle_info.next.max(required_cycle); let previous_cycle_info = cycle_info.clone();
cycle_info.next = cycle_info.next.max(options.required_cycle);
cycle_info.current = 0; cycle_info.current = 0;
global_metrics().clear_current_scan_mode(); global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; let persisted = persist_scanner_cycle_state_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
revision,
leader_epoch,
options.expected_publication_epoch,
)
.await;
if !persisted
&& let Some(expected_epoch) = options.expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
}
cycle_metrics_guard.finish(cycle_info.clone()).await; cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted persisted
} }
+107 -18
View File
@@ -26,31 +26,90 @@ pub struct BackgroundHealInfo {
pub current_scan_mode: HealScanMode, pub current_scan_mode: HealScanMode,
} }
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum BackgroundHealInfoReadStatus {
ErasureSd,
Loaded,
Missing,
Blocked,
Transient,
Failed,
}
pub(super) struct BackgroundHealInfoRead {
pub(super) info: BackgroundHealInfo,
pub(super) expected_epoch: Option<u64>,
pub(super) status: BackgroundHealInfoReadStatus,
}
pub(super) fn classify_background_heal_read_error(error: &EcstoreError) -> BackgroundHealInfoReadStatus {
if matches!(error, EcstoreError::ConfigNotFound) {
BackgroundHealInfoReadStatus::Missing
} else {
BackgroundHealInfoReadStatus::Transient
}
}
pub(super) fn decode_background_heal_info(data: &[u8]) -> Result<BackgroundHealInfo, serde_json::Error> {
serde_json::from_slice(data)
}
/// Read background healing information from storage /// Read background healing information from storage
pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHealInfo { pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHealInfo {
read_background_heal_info_with_epoch(storeapi).await.info
}
/// Read background healing information together with the movement epoch that
/// fenced the read. The epoch must be reused by the matching cycle update so a
/// missing-object default cannot be committed across a movement transition.
pub(super) async fn read_background_heal_info_with_epoch(storeapi: Arc<ECStore>) -> BackgroundHealInfoRead {
// Skip for ErasureSD setup // Skip for ErasureSD setup
if scanner_is_erasure_sd().await { if scanner_is_erasure_sd().await {
return BackgroundHealInfo::default(); return BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch: None,
status: BackgroundHealInfoReadStatus::ErasureSd,
};
}
let expected_epoch = scanner_publication_epoch(storeapi.clone()).await;
if expected_epoch.is_none() {
return BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status: BackgroundHealInfoReadStatus::Blocked,
};
} }
// Get last healing information // Get last healing information
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await { match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
Ok(buf) => serde_json::from_slice::<BackgroundHealInfo>(&buf).unwrap_or_else(|e| { Ok(buf) => match decode_background_heal_info(&buf) {
error!( Ok(info) => BackgroundHealInfoRead {
target: "rustfs::scanner", info,
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, expected_epoch,
component = LOG_COMPONENT_SCANNER, status: BackgroundHealInfoReadStatus::Loaded,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, },
path = %&*BACKGROUND_HEAL_INFO_PATH, Err(e) => {
state = "decode_failed", error!(
error = %e, target: "rustfs::scanner",
"Scanner background heal decode failed" event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
); component = LOG_COMPONENT_SCANNER,
BackgroundHealInfo::default() subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
}), path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal decode failed"
);
BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status: BackgroundHealInfoReadStatus::Failed,
}
}
},
Err(e) => { Err(e) => {
// Only log if it's not a ConfigNotFound error let status = classify_background_heal_read_error(&e);
if e != EcstoreError::ConfigNotFound { if status == BackgroundHealInfoReadStatus::Transient {
warn!( warn!(
target: "rustfs::scanner", target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
@@ -62,7 +121,11 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
"Scanner background heal read failed" "Scanner background heal read failed"
); );
} }
BackgroundHealInfo::default() BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status,
}
} }
} }
} }
@@ -70,6 +133,14 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
/// Save background healing information to storage /// Save background healing information to storage
#[instrument(skip(storeapi))] #[instrument(skip(storeapi))]
pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundHealInfo) { pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundHealInfo) {
save_background_heal_info_for_epoch(storeapi, info, None).await;
}
pub(super) async fn save_background_heal_info_for_epoch(
storeapi: Arc<ECStore>,
info: BackgroundHealInfo,
expected_epoch: Option<u64>,
) {
// Skip for ErasureSD setup // Skip for ErasureSD setup
if scanner_is_erasure_sd().await { if scanner_is_erasure_sd().await {
return; return;
@@ -93,7 +164,25 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
} }
}; };
// Save configuration // Save configuration only after storage-owned movement admission. The
// read path may return an in-memory default for a missing object, but a
// movement transition must not let that default become durable state.
let publication_admission = match expected_epoch {
Some(expected_epoch) => scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await,
None => storeapi.scanner_data_usage_publication_admission().await,
};
let Some(_publication_admission) = publication_admission else {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "publication_admission_unavailable",
"Scanner background heal save skipped without movement admission"
);
return;
};
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await { if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
warn!( warn!(
target: "rustfs::scanner", target: "rustfs::scanner",
+119 -20
View File
@@ -61,16 +61,22 @@ pub(super) async fn reconcile_scanner_leadership_claim(
} }
pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result<DataUsageInfo, ScannerError> { pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result<DataUsageInfo, ScannerError> {
serde_json::from_slice(data) let usage: DataUsageInfo = serde_json::from_slice(data)
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}"))) .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage epoch fence from {path} has no persisted baseline identity"
)));
}
Ok(usage)
} }
pub(super) async fn usage_snapshot_for_epoch_fence( pub(super) async fn usage_snapshot_for_epoch_fence(
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO>,
primary: Option<&[u8]>, primary: Option<&[u8]>,
) -> Result<DataUsageInfo, ScannerError> { ) -> Result<Option<DataUsageInfo>, ScannerError> {
if let Some(primary) = primary { if let Some(primary) = primary {
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()); return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()).map(Some);
} }
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
@@ -78,7 +84,7 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await .await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?;
if let Some(backup) = backup.as_deref() { if let Some(backup) = backup.as_deref() {
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path); return decode_usage_snapshot_for_epoch_fence(backup, &backup_path).map(Some);
} }
for path in [ for path in [
@@ -89,26 +95,53 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await .await
.map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?;
if let Some(legacy) = legacy.as_deref() { if let Some(legacy) = legacy.as_deref() {
return decode_usage_snapshot_for_epoch_fence(legacy, &path); return decode_usage_snapshot_for_epoch_fence(legacy, &path).map(Some);
} }
} }
Ok(DataUsageInfo::default()) // A missing usage snapshot is an uninitialized state, not an empty
// snapshot. Leadership fencing may proceed without creating a plausible
// default; the first authoritative scanner publication will create it.
Ok(None)
} }
pub(super) async fn fence_scanner_usage_epoch( pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
ctx: &CancellationToken, ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64, claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> Result<(), ScannerError> { ) -> Result<(), ScannerError> {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() { if ctx.is_cancelled() {
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
} }
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other(
"scanner usage epoch fence publication is blocked by data movement".to_string(),
));
};
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while recovery reset was in progress".to_string(),
));
}
let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await .await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?;
let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?; let Some(mut usage) = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await? else {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while confirming a missing usage baseline".to_string(),
));
};
return Err(ScannerError::Other("authoritative scanner usage baseline is missing".to_string()));
};
match usage.scanner_epoch { match usage.scanner_epoch {
Some(epoch) if epoch > claimed_epoch => { Some(epoch) if epoch > claimed_epoch => {
return Err(ScannerError::Other(format!( return Err(ScannerError::Other(format!(
@@ -122,9 +155,18 @@ pub(super) async fn fence_scanner_usage_epoch(
let data = serde_json::to_vec(&usage) let data = serde_json::to_vec(&usage)
.map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?;
let save_result = let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
));
};
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
.await; .await
};
if save_result if save_result
.as_ref() .as_ref()
.ok() .ok()
@@ -165,10 +207,13 @@ pub(super) async fn fence_scanner_usage_epoch(
pub(super) async fn complete_scanner_leadership_claim( pub(super) async fn complete_scanner_leadership_claim(
ctx: &CancellationToken, ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64, claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> bool { ) -> bool {
if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await { if let Err(err) =
fence_scanner_usage_epoch_with_expected_epoch(ctx, storeapi, claimed_epoch, expected_publication_epoch).await
{
error!( error!(
target: "rustfs::scanner", target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE, event = EVENT_SCANNER_PERSIST_STATE,
@@ -187,7 +232,7 @@ pub(super) async fn complete_scanner_leadership_claim(
pub(super) async fn claim_scanner_leadership( pub(super) async fn claim_scanner_leadership(
ctx: &CancellationToken, ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle, cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision, revision: &mut DataUsageCacheRevision,
persisted_epoch: &mut u64, persisted_epoch: &mut u64,
@@ -226,15 +271,69 @@ pub(super) async fn claim_scanner_leadership(
}; };
let previous_revision = revision.clone(); let previous_revision = revision.clone();
let save_result = let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return false;
};
let (usage_primary, _) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
Ok(result) => result,
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_read_failed",
error = %err,
"Scanner leadership claim deferred because the usage baseline could not be read"
);
return false;
}
};
match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref()).await {
Ok(Some(_)) => {}
Ok(None) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_missing",
"Scanner leadership claim deferred until a usage baseline is published"
);
return false;
}
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_invalid",
error = %err,
"Scanner leadership claim deferred because the usage baseline is invalid"
);
return false;
}
}
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return false;
};
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions()) save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions())
.await; .await
};
match save_result { match save_result {
Ok(object_info) => { Ok(object_info) => {
if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) { if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) {
*revision = DataUsageCacheRevision::Etag(etag); *revision = DataUsageCacheRevision::Etag(etag);
*persisted_epoch = claimed_epoch; *persisted_epoch = claimed_epoch;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
} }
match reconcile_scanner_leadership_claim( match reconcile_scanner_leadership_claim(
@@ -249,7 +348,7 @@ pub(super) async fn claim_scanner_leadership(
.await .await
{ {
Ok(ScannerLeadershipClaimReconcile::Durable) => { Ok(ScannerLeadershipClaimReconcile::Durable) => {
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
} }
Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue, Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue,
Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => {
@@ -293,7 +392,7 @@ pub(super) async fn claim_scanner_leadership(
.await .await
{ {
Ok(ScannerLeadershipClaimReconcile::Durable) => { Ok(ScannerLeadershipClaimReconcile::Durable) => {
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
} }
Ok(ScannerLeadershipClaimReconcile::Changed) Ok(ScannerLeadershipClaimReconcile::Changed)
if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() =>
+185 -28
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info};
use super::*; use super::*;
use crate::EcstoreResult; use crate::EcstoreResult;
use crate::{ use crate::{
@@ -22,6 +23,7 @@ use crate::{
}; };
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::io::Cursor; use std::io::Cursor;
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::Poll; use std::task::Poll;
use temp_env::{with_var, with_var_unset}; use temp_env::{with_var, with_var_unset};
use tokio::io::AsyncReadExt; use tokio::io::AsyncReadExt;
@@ -343,6 +345,7 @@ struct MemoryConfigStore {
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>, cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>, replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
put_counts: Mutex<HashMap<String, usize>>, put_counts: Mutex<HashMap<String, usize>>,
publication_admission_blocked: AtomicBool,
} }
fn memory_config_key(bucket: &str, object: &str) -> String { fn memory_config_key(bucket: &str, object: &str) -> String {
@@ -1350,7 +1353,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() {
let old_usage = DataUsageInfo { let old_usage = DataUsageInfo {
scanner_epoch: Some(7), scanner_epoch: Some(7),
scanner_cycle: Some(41), scanner_cycle: Some(41),
..Default::default() ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
}; };
let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode"); let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone()) save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone())
@@ -1424,7 +1427,7 @@ async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() {
let usage = DataUsageInfo { let usage = DataUsageInfo {
scanner_epoch: Some(7), scanner_epoch: Some(7),
scanner_cycle: Some(41), scanner_cycle: Some(41),
..Default::default() ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
}; };
save_config( save_config(
store.clone(), store.clone(),
@@ -1695,7 +1698,7 @@ async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() {
DATA_USAGE_OBJ_NAME_PATH.as_str(), DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&DataUsageInfo { serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(u64::MAX - 1), scanner_epoch: Some(u64::MAX - 1),
..Default::default() ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
}) })
.expect("usage floor should encode"), .expect("usage floor should encode"),
) )
@@ -1847,14 +1850,12 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] { for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(epoch);
usage.scanner_cycle = Some(cycle);
store.objects.lock().await.insert( store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path), memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&DataUsageInfo { serde_json::to_vec(&usage).expect("usage snapshot should encode"),
scanner_epoch: Some(epoch),
scanner_cycle: Some(cycle),
..Default::default()
})
.expect("usage snapshot should encode"),
); );
} }
@@ -1879,14 +1880,12 @@ async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] { for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(epoch);
usage.scanner_cycle = Some(cycle);
store.objects.lock().await.insert( store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path), memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&DataUsageInfo { serde_json::to_vec(&usage).expect("usage snapshot should encode"),
scanner_epoch: Some(epoch),
scanner_cycle: Some(cycle),
..Default::default()
})
.expect("usage snapshot should encode"),
); );
} }
@@ -1916,6 +1915,23 @@ fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
})); }));
} }
#[test]
fn scanner_baseline_identity_requires_complete_or_strict_legacy_shape() {
assert!(!data_usage_info_has_persisted_baseline_identity(&DataUsageInfo {
scanner_epoch: Some(3),
scanner_cycle: Some(7),
..Default::default()
}));
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
legacy.usage_snapshot_complete = false;
legacy.scanner_cycle = Some(7);
assert!(data_usage_info_has_persisted_baseline_identity(&legacy));
legacy.scanner_epoch = Some(3);
assert!(!data_usage_info_has_persisted_baseline_identity(&legacy));
}
#[test] #[test]
fn scanner_startup_prompts_only_for_a_newer_valid_observation() { fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
let authoritative = DataUsageInfo { let authoritative = DataUsageInfo {
@@ -1948,12 +1964,9 @@ fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
#[tokio::test] #[tokio::test]
async fn scanner_startup_prefers_v2_over_legacy_usage() { async fn scanner_startup_prefers_v2_over_legacy_usage() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
let legacy = DataUsageInfo { let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
scanner_epoch: Some(19), legacy.scanner_epoch = Some(19);
scanner_cycle: Some(41), legacy.scanner_cycle = Some(41);
last_update: Some(std::time::SystemTime::now()),
..Default::default()
};
let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"); let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode");
store.objects.lock().await.insert( store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
@@ -1976,13 +1989,9 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
} }
); );
let authoritative = DataUsageInfo { let mut authoritative = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
scanner_epoch: Some(23), authoritative.scanner_epoch = Some(23);
scanner_cycle: Some(51), authoritative.scanner_cycle = Some(51);
last_update: Some(std::time::SystemTime::now()),
usage_snapshot_complete: true,
..Default::default()
};
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode"); let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
store.objects.lock().await.insert( store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
@@ -2024,6 +2033,8 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
#[tokio::test] #[tokio::test]
async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() { async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
assert!(persisted_usage_floor(store.clone()).await.is_err());
store.objects.lock().await.insert( store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
b"not-json".to_vec(), b"not-json".to_vec(),
@@ -2087,6 +2098,39 @@ async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
} }
} }
#[tokio::test]
async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_primary() {
let store = Arc::new(MemoryConfigStore::default());
let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
store.objects.lock().await.insert(
primary_key.clone(),
serde_json::to_vec(&primary).expect("primary usage snapshot should encode"),
);
store.revisions.lock().await.insert(primary_key.clone(), 1);
store.publication_admission_blocked.store(true, Ordering::Release);
let movement_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone())
.await
.expect_err("movement admission loss should fail backup synchronization");
assert!(scanner_publication_epoch_changed(&movement_error));
store.publication_admission_blocked.store(false, Ordering::Release);
store.objects.lock().await.remove(&primary_key);
store.revisions.lock().await.remove(&primary_key);
assert!(matches!(
sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()).await,
Err(EcstoreError::ConfigNotFound)
));
store.objects.lock().await.insert(primary_key.clone(), b"not-json".to_vec());
store.revisions.lock().await.insert(primary_key, 1);
let corrupt_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store)
.await
.expect_err("corrupt primary should fail backup synchronization");
assert!(!scanner_publication_epoch_changed(&corrupt_error));
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for MemoryConfigStore { impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> { async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
@@ -2110,6 +2154,10 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
revisions.remove(&key); revisions.remove(&key);
Ok(ObjectInfo::default()) Ok(ObjectInfo::default())
} }
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
(!self.publication_admission_blocked.load(Ordering::Acquire)).then(crate::ScannerDataUsagePublicationAdmission::unfenced)
}
} }
#[test] #[test]
@@ -2270,6 +2318,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
started: Utc::now(), started: Utc::now(),
}; };
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
seed_usage_snapshot_for_leadership_claim(&store).await;
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
let old_epoch_commit = CurrentCycle { let old_epoch_commit = CurrentCycle {
@@ -2313,6 +2362,61 @@ async fn test_leadership_claim_rejects_terminal_epoch() {
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
} }
#[tokio::test]
async fn leadership_claim_defers_without_usage_baseline_before_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
store.objects.lock().await.insert(usage_key.clone(), b"not-json".to_vec());
store.revisions.lock().await.insert(usage_key, 1);
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let data = serde_json::to_vec(&DataUsageInfo::default()).expect("default usage should encode");
store.objects.lock().await.insert(usage_key.clone(), data);
store.revisions.lock().await.insert(usage_key, 1);
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test] #[tokio::test]
async fn test_leadership_claim_confirms_commit_after_returned_error() { async fn test_leadership_claim_confirms_commit_after_returned_error() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
@@ -2329,6 +2433,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() {
started: Utc::now(), started: Utc::now(),
}; };
let mut persisted_epoch = 0; let mut persisted_epoch = 0;
seed_usage_snapshot_for_leadership_claim(&store).await;
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
@@ -2373,6 +2478,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
); );
old_usage.buckets_count = 1; old_usage.buckets_count = 1;
old_usage.calculate_totals(); old_usage.calculate_totals();
old_usage.usage_snapshot_complete = true;
let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode"); let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode");
store.objects.lock().await.insert(usage_key.clone(), old_data.clone()); store.objects.lock().await.insert(usage_key.clone(), old_data.clone());
store.revisions.lock().await.insert(usage_key, 1); store.revisions.lock().await.insert(usage_key, 1);
@@ -2419,6 +2525,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() {
started: Utc::now(), started: Utc::now(),
}; };
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
seed_usage_snapshot_for_leadership_claim(&store).await;
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
store store
@@ -2590,6 +2697,35 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
} }
} }
#[tokio::test]
async fn test_observational_usage_defers_when_authoritative_baseline_is_missing() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(1);
let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
observation.usage_snapshot_converged = Some(false);
sender.send(observation).await.expect("observation should enqueue");
drop(sender);
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
CancellationToken::new(),
store.clone(),
receiver,
None,
None,
|| async { false },
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert!(
!store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()))
);
}
#[tokio::test] #[tokio::test]
async fn test_usage_route_barrier_precedes_durable_reconciliation() { async fn test_usage_route_barrier_precedes_durable_reconciliation() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
@@ -3425,6 +3561,14 @@ fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>,
info info
} }
async fn seed_usage_snapshot_for_leadership_claim(store: &Arc<MemoryConfigStore>) {
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let data = serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0))
.expect("leadership usage baseline should encode");
store.objects.lock().await.insert(key.clone(), data);
store.revisions.lock().await.insert(key, 1);
}
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo { fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
complete_usage_with_bucket_count(last_update, 0) complete_usage_with_bucket_count(last_update, 0)
} }
@@ -5129,6 +5273,19 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() {
assert_eq!(info.bitrot_start_time, Some(now)); assert_eq!(info.bitrot_start_time, Some(now));
} }
#[test]
fn background_heal_read_failures_never_become_initializable_defaults() {
assert_eq!(
classify_background_heal_read_error(&EcstoreError::ConfigNotFound),
BackgroundHealInfoReadStatus::Missing
);
assert_eq!(
classify_background_heal_read_error(&EcstoreError::SlowDown),
BackgroundHealInfoReadStatus::Transient
);
assert!(decode_background_heal_info(b"not-json").is_err());
}
#[test] #[test]
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() { fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || { with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
+221 -50
View File
@@ -112,11 +112,39 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
} }
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>( pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
ctx,
storeapi,
receiver,
leader_epoch,
initial_baseline,
None,
route_probe,
)
.await
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch<
F,
Fut,
>(
ctx: CancellationToken, ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>, mut receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>, leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>, initial_baseline: Option<DataUsagePersistBaseline>,
expected_publication_epoch: Option<u64>,
route_probe: F, route_probe: F,
) -> DataUsagePersistOutcome ) -> DataUsagePersistOutcome
where where
@@ -134,6 +162,14 @@ where
if let Some(leader_epoch) = leader_epoch { if let Some(leader_epoch) = leader_epoch {
data_usage_info.scanner_epoch = Some(leader_epoch); data_usage_info.scanner_epoch = Some(leader_epoch);
} }
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
let observational = data_usage_info.usage_snapshot_converged == Some(false); let observational = data_usage_info.usage_snapshot_converged == Some(false);
let target_path = if observational { let target_path = if observational {
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
@@ -154,7 +190,28 @@ where
break; break;
} }
let mut publication_epoch = expected_publication_epoch;
if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() { if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
expected_epoch
}
None => {
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
};
read_epoch
}
};
publication_epoch = Some(read_epoch);
let authoritative_data = match next_baseline.as_ref() { let authoritative_data = match next_baseline.as_ref() {
Some(baseline) => baseline.data.clone(), Some(baseline) => baseline.data.clone(),
None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
@@ -175,25 +232,48 @@ where
} }
}, },
}; };
let authoritative = match authoritative_data.as_deref() { let Some(authoritative_data) = authoritative_data else {
Some(data) => match serde_json::from_slice::<DataUsageInfo>(data) { warn!(
Ok(info) => info, target: "rustfs::scanner",
Err(err) => { event = EVENT_SCANNER_PERSIST_STATE,
error!( component = LOG_COMPONENT_SCANNER,
target: "rustfs::scanner", subsystem = LOG_SUBSYSTEM_RUNTIME,
event = EVENT_SCANNER_PERSIST_STATE, path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
component = LOG_COMPONENT_SCANNER, state = "observed_baseline_missing",
subsystem = LOG_SUBSYSTEM_RUNTIME, "Scanner deferred observational publication until an authoritative usage baseline exists"
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), );
state = "observed_baseline_decode_failed", outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
error = %err, break 'updates;
"Scanner refused to publish an observation from an invalid authoritative baseline" };
); let authoritative = match serde_json::from_slice::<DataUsageInfo>(&authoritative_data) {
outcome = DataUsagePersistOutcome::Failed; Ok(info) if data_usage_info_has_persisted_baseline_identity(&info) => info,
continue; Ok(_) => {
} error!(
}, target: "rustfs::scanner",
None => DataUsageInfo::default(), event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_identity_missing",
"Scanner refused to publish an observation without authoritative baseline identity"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_decode_failed",
error = %err,
"Scanner refused to publish an observation from an invalid authoritative baseline"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
}; };
data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity()); data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
} }
@@ -240,6 +320,26 @@ where
break 'updates; break 'updates;
} }
let publication_epoch_for_save = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
expected_epoch
}
None => match publication_epoch.take() {
Some(epoch) => epoch,
None => {
let Some(epoch) = scanner_publication_epoch(storeapi.clone()).await else {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
epoch
}
},
};
let baseline = if !observational && cas_retry == 0 { let baseline = if !observational && cas_retry == 0 {
next_baseline.take() next_baseline.take()
} else { } else {
@@ -329,14 +429,22 @@ where
} }
let done_save = Metrics::time(Metric::SaveUsage); let done_save = Metrics::time(Metric::SaveUsage);
let save_result = save_config_shared_with_preconditions( let save_result = {
storeapi.clone(), let Some(_publication_admission) =
target_path, scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
data.clone(), else {
sha256hex.clone(), done_save();
revision.preconditions(), break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
) };
.await; save_config_shared_with_preconditions(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
)
.await
};
done_save(); done_save();
match save_result { match save_result {
@@ -427,7 +535,16 @@ where
if observational { if observational {
invalidate_admin_data_usage_snapshot_cache().await; invalidate_admin_data_usage_snapshot_cache().await;
} else { } else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await; invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await; replace_bucket_usage_memory_from_info(&data_usage_info).await;
} }
@@ -438,7 +555,16 @@ where
if observational { if observational {
invalidate_admin_data_usage_snapshot_cache().await; invalidate_admin_data_usage_snapshot_cache().await;
} else { } else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await; invalidate_data_usage_snapshot_cache().await;
} }
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
@@ -460,7 +586,16 @@ where
if observational { if observational {
invalidate_admin_data_usage_snapshot_cache().await; invalidate_admin_data_usage_snapshot_cache().await;
} else { } else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await; invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await; replace_bucket_usage_memory_from_info(&data_usage_info).await;
} }
@@ -471,7 +606,10 @@ where
if backup_due { if backup_due {
let done_save = Metrics::time(Metric::SaveUsage); let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await { let backup_result =
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
done_save();
if let Err(e) = backup_result {
warn!( warn!(
target: "rustfs::scanner", target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE, event = EVENT_SCANNER_PERSIST_STATE,
@@ -482,22 +620,50 @@ where
error = %e, error = %e,
"Scanner data usage backup save failed" "Scanner data usage backup save failed"
); );
if scanner_publication_epoch_changed(&e) {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
outcome = DataUsagePersistOutcome::Failed;
break 'updates;
} }
done_save();
} }
} }
outcome outcome
} }
pub(super) async fn cleanup_observed_data_usage_snapshot( pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
authoritative: &DataUsageInfo, authoritative: &DataUsageInfo,
) { expected_publication_epoch: Option<u64>,
) -> bool {
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
expected_epoch
}
None => match scanner_publication_epoch(storeapi.clone()).await {
Some(read_epoch) => read_epoch,
None => return false,
},
};
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return false;
}
let (observed_data, revision) = let (observed_data, revision) =
match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
Ok((Some(data), revision)) => (data, revision), Ok((Some(data), revision)) => (data, revision),
Ok((None, _)) => return, Ok((None, _)) => return true,
Err(err) => { Err(err) => {
error!( error!(
target: "rustfs::scanner", target: "rustfs::scanner",
@@ -509,7 +675,7 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err, error = %err,
"Scanner could not inspect observational data usage snapshot before authoritative cleanup" "Scanner could not inspect observational data usage snapshot before authoritative cleanup"
); );
return; return true;
} }
}; };
let observed = match serde_json::from_slice::<DataUsageInfo>(&observed_data) { let observed = match serde_json::from_slice::<DataUsageInfo>(&observed_data) {
@@ -525,25 +691,26 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err, error = %err,
"Scanner refused to remove an invalid observational data usage snapshot after authoritative save" "Scanner refused to remove an invalid observational data usage snapshot after authoritative save"
); );
return; return true;
} }
}; };
if observed_data_usage_is_newer(&observed, authoritative) { if observed_data_usage_is_newer(&observed, authoritative) {
return; return true;
} }
let result = storeapi let result = delete_config_with_publication_admission_for_epoch(
.delete_config_object( storeapi,
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
ScannerObjectOptions { ScannerObjectOptions {
delete_prefix: true, delete_prefix: true,
delete_prefix_object: true, delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()), http_preconditions: Some(revision.preconditions()),
..Default::default() ..Default::default()
}, },
) read_epoch,
.await; )
.await;
match result { match result {
Ok(_) Ok(_)
@@ -564,6 +731,10 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err, error = %err,
"Scanner could not remove stale observational data usage snapshot after authoritative save" "Scanner could not remove stale observational data usage snapshot after authoritative save"
); );
if scanner_publication_epoch_changed(&err) {
return false;
}
} }
} }
true
} }
+19 -3
View File
@@ -56,9 +56,10 @@ use crate::storage_api::scan::NamespaceLocking as _;
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions}; use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
use crate::{ use crate::{
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerConfigObjectDelete as _, ScannerDiskExt as _,
ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError,
get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, storageclass, enqueue_runtime_free_version, get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names,
scanner_publication_admission_for_epoch, scanner_publication_epoch, storageclass,
}; };
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file"; pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
@@ -143,6 +144,10 @@ pub struct ScannerBucketScanPlan {
all_buckets: Arc<Vec<BucketInfo>>, all_buckets: Arc<Vec<BucketInfo>>,
digest: DataUsageScanPlanDigest, digest: DataUsageScanPlanDigest,
leader_epoch: u64, leader_epoch: u64,
/// Epoch captured once for the whole scanner cycle. `None` is retained
/// for unfenced test implementations; production plans always carry the
/// admission token captured before bucket enumeration.
publication_epoch: Option<u64>,
dirty_usage_buckets: Arc<DirtyUsageBuckets>, dirty_usage_buckets: Arc<DirtyUsageBuckets>,
bucket_failures: ScannerBucketFailureState, bucket_failures: ScannerBucketFailureState,
pending_maintenance_work: Arc<AtomicBool>, pending_maintenance_work: Arc<AtomicBool>,
@@ -578,6 +583,7 @@ fn scanner_activity_preflight(
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct ScannerCycleResult { pub(crate) struct ScannerCycleResult {
pub(crate) status: ScannerCycleStatus, pub(crate) status: ScannerCycleStatus,
publication_epoch: Option<u64>,
dirty_usage_clear: Option<DirtyUsageBuckets>, dirty_usage_clear: Option<DirtyUsageBuckets>,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>, remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
failed_dirty_usage: bool, failed_dirty_usage: bool,
@@ -589,6 +595,7 @@ impl ScannerCycleResult {
pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option<DirtyUsageBuckets>) -> Self { pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option<DirtyUsageBuckets>) -> Self {
Self { Self {
status, status,
publication_epoch: None,
dirty_usage_clear, dirty_usage_clear,
remote_dirty_usage_acknowledgements: Vec::new(), remote_dirty_usage_acknowledgements: Vec::new(),
failed_dirty_usage: false, failed_dirty_usage: false,
@@ -597,6 +604,15 @@ impl ScannerCycleResult {
} }
} }
pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option<u64>) -> Self {
self.publication_epoch = publication_epoch;
self
}
pub(crate) fn publication_epoch(&self) -> Option<u64> {
self.publication_epoch
}
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self { fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
self.failed_dirty_usage = failed_dirty_usage; self.failed_dirty_usage = failed_dirty_usage;
self self
+20 -1
View File
@@ -395,6 +395,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
updates: &mpsc::Sender<DataUsageCache>, updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache, mut cache_snapshot: DataUsageCache,
cache_cycle_floor: &AtomicU64, cache_cycle_floor: &AtomicU64,
expected_publication_epoch: u64,
) -> Option<SystemTime> { ) -> Option<SystemTime> {
let source = cache_snapshot.info.source?; let source = cache_snapshot.info.source?;
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await { let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
@@ -489,7 +490,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
let done_save = Metrics::time(Metric::SaveUsage); let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot if let Err(e) = cache_snapshot
.save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions) .save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
.await .await
{ {
error!( error!(
@@ -519,6 +520,24 @@ pub(super) async fn persist_and_publish_cache_snapshot(
); );
return None; return None;
} }
// The persisted-root fast path performs no PUT, so it also needs the
// cycle token re-admission before forwarding the root to the aggregate.
// This final check covers both the fast path and a successful save.
if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch)
.await
.is_none()
{
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "publication_epoch_changed_before_publish",
"Scanner cache root publish skipped after movement epoch change"
);
return None;
}
drop(guard); drop(guard);
let last_update = cache_snapshot.info.last_update; let last_update = cache_snapshot.info.last_update;
+79 -8
View File
@@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks {
all_buckets, all_buckets,
digest: scan_plan_digest, digest: scan_plan_digest,
leader_epoch, leader_epoch,
publication_epoch,
dirty_usage_buckets, dirty_usage_buckets,
bucket_failures, bucket_failures,
pending_maintenance_work, pending_maintenance_work,
@@ -40,6 +41,12 @@ impl ScannerIOCache for SetDisks {
let set_label = self.set_index.to_string(); let set_label = self.set_index.to_string();
let source = DataUsageCacheSource::new(self.pool_index, self.set_index); let source = DataUsageCacheSource::new(self.pool_index, self.set_index);
let expected_publication_epoch = match publication_epoch {
Some(epoch) => epoch,
None => scanner_publication_epoch(self.clone())
.await
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default(); let mut old_cache = DataUsageCache::default();
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await { if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
warn!( warn!(
@@ -76,10 +83,16 @@ impl ScannerIOCache for SetDisks {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
} }
reset_disk_bucket_scan_gauges(&pool_label, &set_label); reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref()) return persist_and_publish_cache_snapshot(
.await self,
.map(|_| ()) &updates,
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope")); cache,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
} }
let (disks, healing) = self.get_online_disks_with_healing(false).await; let (disks, healing) = self.get_online_disks_with_healing(false).await;
@@ -414,6 +427,7 @@ impl ScannerIOCache for SetDisks {
let pending_maintenance_work_clone = pending_maintenance_work.clone(); let pending_maintenance_work_clone = pending_maintenance_work.clone();
let dirty_usage_buckets_clone = dirty_usage_buckets.clone(); let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
let cache_cycle_floor_clone = cache_cycle_floor.clone(); let cache_cycle_floor_clone = cache_cycle_floor.clone();
let expected_publication_epoch_clone = expected_publication_epoch;
let remote_server_epoch = match worker_mode { let remote_server_epoch = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch), NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::Coordinator => None, NamespaceScannerWorkerMode::Coordinator => None,
@@ -753,6 +767,23 @@ impl ScannerIOCache for SetDisks {
); );
continue; continue;
} }
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_before_reuse",
"Current scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
if let Err(e) = if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone) send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await .await
@@ -901,7 +932,12 @@ impl ScannerIOCache for SetDisks {
{ {
let done_save = Metrics::time(Metric::SaveUsage); let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) .save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await .await
{ {
error!( error!(
@@ -958,7 +994,12 @@ impl ScannerIOCache for SetDisks {
false false
} else { } else {
match partial_cache match partial_cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) .save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await .await
{ {
Ok(()) => true, Ok(()) => true,
@@ -1029,7 +1070,12 @@ impl ScannerIOCache for SetDisks {
let done_save = Metrics::time(Metric::SaveUsage); let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions) .save_with_revisions_for_epoch(
store_clone_clone.clone(),
&cache_name,
&revisions,
expected_publication_epoch_clone,
)
.await .await
{ {
done_save(); done_save();
@@ -1064,6 +1110,24 @@ impl ScannerIOCache for SetDisks {
continue; continue;
} }
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch_clone)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_after_save",
"Scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
debug!( debug!(
target: "rustfs::scanner::io", target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM, event = EVENT_SCANNER_DATA_USAGE_STREAM,
@@ -1159,7 +1223,14 @@ impl ScannerIOCache for SetDisks {
cache.info.lkg_scan_plan_digest = None; cache.info.lkg_scan_plan_digest = None;
cache.clone() cache.clone()
}; };
let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await; let _ = persist_and_publish_cache_snapshot(
self.clone(),
&updates,
cache_snapshot,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await;
} else { } else {
let mut incomplete_scope = cache_mutex.lock().await.clone(); let mut incomplete_scope = cache_mutex.lock().await.clone();
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string(); incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();
+21 -3
View File
@@ -68,6 +68,19 @@ impl ScannerIOCycle for ECStore {
)); ));
} }
// Capture one storage-owned movement epoch for the entire cycle. Set
// workers must not each observe a fresh epoch: a movement transition
// between sets would otherwise allow a mixed-generation aggregate.
let publication_epoch = match self.scanner_data_usage_publication_admission().await {
Some(admission) => Some(admission.epoch()),
None => {
return Ok(ScannerCycleResult::new(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
None,
));
}
};
let distributed = self.setup_is_dist_erasure().await; let distributed = self.setup_is_dist_erasure().await;
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) { let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
ScannerActivityPreflight::Ready(snapshot) => snapshot, ScannerActivityPreflight::Ready(snapshot) => snapshot,
@@ -131,7 +144,9 @@ impl ScannerIOCycle for ECStore {
if all_buckets.is_empty() { if all_buckets.is_empty() {
reset_set_scan_gauges(); reset_set_scan_gauges();
if !bucket_plan_complete { if !bucket_plan_complete {
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); return Ok(
ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)
);
} }
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await; let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot); let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
@@ -155,7 +170,7 @@ impl ScannerIOCycle for ECStore {
) )
.await? .await?
{ {
return Ok(ScannerCycleResult::new(status, None)); return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
} }
let dirty_usage_clear = let dirty_usage_clear =
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone()); (status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
@@ -165,6 +180,7 @@ impl ScannerIOCycle for ECStore {
Vec::new() Vec::new()
}; };
return Ok(ScannerCycleResult::new(status, dirty_usage_clear) return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)); .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
} }
@@ -180,7 +196,7 @@ impl ScannerIOCycle for ECStore {
"Scanner set state update detected missing disk sets" "Scanner set state update detected missing disk sets"
); );
reset_set_scan_gauges(); reset_set_scan_gauges();
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
} }
let set_scan_limit = scanner_budgeted_concurrency_limit( let set_scan_limit = scanner_budgeted_concurrency_limit(
@@ -250,6 +266,7 @@ impl ScannerIOCycle for ECStore {
all_buckets: Arc::clone(&all_buckets), all_buckets: Arc::clone(&all_buckets),
digest: scan_plan_digest, digest: scan_plan_digest,
leader_epoch, leader_epoch,
publication_epoch,
dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(), dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(),
bucket_failures: bucket_failures.clone(), bucket_failures: bucket_failures.clone(),
pending_maintenance_work: pending_maintenance_work.clone(), pending_maintenance_work: pending_maintenance_work.clone(),
@@ -430,6 +447,7 @@ impl ScannerIOCycle for ECStore {
Vec::new() Vec::new()
}; };
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear) Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
.with_failed_dirty_usage(!failed_buckets.is_empty()) .with_failed_dirty_usage(!failed_buckets.is_empty())
.with_pending_maintenance_work(pending_maintenance_work) .with_pending_maintenance_work(pending_maintenance_work)
+44
View File
@@ -145,6 +145,50 @@ async fn scanner_cache_locks_allow_cross_source_workers() {
assert!(!second.is_lock_lost()); assert!(!second.is_lock_lost());
} }
#[tokio::test]
async fn scanner_set_cache_admission_tracks_owner_snapshot_and_fails_closed() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let set = store.pools[0].disk_set[0].clone();
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_none(),
"a set must not publish before the owner has refreshed its movement snapshot"
);
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_some(),
"an idle owner snapshot should admit the set cache"
);
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
pool_stats[0] = EcstoreRebalanceStats {
participating: true,
info: EcstoreRebalanceInfo {
start_time: Some(OffsetDateTime::now_utc()),
status: EcstoreRebalStatus::Started,
..Default::default()
},
..Default::default()
};
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
id: Uuid::new_v4().to_string(),
pool_stats,
..Default::default()
});
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_none(),
"active movement must keep set cache publication blocked"
);
*store.rebalance_meta.write().await = None;
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_some(),
"an idle owner refresh must make set cache publication live again"
);
}
#[tokio::test] #[tokio::test]
async fn scanner_cycle_is_deferred_while_rebalance_is_active() { async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await; let (_temp_dir, store) = setup_two_pool_scanner_store().await;