gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence system metadata publication (#6444)

Browse Source 
* feat(scanner): fence usage publication during data movement

* fix(scanner): detect movement refresh state changes

* fix(scanner): fence publication during data movement

* fix(scanner): close movement epoch publication races

* fix(scanner): fence movement-sensitive publication paths

* fix(scanner): fence cache and heal recovery paths

* fix(scanner): carry publication epoch through scan cycle

* fix(scanner): recheck remote cache epoch after save

* fix(scanner): recheck local cache epoch before publish

* fix(scanner): fence data usage writers and baseline

* fix(scanner): expose decommission activity to publication fence

* fix(scanner): release publication gate before reads

* fix(scanner): complete publication fence integration

* fix(scanner): avoid empty usage baseline publication

* chore(scanner): gate test-only helpers

* fix: use decommission canceler in reload test

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-08-23 17:28:21 +08:00
committed by GitHub
parent 9cda615519
commit e196a134cc
26 changed files with 2465 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/core/pools.rs
+155 -33
View File
@@ -135,7 +135,7 @@ struct DecommissionOperation {
}
impl DecommissionCanceler {
fn new(token: CancellationToken) -> Self {
pub(crate) fn new(token: CancellationToken) -> Self {
Self {
operation: Arc::new(DecommissionOperation {
token,
@@ -153,7 +153,7 @@ impl DecommissionCanceler {
&self.operation.token
}
fn is_active(&self) -> bool {
pub(crate) fn is_active(&self) -> bool {
self.operation.active.load(Ordering::Acquire)
}
@@ -1444,6 +1444,30 @@ fn should_replace_pool_status_for_status_refresh(
!has_active_worker && persisted.last_update > current.last_update
}
fn pool_decommission_movement_snapshot(
info: Option<&PoolDecommissionInfo>,
) -> (bool, bool, bool, bool, bool, Option<OffsetDateTime>) {
info.map(|info| {
(
info.has_decommission_state(),
info.complete,
info.failed,
info.canceled,
info.queued,
info.start_time,
)
})
.unwrap_or_default()
}
pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool {
before.pools.len() != after.pools.len()
|| before.pools.iter().zip(after.pools.iter()).any(|(before, after)| {
pool_decommission_movement_snapshot(before.decommission.as_ref())
!= pool_decommission_movement_snapshot(after.decommission.as_ref())
})
}
/// Merges a persisted pool metadata snapshot into `current` monotonically:
/// a pool entry is replaced only when no active worker covers it and the
/// snapshot is strictly newer, so delayed snapshots never roll back local
@@ -3358,6 +3382,8 @@ impl ECStore {
.first()
.cloned()
.ok_or_else(|| Error::other("refresh_pool_status_meta: no pools available"))?;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut persisted = PoolMeta::default();
persisted.load(pool, self.pools.clone()).await?;
@@ -3370,7 +3396,9 @@ impl ECStore {
};
let mut pool_meta = self.pool_meta.write().await;
merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers);
if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
@@ -3489,11 +3517,19 @@ impl ECStore {
async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?;
let _start_guard = self.start_gate.lock().await;
let operation_gate = self.ctx.decommission_operation_gate();
let operation_guard = operation_gate.write().await;
// Lock order: decommission_cancelers before pool_meta. Holding both makes
// owner validation and the terminal transition one atomic operation.
// Signal cancellation before waiting for the movement writer. A worker
// may still hold a publication read guard while it observes this
// signal; waiting for the writer first would deadlock that handoff.
if let Some(owner) = owner {
owner.cancel();
} else if let Some(canceler) = self.decommission_cancelers.read().await.get(idx).and_then(Option::as_ref) {
canceler.cancel();
}
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await;
let mut lock = self.pool_meta.write().await;
@@ -3562,9 +3598,26 @@ impl ECStore {
self.release_decommission_canceler_slot(idx, canceler).await;
}
if should_save_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("decommission_cancel for pool {idx}");
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?;
if let Err(err) =
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
{
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "terminal_reload_failed",
error = %err,
"Decommission cancel saved locally but pool meta reload failed"
);
}
}
Ok(())
@@ -3589,7 +3642,11 @@ impl ECStore {
.unwrap_or((false, false, false, false));
ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?;
}
self.cancel_decommission_routines_and_wait(&[idx]).await;
// Cancel workers before waiting for the movement writer so active
// object operations can observe the signal and release read guards.
self.cancel_decommission_routines(&[idx]).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let (should_reload_pool_meta, previous_pool_meta) = {
let mut pool_meta = self.pool_meta.write().await;
@@ -3606,17 +3663,37 @@ impl ECStore {
return Err(err);
}
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("clear_decommission for pool {idx}");
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?;
if let Err(err) =
resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
{
warn!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_POOLS,
pool_index = idx,
state = "terminal_reload_failed",
error = %err,
"Decommission clear saved locally but pool meta reload failed"
);
}
}
Ok(())
}
async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> {
// Serialize promotion and generation capture with clear/restart transitions.
let (changed, generation, save_error) = {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut pool_meta = self.pool_meta.write().await;
if pool_meta.pools.get(idx).is_none() {
return Err(Error::other("failed to start decommission: target pool was not found"));
@@ -3644,6 +3721,9 @@ impl ECStore {
return Err(err);
}
if changed {
self.ctx.advance_data_movement_operation_epoch();
}
if changed && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("promote_queued_decommission for pool {idx}");
if let Err(err) =
@@ -3668,6 +3748,8 @@ impl ECStore {
}
async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let changed = {
let mut pool_meta = self.pool_meta.write().await;
pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())?
@@ -3708,19 +3790,20 @@ impl ECStore {
is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx))
}
#[cfg(test)]
async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) {
self.cancel_decommission_routines(indices).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
}
async fn cancel_decommission_routines(&self, indices: &[usize]) {
{
let mut cancelers = self.decommission_cancelers.write().await;
for idx in indices {
take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx);
}
}
self.wait_for_decommission_side_effects().await;
}
async fn wait_for_decommission_side_effects(&self) {
let operation_gate = self.ctx.decommission_operation_gate();
let _operation_guard = operation_gate.write().await;
}
async fn reserve_decommission_routines(
@@ -4218,7 +4301,7 @@ impl ECStore {
}
decommission_cancel_signal_result(rx.is_cancelled())?;
self.ensure_decommission_generation_current(idx, generation).await?;
let operation_gate = self.ctx.decommission_operation_gate();
let operation_gate = self.ctx.data_movement_operation_gate();
let bucket_incarnation_fence = match expected_bucket_incarnation_id {
Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
@@ -5113,9 +5196,12 @@ impl ECStore {
{
ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?;
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: decommission_cancelers before pool_meta. Holding both makes
// owner validation and the terminal transition one atomic operation.
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await;
let mut pool_meta = self.pool_meta.write().await;
@@ -5151,6 +5237,12 @@ impl ECStore {
if let Some(canceler) = terminal_canceler.as_ref() {
self.release_decommission_canceler_slot(idx, canceler).await;
}
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("decommission_failed for pool {idx}");
if let Some(err) = observe_decommission_terminal_reload_result(
@@ -5208,7 +5300,12 @@ impl ECStore {
}
self.verify_decommission_durable_ilm_receipts(idx).await?;
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
// Lock order: movement gate, then decommission_cancelers, then pool_meta.
// Holding both state locks makes owner validation and the terminal
// transition one atomic operation.
let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = {
let cancelers = self.decommission_cancelers.read().await;
let mut pool_meta = self.pool_meta.write().await;
@@ -5249,6 +5346,12 @@ impl ECStore {
if let Some(canceler) = terminal_canceler.as_ref() {
self.release_decommission_canceler_slot(idx, canceler).await;
}
if should_reload_pool_meta {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
let stage = format!("complete_decommission for pool {idx}");
if let Some(err) = observe_decommission_terminal_reload_result(
@@ -5479,11 +5582,16 @@ impl ECStore {
self.ensure_decommission_rebalance_idle_after_refresh().await?;
let all_space_infos = self.get_decommission_all_pool_space_infos().await?;
self.cancel_decommission_routines_and_wait(&indices).await;
// Signal cancellation before waiting for the movement writer so active
// object operations can observe the signal and release read guards.
self.cancel_decommission_routines(&indices).await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let index_cancelers = if let Some((rx, local_indices)) = reservation {
// Lock order matches terminal transitions: decommission_cancelers
// before pool_meta while start_gate excludes another start.
// Lock order matches terminal transitions: movement gate, then
// decommission_cancelers, then pool_meta while start_gate excludes
// another start.
let mut cancelers = self.decommission_cancelers.write().await;
let pool_meta = self.pool_meta.read().await;
ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?;
@@ -5505,6 +5613,10 @@ impl ECStore {
let previous_pool_meta = self
.save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets)
.await?;
self.ctx.advance_data_movement_operation_epoch();
// The local durable transition is now fenced. Release the writer
// before any peer RPC; remote reload must not block scanner admission.
drop(_movement_guard);
if let Some(notification_sys) = runtime_sources::notification_sys()
&& let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await)
@@ -5519,11 +5631,20 @@ impl ECStore {
"Decommission start failed after pool metadata save"
);
{
let mut pool_meta = self.pool_meta.write().await;
rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone());
}
if let Err(rollback_save_err) = self.save_current_pool_meta().await {
let rollback_result = {
let movement_guard = movement_gate.write().await;
{
let mut pool_meta = self.pool_meta.write().await;
rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone());
}
let rollback_result = self.save_current_pool_meta().await;
if rollback_result.is_ok() {
self.ctx.advance_data_movement_operation_epoch();
}
drop(movement_guard);
rollback_result
};
if let Err(rollback_save_err) = rollback_result {
error!(
event = EVENT_DECOMMISSION_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -6527,7 +6648,7 @@ impl ECStore {
generation: OffsetDateTime,
) -> Result<()> {
self.ensure_decommission_generation_current(idx, generation).await?;
let operation_gate = self.ctx.decommission_operation_gate();
let operation_gate = self.ctx.data_movement_operation_gate();
run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await
}
@@ -8077,7 +8198,7 @@ mod pools_tests {
..Default::default()
};
merge_pool_status_refresh(&mut current, persisted, &[false]);
assert!(merge_pool_status_refresh(&mut current, persisted, &[false]));
let info = current.pools[0]
.decommission
@@ -8121,7 +8242,7 @@ mod pools_tests {
..Default::default()
};
merge_pool_status_refresh(&mut current, persisted, &[true]);
assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
let info = current.pools[0]
.decommission
@@ -8162,7 +8283,7 @@ mod pools_tests {
..Default::default()
};
merge_pool_status_refresh(&mut current, persisted, &[true]);
assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
let info = current.pools[0]
.decommission
@@ -8591,7 +8712,7 @@ mod pools_tests {
#[tokio::test]
async fn test_decommission_transition_waits_without_registered_canceler() {
let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
let operation_gate = store.ctx.decommission_operation_gate();
let operation_gate = store.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read().await;
let transition = tokio::spawn({
let store = store.clone();
@@ -11338,6 +11459,7 @@ mod pools_tests {
assert!(store.decommission_cancelers.read().await[0].is_none());
assert!(!canceler.is_active());
assert!(canceler.is_cancelled());
assert_eq!(store.ctx.data_movement_operation_epoch(), 1);
}
#[test]
crates/ecstore/src/data_usage/mod.rs
+260 -30
View File
@@ -388,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
));
}
let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
// Prevent older data from overwriting newer persisted stats
if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await
let existing_snapshot = load_data_usage_snapshot(store.clone()).await;
if let Ok((existing, source)) = existing_snapshot
&& source.is_authoritative()
&& let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now())
{
@@ -400,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
return Ok(());
}
save_data_usage_in_backend(data_usage_info, store).await
save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await
}
async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
async fn save_data_usage_in_backend(
data_usage_info: DataUsageInfo,
store: Arc<ECStore>,
expected_publication_epoch: u64,
) -> Result<(), Error> {
let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
// Save to backend using the same mechanism as original code
let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if publication_epoch != expected_publication_epoch {
return Err(Error::other("data usage publication epoch changed before save"));
}
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
.await
.map_err(Error::other)?;
drop(publication_guard);
cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await;
cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
.await;
// Invalidate the cached snapshot so readers observe the new save on their
// next request instead of waiting out the remaining TTL. The next cached
@@ -449,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore {
}
}
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
async fn cleanup_observed_data_usage_after_authoritative_save_with_publication<S>(
store: &S,
authoritative: &DataUsageInfo,
publication_store: Option<&ECStore>,
) where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
let observed_read_epoch = match publication_store {
Some(publication_store) => {
let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else {
return;
};
Some(epoch)
}
None => None,
};
let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await;
let (observed, revision) = match observed_snapshot {
Ok(Some(snapshot)) => snapshot,
Ok(None) => return,
Err(err) => {
@@ -469,6 +498,19 @@ where
return;
}
let publication_guard = match publication_store {
Some(publication_store) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return;
};
if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) {
return;
}
Some(guard)
}
None => None,
};
match store.delete_observed_data_usage_snapshot(&revision).await {
Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {}
Err(err) => {
@@ -479,6 +521,15 @@ where
);
}
}
drop(publication_guard);
}
#[cfg(test)]
async fn cleanup_observed_data_usage_after_authoritative_save<S>(store: &S, authoritative: &DataUsageInfo)
where
S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized,
{
cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await;
}
fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
@@ -519,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str)
pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> {
prepare_bucket_usage_for_namespace_change(bucket, None).await?;
remove_bucket_usage_from_backend_with_guard(store, bucket, None).await
remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await
}
pub(crate) async fn prepare_bucket_usage_for_namespace_change(
@@ -542,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
Ok(())
}
#[cfg(test)]
pub(crate) async fn remove_bucket_usage_from_backend_with_guard<S>(
store: &S,
bucket: &str,
@@ -551,6 +603,24 @@ where
S: EcstoreObjectIO + ?Sized,
{
let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced(
store: &ECStore,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error> {
let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await;
invalidate_bucket_usage_snapshot_caches(guard, bucket).await?;
result
}
async fn invalidate_bucket_usage_snapshot_caches(
guard: Option<&rustfs_lock::NamespaceLockGuard>,
bucket: &str,
) -> Result<(), Error> {
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?;
clear_data_usage_snapshot_cache(&mut snapshot_cache);
@@ -558,7 +628,7 @@ where
let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await;
ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?;
clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache);
result
Ok(())
}
async fn load_data_usage_for_bucket_removal<S>(store: &S, object: &str) -> Result<Option<(DataUsageInfo, String)>, Error>
@@ -617,48 +687,83 @@ fn ensure_bucket_namespace_guard(
Ok(())
}
#[cfg(test)]
async fn remove_bucket_usage_from_backend_with_store_and_guard<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await
}
async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication<S>(
store: &S,
bucket: &str,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication_store: Option<&ECStore>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?;
let primary_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let primary_seed = load_data_usage_seed_for_missing_primary(store).await?;
remove_bucket_usage_from_object_with_retries(
remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&primary_seed),
primary_seed.as_ref(),
guard,
publication_store.map(|store| (store, primary_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?;
let backup_seed_epoch = match publication_store {
Some(publication_store) => Some(
publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?,
),
None => None,
};
let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await?
.map_or(primary_seed, |(data_usage_info, _)| data_usage_info);
remove_bucket_usage_from_object_with_retries(
.map(|(data_usage_info, _)| data_usage_info)
.or_else(|| primary_seed.clone());
remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
Some(&backup_seed),
backup_seed.as_ref(),
guard,
publication_store.map(|store| (store, backup_seed_epoch)),
)
.await?;
ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?;
if let Err(err) = remove_bucket_usage_from_object_with_retries(
if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication(
store,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await
{
@@ -672,12 +777,21 @@ where
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(),
LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(),
] {
remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?;
remove_bucket_usage_from_object_with_retries_and_publication(
store,
object,
bucket,
DATA_USAGE_REMOVE_CAS_RETRIES,
None,
guard,
publication_store.map(|store| (store, None)),
)
.await?;
}
Ok(())
}
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<DataUsageInfo, Error>
async fn load_data_usage_seed_for_missing_primary<S>(store: &S) -> Result<Option<DataUsageInfo>, Error>
where
S: EcstoreObjectIO + ?Sized,
{
@@ -690,12 +804,13 @@ where
if !authoritative {
data_usage_info.usage_snapshot_complete = false;
}
return Ok(data_usage_info);
return Ok(Some(data_usage_info));
}
}
Ok(DataUsageInfo::default())
Ok(None)
}
#[cfg(test)]
async fn remove_bucket_usage_from_object_with_retries<S>(
store: &S,
object: &str,
@@ -704,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries<S>(
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None)
.await
}
async fn remove_bucket_usage_from_object_with_retries_and_publication<S>(
store: &S,
object: &str,
bucket: &str,
cas_retries: usize,
missing_seed: Option<&DataUsageInfo>,
guard: Option<&rustfs_lock::NamespaceLockGuard>,
publication: Option<(&ECStore, Option<u64>)>,
) -> Result<(), Error>
where
S: EcstoreObjectIO + ?Sized,
{
for attempt in 0..=cas_retries {
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?;
let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? {
let read_epoch = match publication {
Some((publication_store, expected_publication_epoch)) => {
let epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?;
if expected_publication_epoch.is_some_and(|expected| expected != epoch) {
return Err(Error::other("data usage publication epoch changed before snapshot read"));
}
Some(epoch)
}
None => None,
};
let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?;
let (mut data_usage_info, revision) = match loaded_snapshot {
Some((data_usage_info, revision)) => (data_usage_info, Some(revision)),
None => match missing_seed {
Some(data_usage_info) => (data_usage_info.clone(), None),
@@ -733,6 +878,22 @@ where
},
};
ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?;
let publication_guard = match publication {
Some((publication_store, expected_publication_epoch)) => {
let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await
else {
return Err(Error::other("data usage publication is blocked by data movement"));
};
if expected_publication_epoch
.or(read_epoch)
.is_some_and(|expected| expected != publication_epoch)
{
return Err(Error::other("data usage publication epoch changed before snapshot commit"));
}
Some(guard)
}
None => None,
};
let save_result = store
.put_object(
RUSTFS_META_BUCKET,
@@ -745,6 +906,7 @@ where
},
)
.await;
drop(publication_guard);
match save_result {
Ok(_) => return Ok(()),
Err(err) => {
@@ -2286,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
#[cfg(test)]
mod tests {
use super::*;
use crate::layout::endpoints::EndpointServerPools;
use crate::runtime::instance::InstanceContext;
use crate::storage_api_contracts::object::ObjectIO as _;
use rustfs_data_usage::BucketUsageInfo;
use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
@@ -2308,6 +2472,7 @@ mod tests {
error_after_commit_put: Option<usize>,
advance_time_on_put: Option<Duration>,
advance_time_after_get: Option<(UsageObjectSlot, Duration)>,
advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc<InstanceContext>)>,
advance_time_before_put: Option<(usize, Duration)>,
advance_time_after_put: Option<(usize, Duration)>,
put_count: usize,
@@ -2385,10 +2550,21 @@ mod tests {
}
_ => None,
};
let advance_publication_epoch = match state.advance_publication_epoch_after_get {
Some((expected_slot, ref ctx)) if expected_slot == slot => {
let ctx = Arc::clone(ctx);
state.advance_publication_epoch_after_get = None;
Some(ctx)
}
_ => None,
};
drop(state);
if let Some(duration) = advance {
tokio::time::advance(duration).await;
}
if let Some(ctx) = advance_publication_epoch {
ctx.advance_data_movement_operation_epoch();
}
Ok(crate::object_api::GetObjectReader {
stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo {
@@ -2589,6 +2765,23 @@ mod tests {
.to_string()
}
fn build_publication_store(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
disk_map: HashMap::new(),
pools: Vec::new(),
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()),
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: TokioMutex::new(()),
pool_meta_save_gate: TokioMutex::new(()),
ctx,
bucket_fence_registry: Arc::default(),
})
}
#[test]
fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() {
// `to_object_err` rewrites the raw storage variants before they reach
@@ -4333,7 +4526,15 @@ mod tests {
.expect("namespace lock acquisition should not fail")
.expect("namespace lock should be acquired"),
);
let store = Arc::new(UsageCasStore::default());
let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now());
let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode");
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((encoded.clone(), 1)),
backup_object: Some((encoded, 1)),
..Default::default()
}),
});
let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now());
let mut snapshot_cache = data_usage_snapshot_cache().write().await;
*snapshot_cache = Some(CachedDataUsageSnapshot {
@@ -4432,21 +4633,17 @@ mod tests {
}
#[tokio::test]
async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() {
async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() {
let store = Arc::new(UsageCasStore::default());
remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a")
.await
.expect("bucket removal should create both usage fences");
.expect("bucket removal should remain a no-op without a usage baseline");
let state = store.state.lock().await;
assert_eq!(state.put_count, 2);
for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() {
let saved = serde_json::from_slice::<DataUsageInfo>(data).expect("saved usage snapshot should decode");
assert_eq!(*revision, 1);
assert!(saved.last_update.is_some());
assert!(!data_usage_contains_bucket(&saved, "bucket-a"));
}
assert_eq!(state.put_count, 0);
assert!(state.object.is_none());
assert!(state.backup_object.is_none());
}
#[tokio::test]
@@ -4598,6 +4795,39 @@ mod tests {
assert_eq!(backup_err, Error::PreconditionFailed);
}
#[tokio::test]
async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() {
let ctx = Arc::new(InstanceContext::new());
let publication_store = build_publication_store(ctx.clone());
let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
let store = Arc::new(UsageCasStore {
state: Mutex::new(UsageCasState {
object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)),
advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)),
..Default::default()
}),
});
let expected_epoch = publication_store
.scanner_data_usage_publication_epoch()
.await
.expect("idle publication store should admit the initial read");
let err = remove_bucket_usage_from_object_with_retries_and_publication(
store.as_ref(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
"bucket-a",
0,
None,
None,
Some((publication_store.as_ref(), Some(expected_epoch))),
)
.await
.expect_err("a movement epoch flip during the read must fence the commit");
assert!(err.to_string().contains("epoch changed"));
assert_eq!(store.state.lock().await.put_count, 0);
}
#[tokio::test]
async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() {
let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now());
crates/ecstore/src/runtime/instance.rs
+76 -8
View File
@@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use s3s::region::Region;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, OnceLock};
use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{OnceCell, RwLock};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0;
const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1;
const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2;
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -160,10 +167,22 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Serializes decommission data-movement operations with cancellation and
/// a subsequent restart. Readers are held across one object side effect;
/// the transition path takes the writer after cancelling the routine.
decommission_operation_gate: Arc<RwLock<()>>,
/// Serializes data-movement transitions with scanner publication commits.
/// Readers are held across one publication commit; movement transitions
/// take the writer at their durable state commit boundary.
data_movement_operation_gate: Arc<RwLock<()>>,
/// Monotonic admission epoch paired with the operation gate. A
/// publication admitted before a movement transition must never be
/// mistaken for one admitted after the transition.
data_movement_operation_epoch: AtomicU64,
/// Once the admission epoch reaches its reserved terminal value, no new
/// publication may be admitted. Keeping this state separate from the
/// saturating counter prevents an unchanged `u64::MAX` value from being
/// mistaken for a fresh epoch after overflow.
data_movement_operation_epoch_exhausted: AtomicBool,
/// Last storage-owned movement snapshot observed under the operation
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
scanner_publication_state: AtomicU8,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
@@ -204,7 +223,10 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
decommission_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_gate: Arc::new(RwLock::new(())),
data_movement_operation_epoch: AtomicU64::new(0),
data_movement_operation_epoch_exhausted: AtomicBool::new(false),
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -223,8 +245,54 @@ impl InstanceContext {
self.lock_manager.clone()
}
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.decommission_operation_gate)
pub(crate) fn data_movement_operation_gate(&self) -> Arc<RwLock<()>> {
Arc::clone(&self.data_movement_operation_gate)
}
pub(crate) fn data_movement_operation_epoch(&self) -> u64 {
self.data_movement_operation_epoch.load(Ordering::Acquire)
}
pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool {
self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire)
}
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
!self.data_movement_operation_epoch_exhausted()
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
}
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
self.scanner_publication_state.store(
if blocked {
SCANNER_PUBLICATION_STATE_BLOCKED
} else {
SCANNER_PUBLICATION_STATE_ALLOWED
},
Ordering::Release,
);
}
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
let _ = self
.data_movement_operation_epoch
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1)));
let result = self.data_movement_operation_epoch.load(Ordering::Acquire);
if result == u64::MAX {
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
}
result
}
#[cfg(test)]
pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) {
self.data_movement_operation_epoch.store(epoch, Ordering::Release);
self.data_movement_operation_epoch_exhausted
.store(epoch == u64::MAX, Ordering::Release);
self.scanner_publication_state
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
}
/// Install the application-owned object-encryption resolver once.
crates/ecstore/src/services/rebalance/control.rs
+76 -11
View File
@@ -48,12 +48,12 @@ fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usi
.unwrap_or_default()
}
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) {
fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
clear_rebalance_status_refresh(current);
return;
return clear_rebalance_status_refresh(current);
}
let before = current.clone();
match current.as_mut() {
Some(current_meta) => {
if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
@@ -66,14 +66,41 @@ fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted
*current = Some(persisted);
}
}
match (before.as_ref(), current.as_ref()) {
(None, None) => false,
(None, Some(_)) | (Some(_), None) => true,
(Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
}
}
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) {
fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
*current = None;
current.take().is_some()
} else {
false
}
}
fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
let Some(current) = current else {
return true;
};
current.id != persisted.id
|| current.stopped_at != persisted.stopped_at
|| current.pool_stats.len() != persisted.pool_stats.len()
|| current
.pool_stats
.iter()
.zip(persisted.pool_stats.iter())
.any(|(current, persisted)| {
current.participating != persisted.participating
|| current.info.status != persisted.info.status
|| current.info.stopping != persisted.info.stopping
})
}
impl ECStore {
pub(super) async fn save_rebalance_meta_with_merge<S>(
&self,
@@ -121,7 +148,10 @@ impl ECStore {
"Loading rebalance metadata"
);
let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if resolve_rebalance_meta_load_result(meta.load(pool).await)? {
let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -130,6 +160,10 @@ impl ECStore {
drop(rebalance_meta);
}
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
debug!(
event = EVENT_REBALANCE_STATE,
@@ -139,10 +173,15 @@ impl ECStore {
"Loaded rebalance metadata"
);
} else {
let movement_changed = self.rebalance_meta.read().await.is_some();
{
let mut rebalance_meta = self.rebalance_meta.write().await;
*rebalance_meta = None;
}
if movement_changed {
self.ctx.advance_data_movement_operation_epoch();
}
drop(_movement_guard);
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -160,14 +199,20 @@ impl ECStore {
pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
let mut persisted = RebalanceMeta::new();
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
match persisted.load(pool).await {
Ok(()) => {
let mut rebalance_meta = self.rebalance_meta.write().await;
merge_rebalance_status_refresh(&mut rebalance_meta, persisted);
if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
self.ctx.advance_data_movement_operation_epoch();
}
}
Err(Error::ConfigNotFound) => {
let mut rebalance_meta = self.rebalance_meta.write().await;
clear_rebalance_status_refresh(&mut rebalance_meta);
if clear_rebalance_status_refresh(&mut rebalance_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
}
Err(err) => {
return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
@@ -349,6 +394,8 @@ impl ECStore {
#[tracing::instrument(skip(self, bucktes))]
pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let decommission_running = self.is_decommission_running().await;
{
@@ -356,12 +403,16 @@ impl ECStore {
validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
}
self.init_rebalance_meta(bucktes).await
let id = self.init_rebalance_meta(bucktes).await?;
self.ctx.advance_data_movement_operation_epoch();
Ok(id)
}
#[tracing::instrument(skip(self))]
pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
{
let rebalance_meta = self.rebalance_meta.read().await;
@@ -379,7 +430,10 @@ impl ECStore {
}
}
self.start_rebalance().await
if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
@@ -548,6 +602,9 @@ impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)
@@ -560,6 +617,7 @@ impl ECStore {
.await,
"stop_rebalance",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -570,6 +628,9 @@ impl ECStore {
expected_id: Option<&str>,
start_error: String,
) -> Result<()> {
let _start_guard = self.start_gate.lock().await;
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
rollback_rebalance_start_meta_snapshot_for_id(
@@ -587,6 +648,7 @@ impl ECStore {
.await,
"rollback_rebalance_start",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -597,6 +659,8 @@ impl ECStore {
return Ok(());
}
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let encoded_error = encode_rebalance_stop_propagation_record(&record);
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -610,6 +674,7 @@ impl ECStore {
.await,
"record_rebalance_stop_propagation",
)?;
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
@@ -682,7 +747,7 @@ mod tests {
..Default::default()
};
merge_rebalance_status_refresh(&mut current, persisted);
assert!(merge_rebalance_status_refresh(&mut current, persisted));
let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
@@ -721,7 +786,7 @@ mod tests {
..Default::default()
};
merge_rebalance_status_refresh(&mut current, persisted);
assert!(!merge_rebalance_status_refresh(&mut current, persisted));
assert!(
current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
crates/ecstore/src/services/rebalance/runtime.rs
+87 -47
View File
@@ -42,6 +42,15 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
impl ECStore {
#[tracing::instrument(skip_all)]
pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
if self.start_rebalance_inner().await? {
self.ctx.advance_data_movement_operation_epoch();
}
Ok(())
}
pub(super) async fn start_rebalance_inner(self: &Arc<Self>) -> Result<bool> {
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
@@ -55,6 +64,7 @@ impl ECStore {
let cancel_tx = CancellationToken::new();
let rx = cancel_tx.clone();
let mut meta_to_save = None;
let mut movement_changed = false;
{
let mut rebalance_meta = self.rebalance_meta.write().await;
@@ -72,14 +82,16 @@ impl ECStore {
reason = "already_in_progress",
"Skipped duplicate rebalance start"
);
return Ok(());
return Ok(false);
}
let now = OffsetDateTime::now_utc();
if complete_rebalance_pools_at_goal(meta, now) {
meta_to_save = Some(meta.clone());
movement_changed = true;
}
if complete_rebalance_pools_with_empty_queue(meta, now) {
meta_to_save = Some(meta.clone());
movement_changed = true;
}
meta.cancel = Some(cancel_tx);
@@ -118,7 +130,7 @@ impl ECStore {
reason = "no_participants",
"Skipped rebalance start because no pools are participating"
);
return Ok(());
return Ok(movement_changed);
}
let mut workers_started = 0usize;
@@ -186,7 +198,7 @@ impl ECStore {
reason = "no_local_participants",
"Skipped rebalance start because no local pools are participating"
);
return Ok(());
return Ok(movement_changed);
}
info!(
@@ -197,7 +209,7 @@ impl ECStore {
worker_count = workers_started,
"Rebalance started"
);
Ok(())
Ok(true)
}
#[tracing::instrument(skip(self, rx))]
@@ -214,53 +226,77 @@ impl ECStore {
let mut quit = false;
loop {
let mut terminal_state_saved = false;
tokio::select! {
result = done_rx.recv() => {
quit = true;
let now = OffsetDateTime::now_utc();
let terminal_event = classify_rebalance_terminal_event(result, now);
msg = terminal_event.message().to_string();
let mut rebalance_meta = store.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() {
let meta_stopped = meta.stopped_at.is_some();
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
&& has_rebalance_cleanup_warnings(pool_stat)
{
pool_stat.info.stopping = false;
pool_stat.info.status = RebalStatus::Failed;
pool_stat.info.end_time = Some(now);
pool_stat.info.last_error = Some(
pool_stat
.cleanup_warnings
.last_message
.clone()
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
);
} else if should_preserve_rebalance_stopped_state(
meta_stopped,
pool_stat.info.status,
&terminal_event,
) {
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
let movement_gate = store.ctx.data_movement_operation_gate();
let movement_guard = movement_gate.write().await;
let previous_meta = store.rebalance_meta.read().await.clone();
let terminal_state_present = {
let mut rebalance_meta = store.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() {
let meta_stopped = meta.stopped_at.is_some();
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
&& has_rebalance_cleanup_warnings(pool_stat)
{
pool_stat.info.stopping = false;
pool_stat.info.status = RebalStatus::Failed;
pool_stat.info.end_time = Some(now);
pool_stat.info.last_error = Some(
pool_stat
.cleanup_warnings
.last_message
.clone()
.unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
);
} else if should_preserve_rebalance_stopped_state(
meta_stopped,
pool_stat.info.status,
&terminal_event,
) {
debug!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "stopped_preserved",
"Preserved stopped rebalance status"
);
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
}
true
} else {
pool_stat.info.stopping = false;
apply_rebalance_terminal_event(
&mut pool_stat.info.status,
&mut pool_stat.info.end_time,
&mut pool_stat.info.last_error,
terminal_event,
now,
);
false
}
} else {
false
}
};
if terminal_state_present {
if let Err(err) = store.save_rebalance_stats_inner(pool_index, RebalSaveOpt::Stats).await {
let mut rebalance_meta = store.rebalance_meta.write().await;
*rebalance_meta = previous_meta;
drop(movement_guard);
return Err(Error::other(format!(
"rebalance terminal state save failed for pool {pool_index}: {err}"
)));
}
store.ctx.advance_data_movement_operation_epoch();
terminal_state_saved = true;
}
}
_ = timer.tick() => {
@@ -269,7 +305,7 @@ impl ECStore {
}
}
if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
if !terminal_state_saved && let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await {
let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
error!("{} err: {:?}", msg, wrapped);
if quit {
@@ -590,16 +626,14 @@ impl ECStore {
meta.percent_free_goal,
)
{
pool_stat.info.status = RebalStatus::Completed;
pool_stat.info.end_time = Some(OffsetDateTime::now_utc());
info!(
event = EVENT_REBALANCE_STATE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REBALANCE,
pool_index,
state = "completed",
state = "completion_ready",
percent_free = pfi,
"Marked rebalance pool completed"
"Rebalance pool reached completion goal"
);
return true;
}
@@ -612,6 +646,12 @@ impl ECStore {
impl ECStore {
#[tracing::instrument(skip(self))]
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
self.save_rebalance_stats_inner(pool_idx, opt).await
}
pub(super) async fn save_rebalance_stats_inner(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
let meta_to_save = {
let mut rebalance_meta = self.rebalance_meta.write().await;
let Some(meta) = rebalance_meta.as_mut() else {
crates/ecstore/src/set_disk/mod.rs
+20
View File
@@ -3614,6 +3614,26 @@ impl SetDisks {
&self.ctx
}
/// Admit one short scanner cache publication under this set's instance
/// movement fence. The caller must hold the returned guard through its
/// final conditional cache write; no scan-round work belongs under it.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.scanner_publication_state_allowed() {
let epoch = self.ctx.data_movement_operation_epoch();
return Some((operation_guard, epoch));
}
// The owner deliberately marks the cached state UNKNOWN after every
// movement epoch advance. Do not strand remote scanner writers in that
// state: release this guard before asking the storage owner to refresh
// its durable movement snapshot, since the owner uses the same gate.
drop(operation_guard);
let owner = runtime_sources::object_store_handle().filter(|owner| Arc::ptr_eq(&owner.ctx, &self.ctx))?;
owner.scanner_data_usage_publication_admission_guard().await
}
/// Whether both sets' namespace-lock implementations cover the same object key.
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
crates/ecstore/src/store/bucket.rs
+1 -1
View File
@@ -327,7 +327,7 @@ impl ECStore {
async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
run_bucket_usage_cleanup(guard, bucket, async {
crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await
crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
})
.await
}
crates/ecstore/src/store/init.rs
+5
View File
@@ -495,6 +495,11 @@ impl ECStore {
);
}
// Initialize the storage-owned scanner publication state only after
// both movement metadata sources have been loaded. SetDisks cache
// writers remain fail-closed until this snapshot is available.
let _ = self.scanner_data_usage_publication_blocked().await;
let pools = installed_pool_meta.return_resumable_pools();
let mut pool_indices = Vec::with_capacity(pools.len());
crates/ecstore/src/store/mod.rs
+149 -5
View File
@@ -44,7 +44,7 @@ use crate::error::{
use crate::runtime::global::DISK_RESERVE_FRACTION;
use crate::runtime::instance::InstanceContext;
use crate::runtime::sources as runtime_sources;
use crate::services::rebalance::RebalanceMeta;
use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -349,16 +349,81 @@ impl ECStore {
/// remain suspended until an operator clears or retries them, so they are
/// a publication barrier even after the worker has stopped.
pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
if self.scanner_data_movement_active().await {
let operation_gate = self.ctx.data_movement_operation_gate();
let _operation_guard = operation_gate.read_owned().await;
self.scanner_data_usage_publication_snapshot_blocked().await
}
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
if self.ctx.data_movement_operation_epoch_exhausted() {
self.ctx.set_scanner_publication_state(true);
return true;
}
let decommission_cancelers = self.decommission_cancelers.read().await;
let decommission_active = decommission_cancelers
.iter()
.any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
let pool_meta = self.pool_meta.read().await;
pool_meta.pools.iter().any(|pool| {
let decommission_active = decommission_active
|| pool_meta.pools.iter().any(|pool| {
pool.decommission
.as_ref()
.is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
});
let decommission_terminal = pool_meta.pools.iter().any(|pool| {
pool.decommission
.as_ref()
.is_some_and(|info| !info.queued && (info.failed || info.canceled))
})
});
drop(pool_meta);
let rebalance_active = self
.rebalance_meta
.read()
.await
.as_ref()
.is_some_and(is_rebalance_conflicting_with_decommission);
let blocked = decommission_active || decommission_terminal || rebalance_active;
self.ctx.set_scanner_publication_state(blocked);
blocked
}
/// Admit one short data-usage publication commit under the same
/// per-instance gate used by decommission side effects and transitions.
/// The epoch is sampled while the read guard is held, so a transition
/// cannot cross this admission without waiting for the commit to finish.
pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
let epoch = self.ctx.data_movement_operation_epoch();
(operation_guard, epoch)
}
/// Acquire the movement gate and inspect the movement owner once. The
/// state inspection is performed after acquiring the read guard so a
/// transition cannot update its durable state between the check and the
/// publication commit.
pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_operation_epoch_exhausted() {
return None;
}
if self.scanner_data_usage_publication_snapshot_blocked().await {
return None;
}
Some((operation_guard, self.ctx.data_movement_operation_epoch()))
}
/// Capture the current publication epoch without holding the movement
/// gate across backend I/O. Callers must re-admit the same epoch before a
/// mutation commits.
pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
drop(operation_guard);
Some(epoch)
}
}
@@ -995,6 +1060,85 @@ mod tests {
}
}
#[tokio::test]
async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let operation_gate = store.ctx.data_movement_operation_gate();
let movement_guard = operation_gate.write().await;
let pending = {
let store = store.clone();
tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
};
tokio::task::yield_now().await;
assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
drop(movement_guard);
let (_, epoch) = pending
.await
.expect("publication admission task should not panic")
.expect("idle store should admit publication");
assert_eq!(epoch, 0);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
let (_, next_epoch) = store
.scanner_data_usage_publication_admission_guard()
.await
.expect("idle store should admit the next publication");
assert_eq!(next_epoch, 1);
}
#[tokio::test]
async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
let epoch = store
.scanner_data_usage_publication_epoch()
.await
.expect("idle store should expose a publication epoch");
assert_eq!(epoch, 0);
let operation_gate = store.ctx.data_movement_operation_gate();
let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
.await
.expect("epoch capture must not hold the movement gate across backend I/O");
}
#[tokio::test]
async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
*store.rebalance_meta.write().await = Some(RebalanceMeta {
pool_stats: vec![crate::services::rebalance::RebalanceStats {
participating: true,
info: crate::services::rebalance::RebalanceInfo {
status: crate::services::rebalance::RebalStatus::Started,
..Default::default()
},
..Default::default()
}],
..Default::default()
});
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"active rebalance must fail closed at the storage-owned admission boundary"
);
}
#[tokio::test]
async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(
store.scanner_data_usage_publication_admission_guard().await.is_none(),
"publication must fail closed at the reserved terminal epoch"
);
assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
assert!(store.ctx.data_movement_operation_epoch_exhausted());
assert!(store.scanner_data_usage_publication_blocked().await);
}
// The object graph is the isolation carrier: two ECStore instances holding
// distinct contexts report independent erasure state through their real
// `&self` accessors — no cross-contamination.
crates/ecstore/src/store/rebalance.rs
+14 -2
View File
@@ -694,6 +694,11 @@ impl ECStore {
/// callers only trigger missing-worker recovery after a real state change;
/// delayed snapshots are merged monotonically and never blind-assigned.
pub async fn reload_pool_meta(&self) -> Result<bool> {
// Serialize the durable reload with local movement transitions. Loading
// before acquiring this gate would allow a stale disk snapshot to
// overwrite a newer local transition after the writer commits.
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.write().await;
let mut reloaded = PoolMeta::default();
resolve_store_rebalance_pool_meta_reload_result(
reloaded.load(self.pools[0].clone(), self.pools.clone()).await,
@@ -701,15 +706,22 @@ impl ECStore {
)?;
// Lock order: release the decommission_cancelers guard before taking
// the pool_meta write guard; neither is held across the disk read.
// the pool_meta write guard; neither is held without the movement gate.
let active_workers = {
let cancelers = self.decommission_cancelers.read().await;
cancelers.iter().map(Option::is_some).collect::<Vec<_>>()
cancelers
.iter()
.map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
.collect::<Vec<_>>()
};
let incoming_has_pools = !reloaded.pools.is_empty();
let mut pool_meta = self.pool_meta.write().await;
let movement_before = pool_meta.clone();
let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
self.ctx.advance_data_movement_operation_epoch();
}
if !merged_newer && !incoming_has_pools {
warn!(
crates/scanner/src/data_usage_define.rs
+17 -9
View File
@@ -36,13 +36,13 @@ use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use tokio::time::{Duration, Instant, sleep, timeout};
use tracing::{debug, warn};
use crate::ScannerObjectIO;
use crate::storage_api::owner::HTTPPreconditions;
use crate::{
BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig,
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, TRANSITION_COMPLETE, save_config,
save_config_with_preconditions, storageclass,
SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError,
TRANSITION_COMPLETE, save_config, save_config_with_preconditions, scanner_publication_admission_for_epoch, storageclass,
};
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
// Data usage constants
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
@@ -119,9 +119,13 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag")))?;
Ok((Some(reader.read_all().await?), revision))
}
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
Ok((None, DataUsageCacheRevision::Missing))
}
Err(
Error::ConfigNotFound
| Error::FileNotFound
| Error::VolumeNotFound
| Error::ObjectNotFound(_, _)
| Error::BucketNotFound(_),
) => Ok((None, DataUsageCacheRevision::Missing)),
Err(err) => Err(err),
}
}
@@ -147,9 +151,13 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
.filter(|etag| !etag.is_empty())
.map(DataUsageCacheRevision::Etag)
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
Ok(DataUsageCacheRevision::Missing)
}
Err(
Error::ConfigNotFound
| Error::FileNotFound
| Error::VolumeNotFound
| Error::ObjectNotFound(_, _)
| Error::BucketNotFound(_),
) => Ok(DataUsageCacheRevision::Missing),
Err(err) => Err(err),
}
}
crates/scanner/src/data_usage_define/persistence.rs
+85 -6
View File
@@ -15,6 +15,42 @@
use super::*;
use std::sync::Arc;
#[derive(Debug)]
struct CachePublicationAdmissionUnavailable;
impl std::fmt::Display for CachePublicationAdmissionUnavailable {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("scanner cache publication admission is unavailable")
}
}
impl std::error::Error for CachePublicationAdmissionUnavailable {}
fn cache_publication_admission_unavailable() -> Error {
Error::Io(std::io::Error::other(CachePublicationAdmissionUnavailable))
}
fn cache_publication_epoch_changed() -> Error {
Error::other(SCANNER_PUBLICATION_EPOCH_CHANGED)
}
fn is_cache_publication_admission_unavailable(error: &StorageError) -> bool {
matches!(
error,
StorageError::Io(io_error)
if io_error
.get_ref()
.is_some_and(|source| source.downcast_ref::<CachePublicationAdmissionUnavailable>().is_some())
)
}
fn is_cache_publication_epoch_changed(error: &StorageError) -> bool {
matches!(
error,
StorageError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED
)
}
pub(super) enum DataUsageCacheLoadAttempt {
Loaded {
cache: Box<DataUsageCache>,
@@ -368,6 +404,12 @@ impl DataUsageCache {
fn should_retry_save_error(err: &StorageError) -> bool {
// Usage-cache files are best-effort scanner checkpoints. Retrying namespace
// lock failures immediately only adds more lock traffic to the same hot object.
if is_cache_publication_admission_unavailable(err) {
return false;
}
if is_cache_publication_epoch_changed(err) {
return false;
}
!matches!(
err,
StorageError::Lock(_)
@@ -423,13 +465,14 @@ impl DataUsageCache {
Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string())))
}
async fn save_path_with_retry<S: ScannerObjectIO>(
async fn save_path_with_retry<S: ScannerObjectIO + ScannerConfigObjectDelete>(
store: Arc<S>,
path: &str,
buf: &[u8],
timeout_duration: Duration,
max_retries: u32,
revision: Option<DataUsageCacheRevision>,
expected_epoch: Option<u64>,
) -> StorageResult<()> {
Self::ensure_cache_save_metrics_registered();
let path_type = Self::cache_path_type(path);
@@ -441,6 +484,17 @@ impl DataUsageCache {
let buf_clone = buf.to_vec();
let revision = revision.clone();
async move {
let publication_admission = match expected_epoch {
Some(expected_epoch) => scanner_publication_admission_for_epoch(store_clone.clone(), expected_epoch).await,
None => store_clone.scanner_data_usage_publication_admission().await,
};
let Some(_publication_admission) = publication_admission else {
return Err(if expected_epoch.is_some() {
cache_publication_epoch_changed()
} else {
cache_publication_admission_unavailable()
});
};
if let Some(revision) = revision {
save_config_with_preconditions(store_clone, &path_clone, buf_clone, revision.preconditions()).await?;
} else {
@@ -454,6 +508,14 @@ impl DataUsageCache {
return Ok(());
};
// An epoch-specific admission failure is authoritative: reconciling
// identical bytes cannot prove that this snapshot belongs to the
// captured movement epoch. Do not turn that fence failure into a
// successful stale publication.
if is_cache_publication_epoch_changed(&save_err) {
return Err(save_err);
}
for attempt in 0..=max_retries {
let reconcile = timeout(timeout_duration, async {
let mut reader = store
@@ -486,24 +548,36 @@ impl DataUsageCache {
Err(save_err)
}
pub async fn save<S: ScannerObjectIO>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
self.save_inner(store, name, None).await
pub async fn save<S: ScannerObjectIO + ScannerConfigObjectDelete>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
self.save_inner(store, name, None, None).await
}
pub(crate) async fn save_with_revisions<S: ScannerObjectIO>(
#[cfg(test)]
pub(crate) async fn save_with_revisions<S: ScannerObjectIO + ScannerConfigObjectDelete>(
&self,
store: Arc<S>,
name: &str,
revisions: &DataUsageCacheRevisions,
) -> StorageResult<()> {
self.save_inner(store, name, Some(revisions)).await
self.save_inner(store, name, Some(revisions), None).await
}
async fn save_inner<S: ScannerObjectIO>(
pub(crate) async fn save_with_revisions_for_epoch<S: ScannerObjectIO + ScannerConfigObjectDelete>(
&self,
store: Arc<S>,
name: &str,
revisions: &DataUsageCacheRevisions,
expected_epoch: u64,
) -> StorageResult<()> {
self.save_inner(store, name, Some(revisions), Some(expected_epoch)).await
}
async fn save_inner<S: ScannerObjectIO + ScannerConfigObjectDelete>(
&self,
store: Arc<S>,
name: &str,
revisions: Option<&DataUsageCacheRevisions>,
expected_epoch: Option<u64>,
) -> StorageResult<()> {
let mut buf = Vec::new();
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
@@ -517,6 +591,7 @@ impl DataUsageCache {
timeout_duration,
DATA_USAGE_CACHE_SAVE_RETRIES,
revisions.map(|revisions| revisions.main.clone()),
expected_epoch,
)
.await?;
@@ -534,6 +609,7 @@ impl DataUsageCache {
backup_timeout_duration,
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
backup_revision,
expected_epoch,
)
.await
{
@@ -548,6 +624,9 @@ impl DataUsageCache {
error = %e,
"Scanner cache backup save failed"
);
if is_cache_publication_admission_unavailable(&e) || is_cache_publication_epoch_changed(&e) {
return Err(e);
}
}
Ok(())
}
crates/scanner/src/data_usage_define/tests.rs
+48
View File
@@ -198,6 +198,22 @@ impl ObjectIO for CacheReadStore {
}
}
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for CacheReadStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_opts: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
#[async_trait::async_trait]
impl ObjectIO for AmbiguousCacheCommitStore {
type Error = Error;
@@ -243,6 +259,22 @@ impl ObjectIO for AmbiguousCacheCommitStore {
}
}
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for AmbiguousCacheCommitStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_opts: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
#[async_trait::async_trait]
impl ObjectIO for BackupFallbackStore {
type Error = Error;
@@ -314,6 +346,22 @@ impl ObjectIO for BackupFallbackStore {
}
}
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for BackupFallbackStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_opts: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
#[test]
fn cache_revisions_map_to_compare_and_swap_preconditions() {
let missing = DataUsageCacheRevision::Missing.preconditions();
crates/scanner/src/lib.rs
+124
View File
@@ -513,6 +513,75 @@ where
.await
}
pub(crate) async fn save_config_with_publication_admission_for_epoch<S>(
api: Arc<S>,
file: &str,
data: Vec<u8>,
preconditions: HTTPPreconditions,
expected_epoch: u64,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO + ScannerConfigObjectDelete,
{
let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
save_config_with_preconditions(api, file, data, preconditions).await
}
pub(crate) const SCANNER_PUBLICATION_EPOCH_CHANGED: &str = "scanner publication epoch changed before commit";
pub(crate) fn scanner_publication_epoch_changed(error: &EcstoreError) -> bool {
matches!(
error,
EcstoreError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED
)
}
pub(crate) async fn delete_config_with_publication_admission_for_epoch<S>(
api: Arc<S>,
bucket: &str,
object: &str,
opts: ScannerObjectOptions,
expected_epoch: u64,
) -> EcstoreResult<ScannerObjectInfo>
where
S: ScannerObjectIO + ScannerConfigObjectDelete,
{
let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
api.delete_config_object(bucket, object, opts).await
}
/// Capture the storage-owned publication epoch without retaining the read
/// guard across a potentially slow metadata read. Callers must compare this
/// token with a fresh admission immediately before their conditional write.
pub(crate) async fn scanner_publication_epoch<S>(api: Arc<S>) -> Option<u64>
where
S: ScannerConfigObjectDelete,
{
let admission = api.scanner_data_usage_publication_admission().await?;
Some(admission.epoch())
}
/// Re-admit a publication only when the storage-owned movement epoch is still
/// the one observed before the caller's metadata read. The returned guard
/// remains held through the caller's short conditional commit.
pub(crate) async fn scanner_publication_admission_for_epoch<S>(
api: Arc<S>,
expected_epoch: u64,
) -> Option<ScannerDataUsagePublicationAdmission>
where
S: ScannerConfigObjectDelete,
{
let admission = api.scanner_data_usage_publication_admission().await?;
if admission.epoch() != expected_epoch {
return None;
}
Some(admission)
}
pub(crate) async fn save_config_shared_with_preconditions<S>(
api: Arc<S>,
file: &str,
@@ -587,6 +656,39 @@ pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static {
object: &str,
opts: ScannerObjectOptions,
) -> EcstoreResult<ScannerObjectInfo>;
/// Acquire storage-owned admission for one short data-usage publication
/// commit. Implementations without a storage-owned movement owner fail
/// closed; test fixtures opt into the explicit unfenced helper.
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
None
}
}
pub struct ScannerDataUsagePublicationAdmission {
epoch: u64,
_read_guard: Option<tokio::sync::OwnedRwLockReadGuard<()>>,
}
impl ScannerDataUsagePublicationAdmission {
#[cfg(test)]
pub(crate) fn unfenced() -> Self {
Self {
epoch: 0,
_read_guard: None,
}
}
fn fenced(read_guard: tokio::sync::OwnedRwLockReadGuard<()>, epoch: u64) -> Self {
Self {
epoch,
_read_guard: Some(read_guard),
}
}
pub(crate) fn epoch(&self) -> u64 {
self.epoch
}
}
#[async_trait::async_trait]
@@ -599,6 +701,28 @@ impl ScannerConfigObjectDelete for ECStore {
) -> EcstoreResult<ScannerObjectInfo> {
ObjectOperations::delete_object(self, bucket, object, opts).await
}
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
}
}
#[async_trait::async_trait]
impl ScannerConfigObjectDelete for SetDisks {
async fn delete_config_object(
&self,
bucket: &str,
object: &str,
opts: ScannerObjectOptions,
) -> EcstoreResult<ScannerObjectInfo> {
ObjectOperations::delete_object(self, bucket, object, opts).await
}
async fn scanner_data_usage_publication_admission(&self) -> Option<ScannerDataUsagePublicationAdmission> {
let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch))
}
}
#[cfg(test)]
crates/scanner/src/remote_scanner/stream.rs
+26 -1
View File
@@ -23,6 +23,7 @@ use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
use crate::{
DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo,
DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle,
scanner_publication_admission_for_epoch, scanner_publication_epoch,
};
use hmac::{Hmac, KeyInit, Mac};
use rustfs_common::heal_channel::HealScanMode;
@@ -686,6 +687,9 @@ async fn scan_and_persist_local_bucket(
source.pool_index, source.set_index
))
})?;
let expected_publication_epoch = scanner_publication_epoch(set.clone()).await.ok_or_else(|| {
RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement")
})?;
let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]);
let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source)
.await
@@ -708,6 +712,14 @@ async fn scan_and_persist_local_bucket(
"remote namespace scanner cache lock was lost before reusing the current snapshot",
));
}
if scanner_publication_admission_for_epoch(set.clone(), expected_publication_epoch)
.await
.is_none()
{
return Err(RemoteScannerServerError::retry_bucket(
"remote namespace scanner cache publication epoch changed before reusing the current snapshot",
));
}
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
source,
scan_plan_digest,
@@ -796,9 +808,22 @@ async fn scan_and_persist_local_bucket(
.await
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?;
let done_save = Metrics::time(Metric::SaveUsage);
let save_result = cache.save_with_revisions(set, &cache_name, &revisions).await;
// Each physical main/backup PUT must still prove the epoch captured before
// the scan. A movement transition that starts and ends during the scan
// therefore cannot admit the stale cache under the new epoch.
let save_result = cache
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch)
.await;
done_save();
save_result.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache save failed: {err}")))?;
if scanner_publication_admission_for_epoch(set, expected_publication_epoch)
.await
.is_none()
{
return Err(RemoteScannerServerError::retry_bucket(
"remote namespace scanner cache publication epoch changed after persistence",
));
}
validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store)
.await
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?;
crates/scanner/src/scanner.rs
+214 -33
View File
@@ -18,6 +18,7 @@ use std::future::Future;
use std::sync::Mutex as StdMutex;
use std::sync::{Arc, LazyLock, RwLock};
use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch};
use crate::data_usage_define::{
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
@@ -30,8 +31,9 @@ use crate::runtime_config::{
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
use crate::scanner_io::{
ScannerCycleDeferReason, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified,
dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed,
scanner_maintenance_generation,
};
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
@@ -66,10 +68,12 @@ use crate::storage_api::scan::{
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
};
use crate::{
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
get_lifecycle_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache,
invalidate_data_usage_snapshot_cache, read_config, replace_bucket_usage_memory_from_info, save_config,
save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd,
ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions,
save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch,
scanner_publication_epoch, scanner_publication_epoch_changed,
};
const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -346,6 +350,23 @@ fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool {
!info.is_complete_bucket_usage_snapshot()
}
pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageInfo) -> bool {
if info.is_complete_bucket_usage_snapshot() {
return true;
}
// Pre-marker snapshots remain readable only when their legacy identity is
// complete: a timestamp, a scanner cycle, and an exact bucket cardinality.
// A current snapshot with only scanner_epoch/scanner_cycle (or an explicit
// incomplete marker) is not evidence of a durable usage baseline.
!info.usage_snapshot_complete
&& info.scanner_epoch.is_none()
&& info.usage_snapshot_converged != Some(false)
&& info.last_update.is_some()
&& info.scanner_cycle.is_some()
&& u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count)
}
fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool {
data_usage_info_is_cold(authoritative)
|| observed.is_some_and(|observed| observed_data_usage_is_newer(observed, authoritative))
@@ -381,9 +402,18 @@ fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool {
.is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0)
}
#[cfg(test)]
async fn sync_data_usage_backup_from_primary(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<(), EcstoreError> {
sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await
}
async fn sync_data_usage_backup_from_primary_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_publication_epoch: Option<u64>,
) -> Result<(), EcstoreError> {
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
@@ -391,26 +421,62 @@ async fn sync_data_usage_backup_from_primary(
return Ok(());
}
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
expected_epoch
}
None => scanner_publication_epoch(storeapi.clone())
.await
.ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?,
};
let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
let primary = primary.ok_or_else(|| EcstoreError::other("authoritative data usage snapshot is missing"))?;
serde_json::from_slice::<DataUsageInfo>(&primary)
let primary = primary.ok_or(EcstoreError::ConfigNotFound)?;
let primary_info = serde_json::from_slice::<DataUsageInfo>(&primary)
.map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?;
if !data_usage_info_has_persisted_baseline_identity(&primary_info) {
return Err(EcstoreError::other(
"authoritative data usage snapshot has no persisted baseline identity",
));
}
let primary = Bytes::from(primary);
let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?;
if backup.as_deref() == Some(primary.as_ref()) {
return Ok(());
if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_some()
{
return Ok(());
}
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower));
let save_result = save_config_shared_with_preconditions(
storeapi.clone(),
&backup_path,
primary.clone(),
sha256hex,
revision.preconditions(),
)
.await;
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
save_config_shared_with_preconditions(
storeapi.clone(),
&backup_path,
primary.clone(),
sha256hex,
revision.preconditions(),
)
.await
};
match save_result {
Ok(_) => {}
@@ -427,9 +493,20 @@ async fn sync_data_usage_backup_from_primary(
}
let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?;
if current_primary.as_deref() == Some(primary.as_ref()) {
if current_primary.as_deref() == Some(primary.as_ref())
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_some()
{
return Ok(());
}
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
}
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
@@ -1052,7 +1129,7 @@ async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
lock_lost: LockLost,
) -> bool
where
Store: ScannerObjectIO,
Store: ScannerObjectIO + ScannerConfigObjectDelete,
LockLost: Future<Output = ()>,
{
let fence_ctx = ctx.child_token();
@@ -1089,7 +1166,7 @@ async fn handle_scanner_cycle_deadline<Store>(
worker_stopped: bool,
guard: &mut NamespaceLockGuard,
) where
Store: ScannerObjectIO,
Store: ScannerObjectIO + ScannerConfigObjectDelete,
{
let fenced = fence_scanner_epoch_after_cycle_timeout(
ctx,
@@ -1182,7 +1259,33 @@ async fn run_data_scanner_cycle_with_budget(
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
let mut background_heal_info = read_background_heal_info(storeapi.clone()).await;
// Refresh the storage-owned movement snapshot before reading background
// heal state. A missing heal object yields an in-memory default; do not
// let that default influence a cycle while publication is blocked.
if storeapi.scanner_data_usage_publication_blocked().await {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
let background_heal_read = read_background_heal_info_with_epoch(storeapi.clone()).await;
match background_heal_read.status {
BackgroundHealInfoReadStatus::Blocked => {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
BackgroundHealInfoReadStatus::Transient => {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
}
BackgroundHealInfoReadStatus::Failed => {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
}
BackgroundHealInfoReadStatus::ErasureSd
| BackgroundHealInfoReadStatus::Loaded
| BackgroundHealInfoReadStatus::Missing => {}
}
let mut background_heal_info = background_heal_read.info;
let background_heal_epoch = background_heal_read.expected_epoch;
let scan_mode = get_cycle_scan_mode(
cycle_info.current,
@@ -1209,11 +1312,23 @@ async fn run_data_scanner_cycle_with_budget(
configured_bitrot_cycle,
) {
background_heal_info = new_heal_info.clone();
save_background_heal_info(storeapi.clone(), new_heal_info).await;
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
}
let cycle_start = std::time::Instant::now();
let usage_persist_baseline = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
// Baseline reads are part of the same publication proof as the eventual
// scanner aggregate. Hold only the short storage-owned admission guard
// across this metadata read; the full bucket scan runs after it is
// released and carries the captured epoch forward.
let Some((baseline_publication_guard, baseline_publication_epoch)) =
storeapi.scanner_data_usage_publication_admission_guard().await
else {
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
let usage_persist_baseline_result = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await;
drop(baseline_publication_guard);
let usage_persist_baseline = match usage_persist_baseline_result {
Ok((data, revision)) => DataUsagePersistBaseline {
data: data.map(Bytes::from),
revision,
@@ -1250,9 +1365,17 @@ async fn run_data_scanner_cycle_with_budget(
)
.await;
let publication_defer_reason = match &scan_result {
Ok(result)
if result
.publication_epoch()
.is_some_and(|publication_epoch| publication_epoch != baseline_publication_epoch) =>
{
Some(ScannerCycleDeferReason::DataMovement)
}
Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await,
Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable),
};
let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch);
let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled();
let usage_persist_outcome = match publication_defer_reason {
Some(reason) => {
@@ -1267,12 +1390,13 @@ async fn run_data_scanner_cycle_with_budget(
let ctx_clone = ctx.clone();
let route_probe_store = storeapi.clone();
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
ctx_clone,
storeapi_clone,
receiver,
Some(leader_epoch),
Some(usage_persist_baseline),
publication_epoch,
move || {
let storeapi = route_probe_store.clone();
async move { storeapi.scanner_data_usage_publication_blocked().await }
@@ -1344,7 +1468,7 @@ async fn run_data_scanner_cycle_with_budget(
if !ctx.is_cancelled()
&& let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false)
{
save_background_heal_info(storeapi.clone(), new_heal_info).await;
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
}
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
@@ -1377,19 +1501,29 @@ async fn run_data_scanner_cycle_with_budget(
"Scanner cycle is recovering to a newer durable cache generation"
);
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
let persisted = persist_required_scanner_cycle_floor(
let persisted = persist_required_scanner_cycle_floor_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
cycle_revision,
leader_epoch,
required_cycle,
&mut cycle_metrics_guard,
ScannerCycleFloorOptions {
required_cycle,
expected_publication_epoch: publication_epoch,
},
)
.await;
return if persisted {
cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial
} else if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
} else {
ScannerCycleOutcome::Failed
};
@@ -1446,18 +1580,26 @@ async fn run_data_scanner_cycle_with_budget(
scan_cycle_partial_reason(budget_reason),
scan_cycle_partial_source(budget_reason),
);
let persisted = finalize_partial_scan_cycle(
let persisted = finalize_partial_scan_cycle_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
cycle_revision,
leader_epoch,
&mut cycle_metrics_guard,
publication_epoch,
)
.await;
return if persisted {
cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial
} else if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
} else {
ScannerCycleOutcome::Failed
};
@@ -1531,18 +1673,26 @@ async fn run_data_scanner_cycle_with_budget(
);
}
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
let persisted = finalize_partial_scan_cycle(
let persisted = finalize_partial_scan_cycle_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
cycle_revision,
leader_epoch,
&mut cycle_metrics_guard,
publication_epoch,
)
.await;
return if persisted {
cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial
} else if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)
} else {
ScannerCycleOutcome::Failed
};
@@ -1572,13 +1722,14 @@ async fn run_data_scanner_cycle_with_budget(
state = "superseded",
"Scanner cycle usage snapshot was superseded by concurrent namespace activity"
);
if finalize_partial_scan_cycle(
if finalize_partial_scan_cycle_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
cycle_revision,
leader_epoch,
&mut cycle_metrics_guard,
publication_epoch,
)
.await
{
@@ -1586,11 +1737,29 @@ async fn run_data_scanner_cycle_with_budget(
emit_scan_cycle_superseded(cycle_start.elapsed());
return ScannerCycleOutcome::Superseded;
}
if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
emit_scan_cycle_complete(false, cycle_start.elapsed());
return ScannerCycleOutcome::Failed;
}
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {}
}
if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
emit_scan_cycle_deferred(cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
let previous_cycle_info = cycle_info.clone();
if let Err(err) = advance_scanner_cycle(cycle_info) {
error!(
target: "rustfs::scanner",
@@ -1610,7 +1779,19 @@ async fn run_data_scanner_cycle_with_budget(
global_metrics().clear_current_scan_mode();
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
if !persist_scanner_cycle_state_for_epoch(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, publication_epoch)
.await
{
if let Some(expected_epoch) = publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
emit_scan_cycle_deferred(cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
cycle_metrics_guard.finish(cycle_info.clone()).await;
emit_scan_cycle_complete(false, cycle_start.elapsed());
return ScannerCycleOutcome::Failed;
@@ -1620,7 +1801,7 @@ async fn run_data_scanner_cycle_with_budget(
done_cycle();
emit_scan_cycle_complete(true, cycle_start.elapsed());
if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) {
save_background_heal_info(storeapi.clone(), new_heal_info).await;
save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await;
}
info!(
crates/scanner/src/scanner/cycle_state.rs
+293 -53
View File
@@ -162,6 +162,8 @@ pub(crate) enum ScannerCycleStateStartup {
enum CycleRecoveryMarkerReadError {
#[error("cycle recovery marker backend read failed: {0}")]
Backend(#[source] EcstoreError),
#[error("cycle recovery marker publication is blocked by data movement")]
PublicationBlocked,
#[error("invalid cycle recovery marker: {0}")]
Invalid(&'static str),
#[error("cycle recovery marker revision changed while publishing")]
@@ -326,12 +328,13 @@ fn cycle_state_generation_and_epoch(buf: &[u8]) -> (u64, u64) {
}
async fn persist_cycle_recovery_marker(
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
primary_revision: &DataUsageCacheRevision,
generation: u64,
leader_epoch: u64,
classification: &'static str,
reason: &'static str,
expected_epoch: u64,
) -> Result<ScannerCycleRecoveryMarker, CycleRecoveryMarkerReadError> {
let now = unix_now_secs();
let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
@@ -370,6 +373,9 @@ async fn persist_cycle_recovery_marker(
state: "blocked".to_string(),
};
let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?;
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(CycleRecoveryMarkerReadError::PublicationBlocked);
};
let save_result = save_config_with_preconditions(
storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -478,7 +484,7 @@ async fn read_cycle_recovery_marker_revision(
}
async fn quarantine_invalid_cycle_state(
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
revision: &DataUsageCacheRevision,
buf: &[u8],
) -> ScannerCycleStateStartup {
@@ -488,7 +494,7 @@ async fn quarantine_invalid_cycle_state(
}
async fn quarantine_invalid_cycle_state_with_reason(
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
revision: &DataUsageCacheRevision,
generation: u64,
leader_epoch: u64,
@@ -515,7 +521,19 @@ async fn quarantine_invalid_cycle_state_with_reason(
reason: Some(reason.to_string()),
};
set_scanner_cycle_recovery_status(base_status);
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason).await {
let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
};
match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason, expected_epoch)
.await
{
Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")),
Err(CycleRecoveryMarkerReadError::Backend(_)) => {
// Keep the poison object untouched and retry marker creation with the
@@ -524,6 +542,16 @@ async fn quarantine_invalid_cycle_state_with_reason(
"failed to persist scanner cycle recovery marker".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::Conflict) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
@@ -546,16 +574,18 @@ async fn mark_cycle_recovery_cleanup_pending(
storeapi: Arc<ECStore>,
mut marker: ScannerCycleRecoveryMarker,
marker_revision: &DataUsageCacheRevision,
expected_epoch: u64,
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
marker.state = "cleanup-pending".to_string();
marker.last_attempt_at_unix_secs = unix_now_secs();
let bytes = serde_json::to_vec(&marker)
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
let info = save_config_with_preconditions(
let info = save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
bytes,
marker_revision.preconditions(),
expected_epoch,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
@@ -567,7 +597,9 @@ async fn mark_cycle_recovery_cleanup_pending(
Ok((marker, revision))
}
pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl ScannerObjectIO>) -> ScannerCycleStateStartup {
pub(crate) async fn load_scanner_cycle_state_for_startup(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> ScannerCycleStateStartup {
let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
Ok((None, _)) => None,
Ok((Some(data), marker_revision)) => match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&data) {
@@ -594,6 +626,16 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc<impl Scan
"failed to read scanner cycle recovery marker: {err}"
)));
}
Err(CycleRecoveryMarkerReadError::PublicationBlocked) => {
set_scanner_cycle_recovery_status(recovery_status(
"transient",
Some("cycle recovery marker publication is blocked by data movement"),
true,
));
return ScannerCycleStateStartup::Transient(ScannerError::Other(
"cycle recovery marker publication is blocked by data movement".to_string(),
));
}
Err(CycleRecoveryMarkerReadError::Invalid(reason)) => {
set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false));
return ScannerCycleStateStartup::Blocked;
@@ -750,6 +792,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
}
let Some(reset_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other("scanner recovery reset is blocked by data movement".to_string()));
};
let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await {
Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false),
Err(CycleRecoveryMarkerReadError::Invalid(_)) => {
@@ -835,7 +881,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
};
if let Some((primary_cycle, primary_epoch)) = primary_state {
let (cleanup_marker, cleanup_marker_revision) =
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision).await?;
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
let fence_epoch = primary_epoch
@@ -855,14 +901,21 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"preserved scanner cycle state exceeds the bounded object size".to_string(),
));
}
let preserved_info = save_config_with_preconditions(
let preserved_info = save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
preserved_data,
primary_revision.preconditions(),
reset_epoch,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")))?;
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}"))
}
})?;
let preserved_revision = preserved_info
.etag
.filter(|etag| !etag.is_empty())
@@ -873,7 +926,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after fencing newer cycle state".to_string(),
));
}
fence_scanner_usage_epoch(&ctx, storeapi.clone(), fence_epoch)
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch))
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
if guard.is_lock_lost() {
@@ -889,20 +942,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner cycle state changed before recovery marker cleanup".to_string(),
));
}
storeapi
.delete_config_object(
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(cleanup_marker_revision.preconditions()),
..Default::default()
},
)
.await
.map_err(|err| ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")))?;
delete_config_with_publication_admission_for_epoch(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(cleanup_marker_revision.preconditions()),
..Default::default()
},
reset_epoch,
)
.await
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}"))
}
})?;
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
super::notify_scanner_cycle_recovery_wake();
return Ok(());
@@ -946,16 +1006,23 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
let (marker, marker_revision) = if marker.state == "cleanup-pending" {
(marker, marker_revision)
} else {
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision).await?
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
};
let rebuilt_info = save_config_with_preconditions(
let rebuilt_info = save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
data,
primary_revision.preconditions(),
reset_epoch,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")))?;
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}"))
}
})?;
let rebuilt_revision = rebuilt_info
.etag
.filter(|etag| !etag.is_empty())
@@ -965,7 +1032,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after rebuilding cycle state".to_string(),
));
}
if let Err(err) = fence_scanner_usage_epoch(&ctx, storeapi.clone(), leader_epoch).await {
if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch)).await
{
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
@@ -1034,20 +1102,40 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
));
}
if let Err(err) = storeapi
.delete_config_object(
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(marker_revision.preconditions()),
..Default::default()
},
)
.await
if let Err(err) = delete_config_with_publication_admission_for_epoch(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
ScannerObjectOptions {
// This is one exact metadata object. Prefix-delete mode
// bypasses HTTP preconditions in the ECStore path.
delete_prefix: false,
http_preconditions: Some(marker_revision.preconditions()),
..Default::default()
},
reset_epoch,
)
.await
{
if scanner_publication_epoch_changed(&err) {
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
state: "cleanup-pending".to_string(),
classification: Some(marker.classification.clone()),
primary_revision: Some(rebuilt_revision.clone()),
generation: Some(next),
leader_epoch: Some(leader_epoch),
retry_count: marker.retry_count,
max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES,
retryable: true,
reason: Some("movement epoch changed before recovery marker cleanup".to_string()),
..Default::default()
});
return Err(ScannerError::Other(
"scanner recovery reset deferred by a movement epoch change".to_string(),
));
}
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()),
@@ -1209,8 +1297,14 @@ pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(),
Ok(())
}
pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -> Result<PersistedUsageFloor, ScannerError> {
pub(super) async fn persisted_usage_floor(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
) -> Result<PersistedUsageFloor, ScannerError> {
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string()));
};
let mut floor = PersistedUsageFloor::default();
let mut found_any = false;
let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> {
floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default());
if let Some(completed_cycle) = usage.scanner_cycle {
@@ -1229,6 +1323,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}"))
})?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {primary_path} has no persisted baseline identity"
)));
}
let epoch = usage.scanner_epoch.unwrap_or_default();
update_floor(&mut floor, &usage, primary_path)?;
Some(epoch)
@@ -1247,6 +1346,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
let usage = serde_json::from_slice::<DataUsageInfo>(&data).map_err(|err| {
ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}"))
})?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage floor from {backup_path} has no persisted baseline identity"
)));
}
let backup_epoch = usage.scanner_epoch.unwrap_or_default();
// A backup write from an older leader may complete after the
// primary epoch has been fenced. It must not advance the startup
@@ -1263,9 +1367,21 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc<impl ScannerObjectIO>) -
}
}
if any_found {
found_any = true;
break;
}
}
if !found_any {
return Err(ScannerError::Other(
"persisted scanner usage floor has no authoritative baseline".to_string(),
));
}
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi, read_epoch).await else {
return Err(ScannerError::Other(
"scanner usage floor changed while its epoch proof was being confirmed".to_string(),
));
};
Ok(floor)
}
@@ -1274,12 +1390,30 @@ pub(super) fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_
*leader_epoch = (*leader_epoch).max(floor.leader_epoch);
}
#[derive(Clone, Copy)]
pub(super) struct ScannerCycleFloorOptions {
pub(super) required_cycle: u64,
pub(super) expected_publication_epoch: Option<u64>,
}
#[cfg(test)]
pub(super) async fn persist_scanner_cycle_state(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
) -> bool {
persist_scanner_cycle_state_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, None).await
}
pub(super) async fn persist_scanner_cycle_state_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> bool {
let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) {
Ok(buf) => buf,
@@ -1315,9 +1449,29 @@ pub(super) async fn persist_scanner_cycle_state(
#[cfg(test)]
notify_scanner_cycle_state_persist_test_hook(leader_epoch);
match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
.await
{
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return false;
};
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
return false;
}
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
state = "publication_admission_unavailable",
"Scanner state persistence skipped without movement admission"
);
return false;
};
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
.await
};
match save_result {
Ok(object_info) => {
let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else {
error!(
@@ -1345,6 +1499,13 @@ pub(super) async fn persist_scanner_cycle_state(
);
return false;
}
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
@@ -1436,6 +1597,13 @@ pub(super) async fn persist_scanner_cycle_state(
}
if persisted_cycle.next >= cycle_info.next {
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
*cycle_info = persisted_cycle;
debug!(
target: "rustfs::scanner",
@@ -1496,19 +1664,33 @@ pub(super) async fn persist_scanner_cycle_state(
false
}
#[cfg(test)]
pub(super) async fn finalize_partial_scan_cycle(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool {
finalize_partial_scan_cycle_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, cycle_metrics_guard, None).await
}
pub(super) async fn finalize_partial_scan_cycle_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
expected_publication_epoch: Option<u64>,
) -> bool {
// A budget-limited cycle is deliberate pacing, not a failure. The cycle counter
// must still advance (and persist) because per-bucket next_cycle is stamped from
// it and compacted folders are only rescanned when their hash matches
// next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle
// expiry and usage refresh on every folder outside the stuck window.
let previous_cycle_info = cycle_info.clone();
if let Err(err) = advance_scanner_cycle(cycle_info) {
error!(
target: "rustfs::scanner",
@@ -1524,28 +1706,69 @@ pub(super) async fn finalize_partial_scan_cycle(
}
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
let persisted = persist_scanner_cycle_state_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
revision,
leader_epoch,
expected_publication_epoch,
)
.await;
if !persisted
&& let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
}
cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted
}
#[cfg(test)]
pub(super) async fn persist_required_scanner_cycle_floor(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
required_cycle: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool {
if required_cycle <= cycle_info.current || required_cycle == u64::MAX {
persist_required_scanner_cycle_floor_for_epoch(
ctx,
storeapi,
cycle_info,
revision,
leader_epoch,
cycle_metrics_guard,
ScannerCycleFloorOptions {
required_cycle,
expected_publication_epoch: None,
},
)
.await
}
pub(super) async fn persist_required_scanner_cycle_floor_for_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
options: ScannerCycleFloorOptions,
) -> bool {
if options.required_cycle <= cycle_info.current || options.required_cycle == u64::MAX {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
current_cycle = cycle_info.current,
required_cycle,
required_cycle = options.required_cycle,
state = "invalid_cache_cycle_floor",
"Scanner cache cycle floor is invalid"
);
@@ -1553,10 +1776,27 @@ pub(super) async fn persist_required_scanner_cycle_floor(
return false;
}
cycle_info.next = cycle_info.next.max(required_cycle);
let previous_cycle_info = cycle_info.clone();
cycle_info.next = cycle_info.next.max(options.required_cycle);
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
let persisted = persist_scanner_cycle_state_for_epoch(
ctx,
storeapi.clone(),
cycle_info,
revision,
leader_epoch,
options.expected_publication_epoch,
)
.await;
if !persisted
&& let Some(expected_epoch) = options.expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi, expected_epoch)
.await
.is_none()
{
*cycle_info = previous_cycle_info;
}
cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted
}
crates/scanner/src/scanner/heal_info.rs
+107 -18
View File
@@ -26,31 +26,90 @@ pub struct BackgroundHealInfo {
pub current_scan_mode: HealScanMode,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum BackgroundHealInfoReadStatus {
ErasureSd,
Loaded,
Missing,
Blocked,
Transient,
Failed,
}
pub(super) struct BackgroundHealInfoRead {
pub(super) info: BackgroundHealInfo,
pub(super) expected_epoch: Option<u64>,
pub(super) status: BackgroundHealInfoReadStatus,
}
pub(super) fn classify_background_heal_read_error(error: &EcstoreError) -> BackgroundHealInfoReadStatus {
if matches!(error, EcstoreError::ConfigNotFound) {
BackgroundHealInfoReadStatus::Missing
} else {
BackgroundHealInfoReadStatus::Transient
}
}
pub(super) fn decode_background_heal_info(data: &[u8]) -> Result<BackgroundHealInfo, serde_json::Error> {
serde_json::from_slice(data)
}
/// Read background healing information from storage
pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHealInfo {
read_background_heal_info_with_epoch(storeapi).await.info
}
/// Read background healing information together with the movement epoch that
/// fenced the read. The epoch must be reused by the matching cycle update so a
/// missing-object default cannot be committed across a movement transition.
pub(super) async fn read_background_heal_info_with_epoch(storeapi: Arc<ECStore>) -> BackgroundHealInfoRead {
// Skip for ErasureSD setup
if scanner_is_erasure_sd().await {
return BackgroundHealInfo::default();
return BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch: None,
status: BackgroundHealInfoReadStatus::ErasureSd,
};
}
let expected_epoch = scanner_publication_epoch(storeapi.clone()).await;
if expected_epoch.is_none() {
return BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status: BackgroundHealInfoReadStatus::Blocked,
};
}
// Get last healing information
match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await {
Ok(buf) => serde_json::from_slice::<BackgroundHealInfo>(&buf).unwrap_or_else(|e| {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal decode failed"
);
BackgroundHealInfo::default()
}),
Ok(buf) => match decode_background_heal_info(&buf) {
Ok(info) => BackgroundHealInfoRead {
info,
expected_epoch,
status: BackgroundHealInfoReadStatus::Loaded,
},
Err(e) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "decode_failed",
error = %e,
"Scanner background heal decode failed"
);
BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status: BackgroundHealInfoReadStatus::Failed,
}
}
},
Err(e) => {
// Only log if it's not a ConfigNotFound error
if e != EcstoreError::ConfigNotFound {
let status = classify_background_heal_read_error(&e);
if status == BackgroundHealInfoReadStatus::Transient {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
@@ -62,7 +121,11 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
"Scanner background heal read failed"
);
}
BackgroundHealInfo::default()
BackgroundHealInfoRead {
info: BackgroundHealInfo::default(),
expected_epoch,
status,
}
}
}
}
@@ -70,6 +133,14 @@ pub async fn read_background_heal_info(storeapi: Arc<ECStore>) -> BackgroundHeal
/// Save background healing information to storage
#[instrument(skip(storeapi))]
pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundHealInfo) {
save_background_heal_info_for_epoch(storeapi, info, None).await;
}
pub(super) async fn save_background_heal_info_for_epoch(
storeapi: Arc<ECStore>,
info: BackgroundHealInfo,
expected_epoch: Option<u64>,
) {
// Skip for ErasureSD setup
if scanner_is_erasure_sd().await {
return;
@@ -93,7 +164,25 @@ pub async fn save_background_heal_info(storeapi: Arc<ECStore>, info: BackgroundH
}
};
// Save configuration
// Save configuration only after storage-owned movement admission. The
// read path may return an in-memory default for a missing object, but a
// movement transition must not let that default become durable state.
let publication_admission = match expected_epoch {
Some(expected_epoch) => scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await,
None => storeapi.scanner_data_usage_publication_admission().await,
};
let Some(_publication_admission) = publication_admission else {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_BACKGROUND_HEAL_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL,
path = %&*BACKGROUND_HEAL_INFO_PATH,
state = "publication_admission_unavailable",
"Scanner background heal save skipped without movement admission"
);
return;
};
if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await {
warn!(
target: "rustfs::scanner",
crates/scanner/src/scanner/leadership.rs
+119 -20
View File
@@ -61,16 +61,22 @@ pub(super) async fn reconcile_scanner_leadership_claim(
}
pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result<DataUsageInfo, ScannerError> {
serde_json::from_slice(data)
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))
let usage: DataUsageInfo = serde_json::from_slice(data)
.map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?;
if !data_usage_info_has_persisted_baseline_identity(&usage) {
return Err(ScannerError::Other(format!(
"scanner usage epoch fence from {path} has no persisted baseline identity"
)));
}
Ok(usage)
}
pub(super) async fn usage_snapshot_for_epoch_fence(
storeapi: Arc<impl ScannerObjectIO>,
primary: Option<&[u8]>,
) -> Result<DataUsageInfo, ScannerError> {
) -> Result<Option<DataUsageInfo>, ScannerError> {
if let Some(primary) = primary {
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str());
return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()).map(Some);
}
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
@@ -78,7 +84,7 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?;
if let Some(backup) = backup.as_deref() {
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path);
return decode_usage_snapshot_for_epoch_fence(backup, &backup_path).map(Some);
}
for path in [
@@ -89,26 +95,53 @@ pub(super) async fn usage_snapshot_for_epoch_fence(
.await
.map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?;
if let Some(legacy) = legacy.as_deref() {
return decode_usage_snapshot_for_epoch_fence(legacy, &path);
return decode_usage_snapshot_for_epoch_fence(legacy, &path).map(Some);
}
}
Ok(DataUsageInfo::default())
// A missing usage snapshot is an uninitialized state, not an empty
// snapshot. Leadership fencing may proceed without creating a plausible
// default; the first authoritative scanner publication will create it.
Ok(None)
}
pub(super) async fn fence_scanner_usage_epoch(
pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> Result<(), ScannerError> {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() {
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
}
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return Err(ScannerError::Other(
"scanner usage epoch fence publication is blocked by data movement".to_string(),
));
};
if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while recovery reset was in progress".to_string(),
));
}
let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?;
let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?;
let Some(mut usage) = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await? else {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while confirming a missing usage baseline".to_string(),
));
};
return Err(ScannerError::Other("authoritative scanner usage baseline is missing".to_string()));
};
match usage.scanner_epoch {
Some(epoch) if epoch > claimed_epoch => {
return Err(ScannerError::Other(format!(
@@ -122,9 +155,18 @@ pub(super) async fn fence_scanner_usage_epoch(
let data = serde_json::to_vec(&usage)
.map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?;
let save_result =
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return Err(ScannerError::Other(
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
));
};
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
.await;
.await
};
if save_result
.as_ref()
.ok()
@@ -165,10 +207,13 @@ pub(super) async fn fence_scanner_usage_epoch(
pub(super) async fn complete_scanner_leadership_claim(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
) -> bool {
if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await {
if let Err(err) =
fence_scanner_usage_epoch_with_expected_epoch(ctx, storeapi, claimed_epoch, expected_publication_epoch).await
{
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
@@ -187,7 +232,7 @@ pub(super) async fn complete_scanner_leadership_claim(
pub(super) async fn claim_scanner_leadership(
ctx: &CancellationToken,
storeapi: Arc<impl ScannerObjectIO>,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
persisted_epoch: &mut u64,
@@ -226,15 +271,69 @@ pub(super) async fn claim_scanner_leadership(
};
let previous_revision = revision.clone();
let save_result =
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
return false;
};
let (usage_primary, _) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
Ok(result) => result,
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_read_failed",
error = %err,
"Scanner leadership claim deferred because the usage baseline could not be read"
);
return false;
}
};
match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref()).await {
Ok(Some(_)) => {}
Ok(None) => {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_missing",
"Scanner leadership claim deferred until a usage baseline is published"
);
return false;
}
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "leader_usage_baseline_invalid",
error = %err,
"Scanner leadership claim deferred because the usage baseline is invalid"
);
return false;
}
}
let save_result = {
let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else {
if retry < SCANNER_PERSIST_CAS_RETRIES {
continue;
}
return false;
};
save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions())
.await;
.await
};
match save_result {
Ok(object_info) => {
if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) {
*revision = DataUsageCacheRevision::Etag(etag);
*persisted_epoch = claimed_epoch;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
}
match reconcile_scanner_leadership_claim(
@@ -249,7 +348,7 @@ pub(super) async fn claim_scanner_leadership(
.await
{
Ok(ScannerLeadershipClaimReconcile::Durable) => {
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
}
Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue,
Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => {
@@ -293,7 +392,7 @@ pub(super) async fn claim_scanner_leadership(
.await
{
Ok(ScannerLeadershipClaimReconcile::Durable) => {
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await;
return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await;
}
Ok(ScannerLeadershipClaimReconcile::Changed)
if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() =>
crates/scanner/src/scanner/tests.rs
+185 -28
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info};
use super::*;
use crate::EcstoreResult;
use crate::{
@@ -22,6 +23,7 @@ use crate::{
};
use std::collections::{HashMap, HashSet};
use std::io::Cursor;
use std::sync::atomic::{AtomicBool, Ordering};
use std::task::Poll;
use temp_env::{with_var, with_var_unset};
use tokio::io::AsyncReadExt;
@@ -343,6 +345,7 @@ struct MemoryConfigStore {
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
put_counts: Mutex<HashMap<String, usize>>,
publication_admission_blocked: AtomicBool,
}
fn memory_config_key(bucket: &str, object: &str) -> String {
@@ -1350,7 +1353,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() {
let old_usage = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(41),
..Default::default()
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
};
let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone())
@@ -1424,7 +1427,7 @@ async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() {
let usage = DataUsageInfo {
scanner_epoch: Some(7),
scanner_cycle: Some(41),
..Default::default()
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
};
save_config(
store.clone(),
@@ -1695,7 +1698,7 @@ async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() {
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(u64::MAX - 1),
..Default::default()
..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
})
.expect("usage floor should encode"),
)
@@ -1847,14 +1850,12 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(epoch);
usage.scanner_cycle = Some(cycle);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(epoch),
scanner_cycle: Some(cycle),
..Default::default()
})
.expect("usage snapshot should encode"),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
@@ -1879,14 +1880,12 @@ async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() {
let store = Arc::new(MemoryConfigStore::default());
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(epoch);
usage.scanner_cycle = Some(cycle);
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, path),
serde_json::to_vec(&DataUsageInfo {
scanner_epoch: Some(epoch),
scanner_cycle: Some(cycle),
..Default::default()
})
.expect("usage snapshot should encode"),
serde_json::to_vec(&usage).expect("usage snapshot should encode"),
);
}
@@ -1916,6 +1915,23 @@ fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() {
}));
}
#[test]
fn scanner_baseline_identity_requires_complete_or_strict_legacy_shape() {
assert!(!data_usage_info_has_persisted_baseline_identity(&DataUsageInfo {
scanner_epoch: Some(3),
scanner_cycle: Some(7),
..Default::default()
}));
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
legacy.usage_snapshot_complete = false;
legacy.scanner_cycle = Some(7);
assert!(data_usage_info_has_persisted_baseline_identity(&legacy));
legacy.scanner_epoch = Some(3);
assert!(!data_usage_info_has_persisted_baseline_identity(&legacy));
}
#[test]
fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
let authoritative = DataUsageInfo {
@@ -1948,12 +1964,9 @@ fn scanner_startup_prompts_only_for_a_newer_valid_observation() {
#[tokio::test]
async fn scanner_startup_prefers_v2_over_legacy_usage() {
let store = Arc::new(MemoryConfigStore::default());
let legacy = DataUsageInfo {
scanner_epoch: Some(19),
scanner_cycle: Some(41),
last_update: Some(std::time::SystemTime::now()),
..Default::default()
};
let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
legacy.scanner_epoch = Some(19);
legacy.scanner_cycle = Some(41);
let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode");
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
@@ -1976,13 +1989,9 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
}
);
let authoritative = DataUsageInfo {
scanner_epoch: Some(23),
scanner_cycle: Some(51),
last_update: Some(std::time::SystemTime::now()),
usage_snapshot_complete: true,
..Default::default()
};
let mut authoritative = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
authoritative.scanner_epoch = Some(23);
authoritative.scanner_cycle = Some(51);
let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode");
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
@@ -2024,6 +2033,8 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() {
#[tokio::test]
async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() {
let store = Arc::new(MemoryConfigStore::default());
assert!(persisted_usage_floor(store.clone()).await.is_err());
store.objects.lock().await.insert(
memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()),
b"not-json".to_vec(),
@@ -2087,6 +2098,39 @@ async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() {
}
}
#[tokio::test]
async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_primary() {
let store = Arc::new(MemoryConfigStore::default());
let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
store.objects.lock().await.insert(
primary_key.clone(),
serde_json::to_vec(&primary).expect("primary usage snapshot should encode"),
);
store.revisions.lock().await.insert(primary_key.clone(), 1);
store.publication_admission_blocked.store(true, Ordering::Release);
let movement_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone())
.await
.expect_err("movement admission loss should fail backup synchronization");
assert!(scanner_publication_epoch_changed(&movement_error));
store.publication_admission_blocked.store(false, Ordering::Release);
store.objects.lock().await.remove(&primary_key);
store.revisions.lock().await.remove(&primary_key);
assert!(matches!(
sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()).await,
Err(EcstoreError::ConfigNotFound)
));
store.objects.lock().await.insert(primary_key.clone(), b"not-json".to_vec());
store.revisions.lock().await.insert(primary_key, 1);
let corrupt_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store)
.await
.expect_err("corrupt primary should fail backup synchronization");
assert!(!scanner_publication_epoch_changed(&corrupt_error));
}
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
@@ -2110,6 +2154,10 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
revisions.remove(&key);
Ok(ObjectInfo::default())
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
(!self.publication_admission_blocked.load(Ordering::Acquire)).then(crate::ScannerDataUsagePublicationAdmission::unfenced)
}
}
#[test]
@@ -2270,6 +2318,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf
started: Utc::now(),
};
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
seed_usage_snapshot_for_leadership_claim(&store).await;
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
let old_epoch_commit = CurrentCycle {
@@ -2313,6 +2362,61 @@ async fn test_leadership_claim_rejects_terminal_epoch() {
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_without_usage_baseline_before_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
store.objects.lock().await.insert(usage_key.clone(), b"not-json".to_vec());
store.revisions.lock().await.insert(usage_key, 1);
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test]
async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_write() {
let store = Arc::new(MemoryConfigStore::default());
let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let data = serde_json::to_vec(&DataUsageInfo::default()).expect("default usage should encode");
store.objects.lock().await.insert(usage_key.clone(), data);
store.revisions.lock().await.insert(usage_key, 1);
let ctx = CancellationToken::new();
let mut revision = DataUsageCacheRevision::Missing;
let mut cycle = CurrentCycle {
next: 12,
..Default::default()
};
let mut persisted_epoch = 0;
assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await);
assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err());
}
#[tokio::test]
async fn test_leadership_claim_confirms_commit_after_returned_error() {
let store = Arc::new(MemoryConfigStore::default());
@@ -2329,6 +2433,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() {
started: Utc::now(),
};
let mut persisted_epoch = 0;
seed_usage_snapshot_for_leadership_claim(&store).await;
assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await);
@@ -2373,6 +2478,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() {
);
old_usage.buckets_count = 1;
old_usage.calculate_totals();
old_usage.usage_snapshot_complete = true;
let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode");
store.objects.lock().await.insert(usage_key.clone(), old_data.clone());
store.revisions.lock().await.insert(usage_key, 1);
@@ -2419,6 +2525,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() {
started: Utc::now(),
};
assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await);
seed_usage_snapshot_for_leadership_claim(&store).await;
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str());
store
@@ -2590,6 +2697,35 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() {
}
}
#[tokio::test]
async fn test_observational_usage_defers_when_authoritative_baseline_is_missing() {
let store = Arc::new(MemoryConfigStore::default());
let (sender, receiver) = mpsc::channel(1);
let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1);
observation.usage_snapshot_converged = Some(false);
sender.send(observation).await.expect("observation should enqueue");
drop(sender);
let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe(
CancellationToken::new(),
store.clone(),
receiver,
None,
None,
|| async { false },
)
.await;
assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement));
assert!(
!store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()))
);
}
#[tokio::test]
async fn test_usage_route_barrier_precedes_durable_reconciliation() {
let store = Arc::new(MemoryConfigStore::default());
@@ -3425,6 +3561,14 @@ fn complete_usage_with_bucket_count(last_update: Option<std::time::SystemTime>,
info
}
async fn seed_usage_snapshot_for_leadership_claim(store: &Arc<MemoryConfigStore>) {
let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str());
let data = serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0))
.expect("leadership usage baseline should encode");
store.objects.lock().await.insert(key.clone(), data);
store.revisions.lock().await.insert(key, 1);
}
fn usage_with_last_update(last_update: Option<std::time::SystemTime>) -> DataUsageInfo {
complete_usage_with_bucket_count(last_update, 0)
}
@@ -5129,6 +5273,19 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() {
assert_eq!(info.bitrot_start_time, Some(now));
}
#[test]
fn background_heal_read_failures_never_become_initializable_defaults() {
assert_eq!(
classify_background_heal_read_error(&EcstoreError::ConfigNotFound),
BackgroundHealInfoReadStatus::Missing
);
assert_eq!(
classify_background_heal_read_error(&EcstoreError::SlowDown),
BackgroundHealInfoReadStatus::Transient
);
assert!(decode_background_heal_info(b"not-json").is_err());
}
#[test]
fn test_background_heal_info_for_scan_start_keeps_deep_window_start() {
with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || {
crates/scanner/src/scanner/usage_store.rs
+221 -50
View File
@@ -112,11 +112,39 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe<F, Fut>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
route_probe: F,
) -> DataUsagePersistOutcome
where
F: Fn() -> Fut + Send + Sync,
Fut: Future<Output = bool> + Send,
{
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch(
ctx,
storeapi,
receiver,
leader_epoch,
initial_baseline,
None,
route_probe,
)
.await
}
pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch<
F,
Fut,
>(
ctx: CancellationToken,
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
mut receiver: mpsc::Receiver<DataUsageInfo>,
leader_epoch: Option<u64>,
initial_baseline: Option<DataUsagePersistBaseline>,
expected_publication_epoch: Option<u64>,
route_probe: F,
) -> DataUsagePersistOutcome
where
@@ -134,6 +162,14 @@ where
if let Some(leader_epoch) = leader_epoch {
data_usage_info.scanner_epoch = Some(leader_epoch);
}
if let Some(expected_epoch) = expected_publication_epoch
&& scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
let observational = data_usage_info.usage_snapshot_converged == Some(false);
let target_path = if observational {
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()
@@ -154,7 +190,28 @@ where
break;
}
let mut publication_epoch = expected_publication_epoch;
if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() {
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
expected_epoch
}
None => {
let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
};
read_epoch
}
};
publication_epoch = Some(read_epoch);
let authoritative_data = match next_baseline.as_ref() {
Some(baseline) => baseline.data.clone(),
None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await {
@@ -175,25 +232,48 @@ where
}
},
};
let authoritative = match authoritative_data.as_deref() {
Some(data) => match serde_json::from_slice::<DataUsageInfo>(data) {
Ok(info) => info,
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_decode_failed",
error = %err,
"Scanner refused to publish an observation from an invalid authoritative baseline"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
},
None => DataUsageInfo::default(),
let Some(authoritative_data) = authoritative_data else {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_missing",
"Scanner deferred observational publication until an authoritative usage baseline exists"
);
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
};
let authoritative = match serde_json::from_slice::<DataUsageInfo>(&authoritative_data) {
Ok(info) if data_usage_info_has_persisted_baseline_identity(&info) => info,
Ok(_) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_identity_missing",
"Scanner refused to publish an observation without authoritative baseline identity"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
Err(err) => {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
path = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
state = "observed_baseline_decode_failed",
error = %err,
"Scanner refused to publish an observation from an invalid authoritative baseline"
);
outcome = DataUsagePersistOutcome::Failed;
continue;
}
};
data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
}
@@ -240,6 +320,26 @@ where
break 'updates;
}
let publication_epoch_for_save = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
}
expected_epoch
}
None => match publication_epoch.take() {
Some(epoch) => epoch,
None => {
let Some(epoch) = scanner_publication_epoch(storeapi.clone()).await else {
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
epoch
}
},
};
let baseline = if !observational && cas_retry == 0 {
next_baseline.take()
} else {
@@ -329,14 +429,22 @@ where
}
let done_save = Metrics::time(Metric::SaveUsage);
let save_result = save_config_shared_with_preconditions(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
)
.await;
let save_result = {
let Some(_publication_admission) =
scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await
else {
done_save();
break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
};
save_config_shared_with_preconditions(
storeapi.clone(),
target_path,
data.clone(),
sha256hex.clone(),
revision.preconditions(),
)
.await
};
done_save();
match save_result {
@@ -427,7 +535,16 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
@@ -438,7 +555,16 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await;
}
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
@@ -460,7 +586,16 @@ where
if observational {
invalidate_admin_data_usage_snapshot_cache().await;
} else {
cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await;
let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch(
storeapi.clone(),
&data_usage_info,
expected_publication_epoch,
)
.await;
if expected_publication_epoch.is_some() && !cleanup_ok {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
invalidate_data_usage_snapshot_cache().await;
replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
@@ -471,7 +606,10 @@ where
if backup_due {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await {
let backup_result =
sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await;
done_save();
if let Err(e) = backup_result {
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
@@ -482,22 +620,50 @@ where
error = %e,
"Scanner data usage backup save failed"
);
if scanner_publication_epoch_changed(&e) {
outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement);
break 'updates;
}
outcome = DataUsagePersistOutcome::Failed;
break 'updates;
}
done_save();
}
}
outcome
}
pub(super) async fn cleanup_observed_data_usage_snapshot(
pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
authoritative: &DataUsageInfo,
) {
expected_publication_epoch: Option<u64>,
) -> bool {
let read_epoch = match expected_publication_epoch {
Some(expected_epoch) => {
if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch)
.await
.is_none()
{
return false;
}
expected_epoch
}
None => match scanner_publication_epoch(storeapi.clone()).await {
Some(read_epoch) => read_epoch,
None => return false,
},
};
if expected_publication_epoch.is_some()
&& scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch)
.await
.is_none()
{
return false;
}
let (observed_data, revision) =
match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await {
Ok((Some(data), revision)) => (data, revision),
Ok((None, _)) => return,
Ok((None, _)) => return true,
Err(err) => {
error!(
target: "rustfs::scanner",
@@ -509,7 +675,7 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err,
"Scanner could not inspect observational data usage snapshot before authoritative cleanup"
);
return;
return true;
}
};
let observed = match serde_json::from_slice::<DataUsageInfo>(&observed_data) {
@@ -525,25 +691,26 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err,
"Scanner refused to remove an invalid observational data usage snapshot after authoritative save"
);
return;
return true;
}
};
if observed_data_usage_is_newer(&observed, authoritative) {
return;
return true;
}
let result = storeapi
.delete_config_object(
RUSTFS_META_BUCKET,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
ScannerObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()),
..Default::default()
},
)
.await;
let result = delete_config_with_publication_admission_for_epoch(
storeapi,
RUSTFS_META_BUCKET,
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
ScannerObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
http_preconditions: Some(revision.preconditions()),
..Default::default()
},
read_epoch,
)
.await;
match result {
Ok(_)
@@ -564,6 +731,10 @@ pub(super) async fn cleanup_observed_data_usage_snapshot(
error = %err,
"Scanner could not remove stale observational data usage snapshot after authoritative save"
);
if scanner_publication_epoch_changed(&err) {
return false;
}
}
}
true
}
crates/scanner/src/scanner_io.rs
+19 -3
View File
@@ -56,9 +56,10 @@ use crate::storage_api::scan::NamespaceLocking as _;
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
use crate::{
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version,
get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, storageclass,
RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerConfigObjectDelete as _, ScannerDiskExt as _,
ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError,
enqueue_runtime_free_version, get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names,
scanner_publication_admission_for_epoch, scanner_publication_epoch, storageclass,
};
pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file";
@@ -143,6 +144,10 @@ pub struct ScannerBucketScanPlan {
all_buckets: Arc<Vec<BucketInfo>>,
digest: DataUsageScanPlanDigest,
leader_epoch: u64,
/// Epoch captured once for the whole scanner cycle. `None` is retained
/// for unfenced test implementations; production plans always carry the
/// admission token captured before bucket enumeration.
publication_epoch: Option<u64>,
dirty_usage_buckets: Arc<DirtyUsageBuckets>,
bucket_failures: ScannerBucketFailureState,
pending_maintenance_work: Arc<AtomicBool>,
@@ -578,6 +583,7 @@ fn scanner_activity_preflight(
#[derive(Debug)]
pub(crate) struct ScannerCycleResult {
pub(crate) status: ScannerCycleStatus,
publication_epoch: Option<u64>,
dirty_usage_clear: Option<DirtyUsageBuckets>,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
failed_dirty_usage: bool,
@@ -589,6 +595,7 @@ impl ScannerCycleResult {
pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option<DirtyUsageBuckets>) -> Self {
Self {
status,
publication_epoch: None,
dirty_usage_clear,
remote_dirty_usage_acknowledgements: Vec::new(),
failed_dirty_usage: false,
@@ -597,6 +604,15 @@ impl ScannerCycleResult {
}
}
pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option<u64>) -> Self {
self.publication_epoch = publication_epoch;
self
}
pub(crate) fn publication_epoch(&self) -> Option<u64> {
self.publication_epoch
}
fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self {
self.failed_dirty_usage = failed_dirty_usage;
self
crates/scanner/src/scanner_io/cache.rs
+20 -1
View File
@@ -395,6 +395,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache,
cache_cycle_floor: &AtomicU64,
expected_publication_epoch: u64,
) -> Option<SystemTime> {
let source = cache_snapshot.info.source?;
let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await {
@@ -489,7 +490,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot
.save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions)
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
.await
{
error!(
@@ -519,6 +520,24 @@ pub(super) async fn persist_and_publish_cache_snapshot(
);
return None;
}
// The persisted-root fast path performs no PUT, so it also needs the
// cycle token re-admission before forwarding the root to the aggregate.
// This final check covers both the fast path and a successful save.
if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch)
.await
.is_none()
{
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
cache_name = DATA_USAGE_CACHE_NAME,
state = "publication_epoch_changed_before_publish",
"Scanner cache root publish skipped after movement epoch change"
);
return None;
}
drop(guard);
let last_update = cache_snapshot.info.last_update;
crates/scanner/src/scanner_io/io_cache.rs
+79 -8
View File
@@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks {
all_buckets,
digest: scan_plan_digest,
leader_epoch,
publication_epoch,
dirty_usage_buckets,
bucket_failures,
pending_maintenance_work,
@@ -40,6 +41,12 @@ impl ScannerIOCache for SetDisks {
let set_label = self.set_index.to_string();
let source = DataUsageCacheSource::new(self.pool_index, self.set_index);
let expected_publication_epoch = match publication_epoch {
Some(epoch) => epoch,
None => scanner_publication_epoch(self.clone())
.await
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default();
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
warn!(
@@ -76,10 +83,16 @@ impl ScannerIOCache for SetDisks {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref())
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
return persist_and_publish_cache_snapshot(
self,
&updates,
cache,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
@@ -414,6 +427,7 @@ impl ScannerIOCache for SetDisks {
let pending_maintenance_work_clone = pending_maintenance_work.clone();
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
let cache_cycle_floor_clone = cache_cycle_floor.clone();
let expected_publication_epoch_clone = expected_publication_epoch;
let remote_server_epoch = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::Coordinator => None,
@@ -753,6 +767,23 @@ impl ScannerIOCache for SetDisks {
);
continue;
}
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_before_reuse",
"Current scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
@@ -901,7 +932,12 @@ impl ScannerIOCache for SetDisks {
{
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await
{
error!(
@@ -958,7 +994,12 @@ impl ScannerIOCache for SetDisks {
false
} else {
match partial_cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await
{
Ok(()) => true,
@@ -1029,7 +1070,12 @@ impl ScannerIOCache for SetDisks {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
&cache_name,
&revisions,
expected_publication_epoch_clone,
)
.await
{
done_save();
@@ -1064,6 +1110,24 @@ impl ScannerIOCache for SetDisks {
continue;
}
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch_clone)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_after_save",
"Scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
@@ -1159,7 +1223,14 @@ impl ScannerIOCache for SetDisks {
cache.info.lkg_scan_plan_digest = None;
cache.clone()
};
let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await;
let _ = persist_and_publish_cache_snapshot(
self.clone(),
&updates,
cache_snapshot,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await;
} else {
let mut incomplete_scope = cache_mutex.lock().await.clone();
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();
crates/scanner/src/scanner_io/io_cycle.rs
+21 -3
View File
@@ -68,6 +68,19 @@ impl ScannerIOCycle for ECStore {
));
}
// Capture one storage-owned movement epoch for the entire cycle. Set
// workers must not each observe a fresh epoch: a movement transition
// between sets would otherwise allow a mixed-generation aggregate.
let publication_epoch = match self.scanner_data_usage_publication_admission().await {
Some(admission) => Some(admission.epoch()),
None => {
return Ok(ScannerCycleResult::new(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
None,
));
}
};
let distributed = self.setup_is_dist_erasure().await;
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
ScannerActivityPreflight::Ready(snapshot) => snapshot,
@@ -131,7 +144,9 @@ impl ScannerIOCycle for ECStore {
if all_buckets.is_empty() {
reset_set_scan_gauges();
if !bucket_plan_complete {
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
return Ok(
ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)
);
}
let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await;
let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot);
@@ -155,7 +170,7 @@ impl ScannerIOCycle for ECStore {
)
.await?
{
return Ok(ScannerCycleResult::new(status, None));
return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
}
let dirty_usage_clear =
(status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
@@ -165,6 +180,7 @@ impl ScannerIOCycle for ECStore {
Vec::new()
};
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
}
@@ -180,7 +196,7 @@ impl ScannerIOCycle for ECStore {
"Scanner set state update detected missing disk sets"
);
reset_set_scan_gauges();
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
}
let set_scan_limit = scanner_budgeted_concurrency_limit(
@@ -250,6 +266,7 @@ impl ScannerIOCycle for ECStore {
all_buckets: Arc::clone(&all_buckets),
digest: scan_plan_digest,
leader_epoch,
publication_epoch,
dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(),
bucket_failures: bucket_failures.clone(),
pending_maintenance_work: pending_maintenance_work.clone(),
@@ -430,6 +447,7 @@ impl ScannerIOCycle for ECStore {
Vec::new()
};
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)
.with_failed_dirty_usage(!failed_buckets.is_empty())
.with_pending_maintenance_work(pending_maintenance_work)
crates/scanner/src/scanner_io/tests.rs
+44
View File
@@ -145,6 +145,50 @@ async fn scanner_cache_locks_allow_cross_source_workers() {
assert!(!second.is_lock_lost());
}
#[tokio::test]
async fn scanner_set_cache_admission_tracks_owner_snapshot_and_fails_closed() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let set = store.pools[0].disk_set[0].clone();
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_none(),
"a set must not publish before the owner has refreshed its movement snapshot"
);
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_some(),
"an idle owner snapshot should admit the set cache"
);
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
pool_stats[0] = EcstoreRebalanceStats {
participating: true,
info: EcstoreRebalanceInfo {
start_time: Some(OffsetDateTime::now_utc()),
status: EcstoreRebalStatus::Started,
..Default::default()
},
..Default::default()
};
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
id: Uuid::new_v4().to_string(),
pool_stats,
..Default::default()
});
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_none(),
"active movement must keep set cache publication blocked"
);
*store.rebalance_meta.write().await = None;
assert!(!store.scanner_data_usage_publication_blocked().await);
assert!(
set.scanner_data_usage_publication_admission_guard().await.is_some(),
"an idle owner refresh must make set cache publication live again"
);
}
#[tokio::test]
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;