diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 62ed02ffa..6ea3f15d1 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -489,9 +489,9 @@ pub mod storage { pub use crate::core::pools::HealLifecycleExpiryContext; pub use crate::store::HealWalkVersion; pub use crate::store::{ - ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, - init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map, - prewarm_local_disk_id_map_with_instance_ctx, + ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path, + find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, + prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx, }; } diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index ae470d017..c5d3bfe4c 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -952,6 +952,8 @@ fn record_decommission_unresolved_entry( idx: usize, generation: OffsetDateTime, entry: DecommissionUnresolvedEntry, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, ) -> Result { ensure_decommission_generation(meta, idx, generation)?; if entry.pool_index != idx || entry.source_generation != generation { @@ -959,34 +961,37 @@ fn record_decommission_unresolved_entry( } let pool_count = meta.pools.len(); - let Some(pool) = meta.pools.get_mut(idx) else { + let Some(pool) = meta.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; - let Some(info) = pool.decommission.as_mut() else { + let Some(info) = pool.decommission.as_ref() else { return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry")); }; - let existing = info.unresolved_entries.iter_mut().find(|existing| { + let existing_index = info.unresolved_entries.iter().position(|existing| { existing.bucket == entry.bucket && existing.object == entry.object && existing.pool_index == entry.pool_index && existing.set_index == entry.set_index && existing.source_generation == entry.source_generation }); - let changed = match existing { - Some(existing) if existing == &entry => false, - Some(existing) => { - *existing = entry; - true - } - None => { - info.unresolved_entries.push(entry); - true - } - }; - if changed { - pool.last_update = OffsetDateTime::now_utc(); + if existing_index.is_some_and(|index| info.unresolved_entries[index] == entry) { + return Ok(false); } - Ok(changed) + + let last_update = meta.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = meta.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_mut() else { + return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry")); + }; + if let Some(index) = existing_index { + info.unresolved_entries[index] = entry; + } else { + info.unresolved_entries.push(entry); + } + pool.last_update = last_update; + Ok(true) } type DecommissionUnresolvedEntryIdentity = (usize, String, String); @@ -1760,7 +1765,11 @@ fn merge_pool_meta_updates_for_save( if current_pool.id != idx || persisted_pool.id != idx || current_pool.cmd_line != persisted_pool.cmd_line { return Err(Error::other(format!("{operation}: pool metadata layout changed for pool {idx}"))); } - if current_pool.decommission.is_none() + let current_clears_decommission = current_pool + .decommission + .as_ref() + .is_none_or(|info| !info.has_decommission_state()); + if current_clears_decommission && persisted_pool.decommission.as_ref().is_some_and(|info| { info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled) }) @@ -1770,6 +1779,19 @@ fn merge_pool_meta_updates_for_save( "{operation}: stale pool metadata update rejected for pool {idx}; persisted active or queued decommission cannot be cleared" ))); } + if current_clears_decommission + && persisted_pool + .decommission + .as_ref() + .is_some_and(|info| info.complete || !info.unresolved_entries.is_empty()) + { + record_pool_meta_stale_write_rejection("unsafe_terminal_decommission_clear"); + return Err(Error::StalePoolMetadataUpdate { + operation: operation.to_string(), + pool_index: idx, + reason: "completed or unresolved decommission state cannot be cleared", + }); + } if current_pool.last_update < persisted_pool.last_update { record_pool_meta_stale_write_rejection("older_pool_revision"); return Err(Error::other(format!( @@ -1778,6 +1800,7 @@ fn merge_pool_meta_updates_for_save( } if let (Some(persisted_info), Some(current_info)) = (persisted_pool.decommission.as_ref(), current_pool.decommission.as_ref()) + && current_info.has_decommission_state() { let persisted_terminal = (persisted_info.complete, persisted_info.failed, persisted_info.canceled); let current_terminal = (current_info.complete, current_info.failed, current_info.canceled); @@ -4594,6 +4617,45 @@ impl PoolMetaSaveOutcome { } impl PoolMeta { + fn current_decommission_movement_update(&self) -> Option { + self.pools + .iter() + .filter(|pool| pool.decommission.is_some()) + .map(|pool| pool.last_update) + .max() + } + + fn current_rebalance_movement_update(rebalance_meta: Option<&RebalanceMeta>) -> Option { + rebalance_meta + .into_iter() + .flat_map(|meta| { + meta.stopped_at.into_iter().chain( + meta.pool_stats + .iter() + .flat_map(|pool| [pool.info.start_time, pool.info.end_time]) + .flatten(), + ) + }) + .max() + } + + pub(crate) fn next_scanner_data_movement_update( + &self, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> OffsetDateTime { + let max_seen = self + .current_decommission_movement_update() + .max(Self::current_rebalance_movement_update(rebalance_meta)); + match max_seen { + Some(max_seen) => max_seen + .checked_add(Duration::nanoseconds(1)) + .map(|next| now.max(next)) + .unwrap_or(max_seen), + None => now, + } + } + fn decode_pool_meta_payload(version: u16, payload: &[u8]) -> Result { match version { POOL_META_VERSION => rmp_serde::from_slice::(payload) @@ -4673,6 +4735,7 @@ impl PoolMeta { idx: usize, duration: Duration, now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, ) -> Result> { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; @@ -4698,7 +4761,7 @@ impl PoolMeta { start_time: info.start_time, queued: info.queued, counted_items: info.counted_items(), - checkpoint_at: now, + checkpoint_at: self.next_scanner_data_movement_update(now, rebalance_meta), })) } @@ -5245,63 +5308,101 @@ impl PoolMeta { } pub fn decommission_cancel(&mut self, idx: usize) -> bool { - if let Some(stats) = self.pools.get_mut(idx) { - if let Some(d) = &stats.decommission { - if !d.canceled { - stats.last_update = OffsetDateTime::now_utc(); + self.decommission_cancel_at(idx, OffsetDateTime::now_utc(), None) + } - let mut pd = d.clone(); - pd.canceled = true; - pd.failed = false; - pd.complete = false; - pd.start_time = None; - pd.terminal_reload_attempt_at = None; - pd.terminal_reload_failures.clear(); + #[cfg(test)] + pub(crate) fn decommission_cancel_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.decommission_cancel_at(idx, now, rebalance_meta) + } - stats.decommission = Some(pd); - true - } else { - false - } - } else { - false - } - } else { - false + fn decommission_cancel_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool { + let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else { + return false; + }; + if d.canceled { + return false; } + + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let mut pd = d.clone(); + pd.canceled = true; + pd.failed = false; + pd.complete = false; + pd.start_time = None; + pd.terminal_reload_attempt_at = None; + pd.terminal_reload_failures.clear(); + + let Some(stats) = self.pools.get_mut(idx) else { + return false; + }; + stats.last_update = last_update; + stats.decommission = Some(pd); + true } pub fn decommission_failed(&mut self, idx: usize) -> bool { - if let Some(stats) = self.pools.get_mut(idx) { - if let Some(d) = &stats.decommission { - if is_decommission_active(d.complete, d.failed, d.canceled) { - stats.last_update = OffsetDateTime::now_utc(); + self.decommission_failed_at(idx, OffsetDateTime::now_utc(), None) + } - let mut pd = d.clone(); - pd.canceled = false; - pd.failed = true; - pd.complete = false; - pd.start_time = None; - pd.terminal_reload_attempt_at = None; - pd.terminal_reload_failures.clear(); + #[cfg(test)] + pub(crate) fn decommission_failed_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.decommission_failed_at(idx, now, rebalance_meta) + } - stats.decommission = Some(pd); - true - } else { - false - } - } else { - false - } - } else { - false + fn decommission_failed_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool { + let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else { + return false; + }; + if !is_decommission_active(d.complete, d.failed, d.canceled) { + return false; } + + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let mut pd = d.clone(); + pd.canceled = false; + pd.failed = true; + pd.complete = false; + pd.start_time = None; + pd.terminal_reload_attempt_at = None; + pd.terminal_reload_failures.clear(); + + let Some(stats) = self.pools.get_mut(idx) else { + return false; + }; + stats.last_update = last_update; + stats.decommission = Some(pd); + true } pub fn clear_decommission(&mut self, idx: usize) -> Result { + self.clear_decommission_at(idx, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn clear_decommission_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result { + self.clear_decommission_at(idx, now, rebalance_meta) + } + + fn clear_decommission_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> Result { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; - let Some(pool) = self.pools.get_mut(idx) else { + let Some(pool) = self.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; @@ -5321,54 +5422,97 @@ impl PoolMeta { ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?; - pool.last_update = OffsetDateTime::now_utc(); - pool.decommission = None; + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + pool.last_update = last_update; + // Preserve a state-empty tombstone so scanner catch-up can recover the + // durable movement generation after a clear followed by a restart. + pool.decommission = Some(PoolDecommissionInfo::default()); Ok(true) } pub fn decommission_complete(&mut self, idx: usize) -> bool { - if let Some(stats) = self.pools.get_mut(idx) { - if let Some(d) = &stats.decommission { - if is_decommission_active(d.complete, d.failed, d.canceled) { - stats.last_update = OffsetDateTime::now_utc(); - - let mut pd = d.clone(); - pd.canceled = false; - pd.failed = false; - pd.complete = true; - pd.terminal_reload_attempt_at = None; - pd.terminal_reload_failures.clear(); - - stats.decommission = Some(pd); - true - } else { - false - } - } else { - false - } - } else { - false - } + self.decommission_complete_at(idx, OffsetDateTime::now_utc(), None) } - fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> { + + #[cfg(test)] + pub(crate) fn decommission_complete_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.decommission_complete_at(idx, now, rebalance_meta) + } + + fn decommission_complete_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool { + let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else { + return false; + }; + if !is_decommission_active(d.complete, d.failed, d.canceled) { + return false; + } + + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let mut pd = d.clone(); + pd.canceled = false; + pd.failed = false; + pd.complete = true; + pd.terminal_reload_attempt_at = None; + pd.terminal_reload_failures.clear(); + + let Some(stats) = self.pools.get_mut(idx) else { + return false; + }; + stats.last_update = last_update; + stats.decommission = Some(pd); + true + } + fn set_decommission_state_at( + &mut self, + idx: usize, + pi: PoolSpaceInfo, + queued: bool, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result<()> { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; - let Some(pool) = self.pools.get_mut(idx) else { + let Some(pool) = self.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; ensure_decommission_start_allowed(decommission_start_pool_state(Some(pool)))?; + let generation = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; let previous = pool.decommission.as_ref(); - let now = OffsetDateTime::now_utc(); - pool.last_update = now; - pool.decommission = Some(build_decommission_start_state(pi, queued, now, previous)); + pool.last_update = generation; + pool.decommission = Some(build_decommission_start_state(pi, queued, generation, previous)); Ok(()) } + fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> { + self.set_decommission_state_at(idx, pi, queued, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn decommission_at_for_test( + &mut self, + idx: usize, + pi: PoolSpaceInfo, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result<()> { + self.set_decommission_state_at(idx, pi, false, now, rebalance_meta) + } + pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> { self.set_decommission_state(idx, pi, false) } @@ -5378,13 +5522,36 @@ impl PoolMeta { } pub fn record_decommission_terminal_reload_failure(&mut self, idx: usize, stage: &str, message: String) -> Result { + self.record_decommission_terminal_reload_failure_at(idx, stage, message, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn record_decommission_terminal_reload_failure_at_for_test( + &mut self, + idx: usize, + stage: &str, + message: String, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result { + self.record_decommission_terminal_reload_failure_at(idx, stage, message, now, rebalance_meta) + } + + fn record_decommission_terminal_reload_failure_at( + &mut self, + idx: usize, + stage: &str, + message: String, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; - let Some(pool) = self.pools.get_mut(idx) else { + let Some(pool) = self.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; - let Some(info) = pool.decommission.as_mut() else { + let Some(info) = pool.decommission.as_ref() else { return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure")); }; @@ -5393,29 +5560,60 @@ impl PoolMeta { return Ok(false); } - pool.last_update = OffsetDateTime::now_utc(); - info.terminal_reload_attempt_at = Some(pool.last_update); + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_mut() else { + return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure")); + }; + pool.last_update = last_update; + info.terminal_reload_attempt_at = Some(last_update); info.terminal_reload_failures.push(failure); Ok(true) } pub fn promote_queued_decommission(&mut self, idx: usize) -> bool { - if let Some(pool) = self.pools.get_mut(idx) - && let Some(info) = pool.decommission.as_mut() - && info.queued - && is_decommission_active(info.complete, info.failed, info.canceled) - { - let now = OffsetDateTime::now_utc(); - pool.last_update = now; - info.queued = false; - let generation = *info.start_time.get_or_insert(now); - for entry in &mut info.unresolved_entries { - entry.source_generation = generation; - } - return true; + self.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn promote_queued_decommission_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.promote_queued_decommission_at(idx, now, rebalance_meta) + } + + fn promote_queued_decommission_at( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + let Some(info) = self.pools.get(idx).and_then(|pool| pool.decommission.as_ref()) else { + return false; + }; + if !info.queued || !is_decommission_active(info.complete, info.failed, info.canceled) { + return false; } - false + let generation = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return false; + }; + let Some(info) = pool.decommission.as_mut() else { + return false; + }; + pool.last_update = generation; + info.queued = false; + info.start_time = Some(generation); + for entry in &mut info.unresolved_entries { + entry.source_generation = generation; + } + true } pub fn queue_buckets(&mut self, idx: usize, bks: Vec) { if let Some(pool) = self.pools.get_mut(idx) @@ -5494,7 +5692,7 @@ impl PoolMeta { pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result { Ok(self - .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc())? + .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc(), None)? .is_some()) } @@ -6241,9 +6439,17 @@ impl ECStore { entry: DecommissionUnresolvedEntry, ) -> Result<()> { { + let rebalance_meta = self.rebalance_meta.read().await.clone(); let mut pool_meta = self.pool_meta.write().await; ensure_decommission_ledger_persistence_supported(&pool_meta)?; - record_decommission_unresolved_entry(&mut pool_meta, idx, generation, entry)?; + record_decommission_unresolved_entry( + &mut pool_meta, + idx, + generation, + entry, + OffsetDateTime::now_utc(), + rebalance_meta.as_ref(), + )?; } self.save_current_pool_meta(&[idx]) .await @@ -6251,19 +6457,32 @@ impl ECStore { } async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result { - // Lock order: save gate, then the short pool metadata read/write sections. Peer - // reloads are intentionally performed by the caller after both locks are released. + self.save_decommission_progress_checkpoint_at(idx, generation, OffsetDateTime::now_utc()) + .await + } + + async fn save_decommission_progress_checkpoint_at( + &self, + idx: usize, + generation: OffsetDateTime, + now: OffsetDateTime, + ) -> Result { + // Lock order: save gate, rebalance metadata, then the short pool + // metadata read/write sections. Peer reloads are intentionally + // performed by the caller after both locks are released. let mut save_guard = self.pool_meta_save_gate.lock().await; let (pool_meta_guard, mut snapshot) = self .acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed") .await?; let (snapshot, checkpoint) = { + let rebalance_meta = self.rebalance_meta.read().await.clone(); let pool_meta = self.pool_meta.read().await; ensure_decommission_generation(&pool_meta, idx, generation)?; let Some(checkpoint) = pool_meta.decommission_progress_checkpoint( idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL, - OffsetDateTime::now_utc(), + now, + rebalance_meta.as_ref(), )? else { return Ok(false); @@ -6398,11 +6617,13 @@ impl ECStore { let previous_pool_meta = latest_pool_meta.clone(); let first_idx = indices.first().copied(); for (idx, pi) in space_infos { - if Some(idx) == first_idx { - latest_pool_meta.decommission(idx, pi)?; - } else { - latest_pool_meta.queue_decommission(idx, pi)?; - } + latest_pool_meta.set_decommission_state_at( + idx, + pi, + Some(idx) != first_idx, + OffsetDateTime::now_utc(), + Some(&rebalance_meta), + )?; latest_pool_meta.queue_buckets(idx, decom_buckets.clone()); } @@ -6517,6 +6738,12 @@ impl ECStore { Ok(space_infos) } + pub(crate) async fn next_scanner_data_movement_update(&self, now: OffsetDateTime) -> OffsetDateTime { + let pool_meta = self.pool_meta.read().await; + let rebalance_meta = self.rebalance_meta.read().await; + pool_meta.next_scanner_data_movement_update(now, rebalance_meta.as_ref()) + } + #[tracing::instrument(skip(self))] pub async fn decommission_cancel(self: &Arc, idx: usize) -> Result<()> { self.decommission_cancel_with_owner(idx, None).await @@ -6575,12 +6802,14 @@ impl ECStore { .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal); // Lock order: start gate, save gate, distributed pool metadata fence, - // decommission_cancelers, then pool_meta. The state guards stay held - // across persistence so the active generation cannot change before - // the cancel is published. + // rebalance_meta, decommission_cancelers, then pool_meta. The state + // guards stay held across persistence so the active generation cannot + // change before the cancel is published. + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let mut cancelers = self.decommission_cancelers.write().await; let mut pool_meta = self.pool_meta.write().await; - let (pending, should_reload_pool_meta, already_canceled, terminal_canceler) = { + let (pending, should_reload_pool_meta, already_canceled, terminal_canceler, durable_movement_generation) = { let mut already_canceled = false; let (pool_present, decommission_present, terminal) = if let Some(pool) = pool_meta.pools.get(idx) { if let Some(info) = pool.decommission.as_ref() { @@ -6609,7 +6838,7 @@ impl ECStore { .ok_or_else(|| decommission_metadata_not_initialized_error("cancel decommission"))?; let mut snapshot = pool_meta.clone(); let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut snapshot, idx, owner, |pool_meta| { - pool_meta.decommission_cancel(idx) + pool_meta.decommission_cancel_at(idx, terminal_at, rebalance_meta.as_ref()) }) else { return Ok(()); }; @@ -6640,11 +6869,16 @@ impl ECStore { } else { cancelers.get(idx).and_then(Option::as_ref).cloned() }; + let durable_movement_generation = pending + .as_ref() + .map(|(_, commit)| crate::store::scanner_data_movement_timestamp_generation(commit.canceled_pool.last_update)) + .unwrap_or_default(); ( pending, should_retry_decommission_cancel_reload(changed, already_canceled), already_canceled, terminal_canceler, + durable_movement_generation, ) }; let active_worker = terminal_canceler.as_ref().is_some_and(DecommissionCanceler::is_active); @@ -6698,7 +6932,8 @@ impl ECStore { // the shared gate without deadlocking the terminal transition. let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; - self.ctx.advance_data_movement_operation_epoch(); + self.ctx + .advance_data_movement_operation_epoch_to_durable_generation(durable_movement_generation); } if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { @@ -6897,9 +7132,11 @@ impl ECStore { let _movement_guard = movement_gate.write().await; let (should_reload_pool_meta, previous_pool_meta) = { + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); - let changed = pool_meta.clear_decommission(idx)?; + let changed = pool_meta.clear_decommission_at(idx, terminal_at, rebalance_meta.as_ref())?; (changed, changed.then_some(previous_pool_meta)) }; @@ -6946,12 +7183,13 @@ impl ECStore { let (pool_meta_guard, mut snapshot) = self .acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed") .await?; + let rebalance_meta = self.rebalance_meta.read().await.clone(); let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); } let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx); - let promoted = pool_meta.promote_queued_decommission(idx); + let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref()); let changed = reconciled || promoted; if changed { merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &[idx], "decommission promotion failed")?; @@ -7019,8 +7257,15 @@ impl ECStore { let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; let changed = { + let rebalance_meta = self.rebalance_meta.read().await.clone(); let mut pool_meta = self.pool_meta.write().await; - pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())? + pool_meta.record_decommission_terminal_reload_failure_at( + idx, + stage, + err.to_string(), + OffsetDateTime::now_utc(), + rebalance_meta.as_ref(), + )? }; if changed { @@ -9018,16 +9263,18 @@ impl ECStore { let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; - // Lock order: movement gate, then decommission_cancelers, then pool_meta. - // Holding both state locks makes owner validation and the terminal - // transition one atomic operation. + // Lock order: movement gate, rebalance_meta, decommission_cancelers, + // then pool_meta. Holding both state locks makes owner validation and + // the terminal transition one atomic operation. + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| { - pool_meta.decommission_failed(idx) + pool_meta.decommission_failed_at(idx, terminal_at, rebalance_meta.as_ref()) }) else { return Ok(()); @@ -9135,9 +9382,11 @@ impl ECStore { let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; - // Lock order: movement gate, then decommission_cancelers, then pool_meta. - // Holding both state locks makes owner validation and the terminal - // transition one atomic operation. + // Lock order: movement gate, rebalance_meta, decommission_cancelers, + // then pool_meta. Holding both state locks makes owner validation and + // the terminal transition one atomic operation. + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; @@ -9150,7 +9399,7 @@ impl ECStore { verified_generation, verified_unresolved_entries.as_deref(), )?; - Ok::(pool_meta.decommission_complete(idx)) + Ok::(pool_meta.decommission_complete_at(idx, terminal_at, rebalance_meta.as_ref())) }) else { return Ok(()); @@ -13661,8 +13910,10 @@ mod pools_tests { ); let pool_meta = store.pool_meta.read().await; assert!( - pool_meta.pools[0].decommission.is_none(), - "the detached transaction should publish the persisted clear" + pool_meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) ); } @@ -14100,7 +14351,12 @@ mod pools_tests { merge_pool_meta_updates_for_save(&mut persisted, ¤t, &[0], "clear decommission") .expect("the target pool update should merge into the latest snapshot"); - assert!(persisted.pools[0].decommission.is_none()); + assert!( + persisted.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); assert_eq!(persisted.pools[1].last_update, newer); assert!(persisted.pools[1].decommission.as_ref().is_some_and(|info| info.failed)); } @@ -14141,6 +14397,59 @@ mod pools_tests { assert!(persisted.pools[0].decommission.as_ref().is_some_and(|info| info.canceled)); } + #[test] + fn test_pool_meta_save_merge_rejects_clear_over_completed_or_unresolved_state() { + let timestamp = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid"); + for persisted_info in [ + PoolDecommissionInfo { + complete: true, + ..Default::default() + }, + PoolDecommissionInfo { + failed: true, + unresolved_entries: vec![DecommissionUnresolvedEntry { + bucket: "bucket-a".to_string(), + object: "object-a".to_string(), + pool_index: 0, + set_index: 0, + source_generation: timestamp, + candidate_count: 1, + disk_error_count: 1, + observed_at: timestamp, + reason: "test unresolved entry".to_string(), + }], + ..Default::default() + }, + ] { + let mut persisted = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: timestamp, + decommission: Some(persisted_info), + }], + ..Default::default() + }; + let cleared = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::now_utc(), + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + + let err = merge_pool_meta_updates_for_save(&mut persisted, &cleared, &[0], "clear decommission") + .expect_err("a stale clear must not erase completed or unresolved state"); + + assert!( + err.to_string() + .contains("completed or unresolved decommission state cannot be cleared") + ); + } + } + #[test] fn test_pool_meta_publish_preserves_untouched_runtime_progress() { let mut current = PoolMeta { @@ -15532,8 +15841,15 @@ mod pools_tests { }; assert!( - record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry.clone()) - .expect("active generation should accept unresolved entry") + record_decommission_unresolved_entry( + &mut pool_meta, + 0, + generation, + unresolved_entry.clone(), + generation + Duration::nanoseconds(1), + None, + ) + .expect("active generation should accept unresolved entry") ); let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, None, None) .expect_err("unverified completion must retain unresolved entries"); @@ -15557,8 +15873,15 @@ mod pools_tests { replacement.disk_error_count = 2; replacement.observed_at = generation + Duration::seconds(1); assert!( - record_decommission_unresolved_entry(&mut pool_meta, 0, generation, replacement.clone()) - .expect("a newer observation should replace the ledger entry") + record_decommission_unresolved_entry( + &mut pool_meta, + 0, + generation, + replacement.clone(), + generation + Duration::nanoseconds(2), + None, + ) + .expect("a newer observation should replace the ledger entry") ); let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&stale_verified)) @@ -15654,6 +15977,100 @@ mod pools_tests { ); } + #[tokio::test] + async fn decommission_metadata_saves_stay_monotonic_across_clock_rollback_before_terminal_restart() { + let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await; + let persisted_floor = store.pool_meta.read().await.pools[0].last_update; + let generation = persisted_floor + .checked_add(Duration::seconds(1)) + .expect("test generation should advance the initialized pool metadata"); + let earlier_tick = generation - Duration::nanoseconds(10); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.dont_save = false; + pool_meta.pools[0].last_update = generation; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(generation), + ..Default::default() + }); + } + store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("active decommission metadata should persist before rollback checkpoints"); + + let unresolved_entry = DecommissionUnresolvedEntry { + bucket: "bucket-a".to_string(), + object: "object-a".to_string(), + pool_index: 0, + set_index: 0, + source_generation: generation, + candidate_count: 1, + disk_error_count: 1, + observed_at: earlier_tick, + reason: "metadata_resolution_failed".to_string(), + }; + { + let mut pool_meta = store.pool_meta.write().await; + assert!( + record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry, earlier_tick, None) + .expect("unresolved entry should record under active generation") + ); + assert_eq!(pool_meta.pools[0].last_update, generation + Duration::nanoseconds(1)); + } + store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("rollback unresolved-entry save should not stale-reject"); + store.pool_meta.write().await.pools[0] + .decommission + .as_mut() + .expect("decommission metadata should exist") + .items_decommissioned = DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD; + + assert!( + store + .save_decommission_progress_checkpoint_at(0, generation, earlier_tick) + .await + .expect("rollback progress checkpoint should use a monotonic durable identity") + ); + let progress_at = generation + Duration::nanoseconds(2); + assert_eq!(store.pool_meta.read().await.pools[0].last_update, progress_at); + + let terminal_at = generation + Duration::nanoseconds(3); + { + let mut pool_meta = store.pool_meta.write().await; + assert!(pool_meta.decommission_failed_at_for_test(0, earlier_tick, None)); + assert_eq!(pool_meta.pools[0].last_update, terminal_at); + } + store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("terminal failure after rollback checkpoints should persist"); + + let mut restored = PoolMeta::default(); + restored + .load_no_lock_from_replicas(vec![store.pools[0].clone()]) + .await + .expect("terminal metadata should reload after restart"); + assert_eq!(restored.pools[0].last_update, terminal_at); + assert!( + restored.pools[0] + .decommission + .as_ref() + .is_some_and(|info| info.failed && !info.complete && !info.canceled) + ); + + let restarted = decommission_worker_test_store(restored, Vec::new()); + let status = restarted.scanner_data_movement_pause_status().await; + let expected_generation = + u64::try_from(terminal_at.unix_timestamp_nanos()).expect("fixed positive timestamp should fit generation"); + + assert_eq!(status.movement_generation, expected_generation); + assert_eq!(status.reasons, vec![crate::store::ScannerDataMovementPauseReason::DecommissionFailed]); + assert_eq!(restarted.scanner_data_movement_generation(), expected_generation); + } + #[tokio::test] async fn unresolved_entry_probe_verifies_absence_on_every_source_disk() { let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await; @@ -16126,7 +16543,7 @@ mod pools_tests { }; let checkpoint = meta - .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("valid decommission state should produce a checkpoint") .expect("item threshold should produce a checkpoint"); meta.count_item(0, 1, false); @@ -16158,13 +16575,13 @@ mod pools_tests { }; let checkpoint = meta - .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("valid decommission state should produce a checkpoint") .expect("item threshold should produce a checkpoint"); meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after); assert!( - meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("retry backoff check should succeed") .is_none() ); @@ -16200,7 +16617,7 @@ mod pools_tests { for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) { meta.count_item(0, 1, false); if let Some(checkpoint) = meta - .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("valid decommission state should produce a checkpoint") { checkpoint_count += 1; @@ -17081,7 +17498,12 @@ mod pools_tests { assert!(meta.is_suspended(0)); assert!(meta.clear_decommission(0).expect("terminal decommission should clear")); - assert!(meta.pools[0].decommission.is_none()); + assert!( + meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); assert!(!meta.is_suspended(0)); } } @@ -17299,6 +17721,40 @@ mod pools_tests { assert_eq!(promoted.unresolved_entries[0].source_generation, generation); } + #[test] + fn test_queued_decommission_promotion_advances_generation_after_clock_rollback() { + let queued_at = OffsetDateTime::from_unix_timestamp(1_260).expect("fixed timestamp should be valid"); + let earlier_tick = queued_at - Duration::nanoseconds(10); + let rebalance_floor = queued_at + Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: queued_at, + decommission: Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!(meta.promote_queued_decommission_at_for_test(0, earlier_tick, Some(&rebalance))); + let expected_generation = rebalance_floor + Duration::nanoseconds(1); + let promoted = meta.pools[0] + .decommission + .as_ref() + .expect("promoted decommission metadata should exist"); + assert_eq!(meta.pools[0].last_update, expected_generation); + assert_eq!(promoted.start_time, Some(expected_generation)); + assert!(!promoted.queued); + } + #[test] fn test_pool_meta_queued_decommission_is_not_suspended_until_promoted() { let mut meta = PoolMeta { @@ -17409,7 +17865,12 @@ mod pools_tests { assert_eq!(blocked.bytes_done, 1024); assert!(meta.clear_decommission(0).expect("failed decommission should clear")); - assert!(meta.pools[0].decommission.is_none()); + assert!( + meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); meta.decommission( 0, @@ -17496,7 +17957,12 @@ mod pools_tests { assert_eq!(blocked.bytes_done, 512); assert!(meta.clear_decommission(0).expect("canceled decommission should clear")); - assert!(meta.pools[0].decommission.is_none()); + assert!( + meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); meta.queue_decommission( 0, @@ -18514,6 +18980,7 @@ mod pools_tests { let save_started = Arc::new(tokio::sync::Notify::new()); let save_release = Arc::new(tokio::sync::Notify::new()); let save_entered = Arc::new(AtomicBool::new(false)); + let persisted = Arc::new(std::sync::Mutex::new(None)); let mut cancel = tokio::spawn({ let store = store.clone(); @@ -18521,9 +18988,12 @@ mod pools_tests { let save_started = save_started.clone(); let save_release = save_release.clone(); let save_entered = save_entered.clone(); + let persisted = persisted.clone(); async move { store - .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move { + .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move { + *persisted.lock().expect("persisted cancel lock should not be poisoned") = + Some(snapshot.encode_config_data()?); save_entered.store(true, Ordering::SeqCst); save_started.notify_one(); save_release.notified().await; @@ -18560,7 +19030,7 @@ mod pools_tests { .await .expect("the durable cancel must signal the active worker"); assert!(!cancel.is_finished(), "cancel must wait for in-flight movement after the durable signal"); - { + let canceled_at = { let pool_meta = store.pool_meta.read().await; let info = pool_meta.pools[0] .decommission @@ -18570,7 +19040,11 @@ mod pools_tests { assert!(!info.complete); assert!(!info.failed); assert!(info.start_time.is_none()); - } + pool_meta.pools[0].last_update + }; + let expected_generation = crate::store::scanner_data_movement_timestamp_generation(canceled_at); + let scanner_status = store.scanner_data_movement_pause_snapshot_for_test().await; + assert_eq!(scanner_status.movement_generation, expected_generation); drop(side_effect); tokio::time::timeout(StdDuration::from_secs(1), &mut cancel) @@ -18597,6 +19071,27 @@ mod pools_tests { assert!(canceler.is_cancelled()); assert!(!canceler.is_active()); assert!(store.decommission_cancelers.read().await[0].is_none()); + assert_eq!(store.scanner_data_movement_generation(), expected_generation); + + let persisted = persisted + .lock() + .expect("persisted cancel lock should not be poisoned") + .take() + .expect("cancel should persist a durable snapshot"); + let mut durable = PoolMeta::default(); + durable + .load_from_config_data(persisted) + .expect("durable cancel snapshot should decode after restart"); + let restarted = decommission_worker_test_store(durable.clone(), Vec::new()); + let restarted_status = restarted.scanner_data_movement_pause_status().await; + assert_eq!(restarted_status.movement_generation, expected_generation); + + assert!( + durable + .clear_decommission_at_for_test(0, canceled_at, None) + .expect("same-tick clear should succeed") + ); + assert!(durable.pools[0].last_update > canceled_at); } #[test] diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index e1bc6b739..c6002250c 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -185,6 +185,12 @@ pub enum StorageError { DecommissionAlreadyRunning, #[error("Rebalance already running")] RebalanceAlreadyRunning, + #[error("{operation}: stale pool metadata update rejected for pool {pool_index}; {reason}")] + StalePoolMetadataUpdate { + operation: String, + pool_index: usize, + reason: &'static str, + }, #[error("Operation canceled")] OperationCanceled, #[error("No heal required")] @@ -564,6 +570,15 @@ impl Clone for StorageError { StorageError::DoneForNow => StorageError::DoneForNow, StorageError::DecommissionAlreadyRunning => StorageError::DecommissionAlreadyRunning, StorageError::RebalanceAlreadyRunning => StorageError::RebalanceAlreadyRunning, + StorageError::StalePoolMetadataUpdate { + operation, + pool_index, + reason, + } => StorageError::StalePoolMetadataUpdate { + operation: operation.clone(), + pool_index: *pool_index, + reason, + }, StorageError::OperationCanceled => StorageError::OperationCanceled, StorageError::ErasureReadQuorum => StorageError::ErasureReadQuorum, StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum, @@ -667,6 +682,7 @@ impl StorageError { StorageError::DoneForNow => StorageErrorCode::DoneForNow, StorageError::DecommissionAlreadyRunning => StorageErrorCode::DecommissionAlreadyRunning, StorageError::RebalanceAlreadyRunning => StorageErrorCode::RebalanceAlreadyRunning, + StorageError::StalePoolMetadataUpdate { .. } => StorageErrorCode::InvalidArgument, StorageError::OperationCanceled => StorageErrorCode::OperationCanceled, StorageError::ErasureReadQuorum => StorageErrorCode::ErasureReadQuorum, StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum, diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 673de7b92..b7090bcdd 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -368,6 +368,19 @@ impl InstanceContext { Arc::clone(&self.data_movement_generation_notify) } + pub(crate) fn observe_durable_data_movement_generation(&self, generation: u64) { + if generation == 0 || self.data_movement_generation_exhausted.load(Ordering::Acquire) { + return; + } + let previous = self.data_movement_generation.fetch_max(generation, Ordering::AcqRel); + if generation == u64::MAX { + self.data_movement_generation_exhausted.store(true, Ordering::Release); + } + if generation > previous { + self.data_movement_generation_notify.notify_waiters(); + } + } + pub(crate) fn scanner_publication_state_allowed(&self) -> bool { !self.data_movement_operation_epoch_exhausted() && !self.data_movement_generation_exhausted() @@ -386,6 +399,20 @@ impl InstanceContext { } pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 { + let (previous, result) = self.advance_data_movement_operation_epoch_only(); + if result != previous { + let _ = self.advance_data_movement_generation(); + } + result + } + + pub(crate) fn advance_data_movement_operation_epoch_to_durable_generation(&self, generation: u64) -> u64 { + let (_, result) = self.advance_data_movement_operation_epoch_only(); + self.observe_durable_data_movement_generation(generation); + result + } + + fn advance_data_movement_operation_epoch_only(&self) -> (u64, u64) { self.scanner_publication_state .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release); let previous = self.data_movement_operation_epoch.load(Ordering::Acquire); @@ -396,10 +423,7 @@ impl InstanceContext { if result == u64::MAX { self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release); } - if result != previous { - let _ = self.advance_data_movement_generation(); - } - result + (previous, result) } /// Advance the movement generation after a durable movement transition. diff --git a/crates/ecstore/src/services/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs index 731a3299b..d9939702d 100644 --- a/crates/ecstore/src/services/rebalance/control.rs +++ b/crates/ecstore/src/services/rebalance/control.rs @@ -845,7 +845,7 @@ impl ECStore { let mut pool_stats = Vec::with_capacity(self.pools.len()); - let now = OffsetDateTime::now_utc(); + let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; for disk_stat in disk_stats.iter() { let mut pool_stat = RebalanceStats { @@ -868,8 +868,10 @@ impl ECStore { pool_stats.push(pool_stat); } + let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating); let meta = RebalanceMeta { id: Uuid::new_v4().to_string(), + stopped_at: (!has_participating_pool).then_some(now), percent_free_goal, pool_stats, ..Default::default() @@ -963,6 +965,18 @@ impl ECStore { ))); } if meta.stopped_at.is_some() { + if !is_rebalance_conflicting_with_decommission(meta) { + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + state = "start_skipped", + reason = "not_started_terminal", + rebalance_id = %expected_id, + "Skipped rebalance start because metadata is already terminal" + ); + return Ok(()); + } return Err(Error::other(format!("rebalance {expected_id} was stopped before start"))); } } @@ -1214,11 +1228,11 @@ impl ECStore { }; let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; + let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; let (previous_meta, meta_to_save) = { let mut rebalance_meta = self.rebalance_meta.write().await; let previous_meta = rebalance_meta.clone(); - let meta_to_save = - stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)?; + let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?; (previous_meta, meta_to_save) }; @@ -1250,14 +1264,10 @@ impl ECStore { .await?; let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; + let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; - rollback_rebalance_start_meta_snapshot_for_id( - rebalance_meta.as_mut(), - OffsetDateTime::now_utc(), - expected_id, - start_error, - ) + rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error) }; if let Some(meta_to_save) = meta_to_save { @@ -1402,6 +1412,62 @@ mod tests { assert!(cancel.is_cancelled()); } + #[tokio::test] + #[serial_test::serial] + async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() { + let (_temp_dirs, store, restarted) = + crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await; + let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid"); + *store.rebalance_meta.write().await = Some(RebalanceMeta { + id: "previous-terminal-rebalance".to_string(), + stopped_at: Some(movement_floor), + ..Default::default() + }); + set_rebalance_disk_stats_override_for_test( + store.id, + vec![ + DiskStat { + total_space: 100, + available_space: 50, + }, + DiskStat { + total_space: 100, + available_space: 50, + }, + ], + ); + + let rebalance_id = store + .init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()]) + .await + .expect("equal free ratio admin rebalance should succeed as a terminal no-op"); + let stopped_at = { + let local = store.rebalance_meta.read().await; + let local = local.as_ref().expect("no-op rebalance metadata should remain available"); + assert_eq!(local.id, rebalance_id); + assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating)); + let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp"); + assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1)); + stopped_at + }; + + let stopped_generation = + u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation"); + let live_status = store.scanner_data_movement_pause_status().await; + assert!(!live_status.paused); + assert_eq!(live_status.movement_generation, stopped_generation); + + restarted + .load_rebalance_meta() + .await + .expect("restarted store should load the persisted no-op rebalance metadata"); + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, stopped_generation); + assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation); + } + #[tokio::test] #[serial_test::serial] async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() { diff --git a/crates/ecstore/src/services/rebalance/runtime.rs b/crates/ecstore/src/services/rebalance/runtime.rs index fe8bb77f9..f85184742 100644 --- a/crates/ecstore/src/services/rebalance/runtime.rs +++ b/crates/ecstore/src/services/rebalance/runtime.rs @@ -161,6 +161,7 @@ impl ECStore { let cancel_tx = CancellationToken::new(); let rx = cancel_tx.clone(); + let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; let activation_outcome; let candidate; let expected_cancel; @@ -185,12 +186,8 @@ impl ECStore { return Ok(false); } expected_cancel = meta.cancel.clone(); - (candidate, activation_outcome, must_persist) = stage_local_rebalance_worker_activation( - meta, - expected_id.as_ref(), - cancel_tx.clone(), - OffsetDateTime::now_utc(), - )?; + (candidate, activation_outcome, must_persist) = + stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?; if let Err(err) = activation_fence.ensure_held() { cancel_tx.cancel(); return Err(err); @@ -384,11 +381,11 @@ impl ECStore { tokio::select! { result = done_rx.recv() => { quit = true; - let now = OffsetDateTime::now_utc(); - let terminal_event = classify_rebalance_terminal_event(result, now); - msg = terminal_event.message().to_string(); let movement_gate = store.ctx.data_movement_operation_gate(); let movement_guard = movement_gate.write().await; + let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; + let terminal_event = classify_rebalance_terminal_event(result, terminal_at); + msg = terminal_event.message().to_string(); let previous_meta = store.rebalance_meta.read().await.clone(); let terminal_state_present = { let mut rebalance_meta = store.rebalance_meta.write().await; @@ -405,7 +402,7 @@ impl ECStore { { pool_stat.info.stopping = false; pool_stat.info.status = RebalStatus::Failed; - pool_stat.info.end_time = Some(now); + pool_stat.info.end_time = Some(terminal_at); pool_stat.info.last_error = Some( pool_stat .cleanup_warnings @@ -433,7 +430,7 @@ impl ECStore { &mut pool_stat.info.end_time, &mut pool_stat.info.last_error, terminal_event, - now, + terminal_at, ); } true @@ -835,6 +832,10 @@ impl ECStore { opt: RebalSaveOpt, expected_id: Option<&str>, ) -> Result<()> { + let now = match opt { + RebalSaveOpt::Stats => OffsetDateTime::now_utc(), + RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await, + }; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; if let Some(expected_id) = expected_id { @@ -844,7 +845,6 @@ impl ECStore { return Ok(()); }; - let now = OffsetDateTime::now_utc(); apply_rebalance_save_option(meta, pool_idx, opt, now); meta.clone() }; diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 53c5346e4..c3dd77605 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -44,7 +44,7 @@ use crate::error::{ use crate::runtime::global::DISK_RESERVE_FRACTION; use crate::runtime::instance::InstanceContext; use crate::runtime::sources as runtime_sources; -use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission}; +use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission}; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, @@ -273,6 +273,215 @@ pub struct ECStore { pub(crate) bucket_fence_registry: Arc, } +const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused"; +const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds"; +const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items"; +const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScannerDataMovementPauseReason { + OperationEpochExhausted, + MovementGenerationExhausted, + DecommissionActive, + DecommissionFailed, + DecommissionCanceled, + RebalanceActive, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct ScannerDataMovementPauseStatus { + pub paused: bool, + pub policy: &'static str, + pub reasons: Vec, + pub started_at_unix_secs: u64, + pub duration_seconds: u64, + pub operation_epoch: u64, + pub movement_generation: u64, + pub movement_backlog_work_items: u64, + pub movement_backlog_estimated: bool, +} + +impl Default for ScannerDataMovementPauseStatus { + fn default() -> Self { + Self { + paused: false, + policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY, + reasons: Vec::new(), + started_at_unix_secs: 0, + duration_seconds: 0, + operation_epoch: 0, + movement_generation: 0, + movement_backlog_work_items: 0, + movement_backlog_estimated: false, + } + } +} + +fn offset_unix_seconds(value: OffsetDateTime) -> u64 { + u64::try_from(value.unix_timestamp()).unwrap_or(0) +} + +fn earliest_timestamp(current: Option, candidate: Option) -> Option { + match (current, candidate) { + (Some(current), Some(candidate)) => Some(current.min(candidate)), + (Some(current), None) => Some(current), + (None, candidate) => candidate, + } +} + +fn usize_to_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +fn metric_u64(value: u64) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 { + let timestamp = value.unix_timestamp_nanos(); + if timestamp <= 0 { + 0 + } else { + u64::try_from(timestamp).unwrap_or(u64::MAX) + } +} + +fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option { + let generation = scanner_data_movement_timestamp_generation(value); + (generation != 0 && generation != u64::MAX).then_some(generation) +} + +fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 { + let mut generation = 0; + for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) { + let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else { + return u64::MAX; + }; + generation = generation.max(pool_generation); + } + + for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| { + meta.stopped_at.into_iter().chain( + meta.pool_stats + .iter() + .flat_map(|pool| [pool.info.start_time, pool.info.end_time]) + .flatten(), + ) + }) { + let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else { + return u64::MAX; + }; + generation = generation.max(rebalance_generation); + } + + if generation == 0 + && rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some()) + { + u64::MAX + } else { + generation + } +} + +#[derive(Clone, Copy)] +struct ScannerDataMovementSequenceState { + operation_epoch: u64, + operation_epoch_exhausted: bool, + movement_generation: u64, + movement_generation_exhausted: bool, +} + +fn resolve_scanner_data_movement_pause_status( + pool_meta: &PoolMeta, + rebalance_meta: Option<&RebalanceMeta>, + decommission_worker_active: bool, + sequence: ScannerDataMovementSequenceState, + now: OffsetDateTime, +) -> ScannerDataMovementPauseStatus { + let mut decommission_active = decommission_worker_active; + let mut decommission_failed = false; + let mut decommission_canceled = false; + let mut rebalance_active = false; + let mut started_at = None; + let mut movement_backlog_work_items = 0_u64; + + for pool in &pool_meta.pools { + let Some(info) = pool.decommission.as_ref() else { + continue; + }; + let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled; + let failed = !info.queued && info.failed; + let canceled = !info.queued && info.canceled; + if !(active || failed || canceled) { + continue; + } + + decommission_active |= active; + decommission_failed |= failed; + decommission_canceled |= canceled; + started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update))); + let queued = usize_to_u64(info.queued_buckets.len()); + let current_bucket = if info.bucket.is_empty() { 0 } else { 1 }; + movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket)); + } + + if let Some(rebalance_meta) = rebalance_meta { + for pool in &rebalance_meta.pool_stats { + let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping; + if !active { + continue; + } + rebalance_active = true; + started_at = earliest_timestamp(started_at, pool.info.start_time); + movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len())); + } + } + + let mut reasons = Vec::with_capacity(6); + if sequence.operation_epoch_exhausted { + reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted); + } + if sequence.movement_generation_exhausted { + reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted); + } + if decommission_active { + reasons.push(ScannerDataMovementPauseReason::DecommissionActive); + } + if decommission_failed { + reasons.push(ScannerDataMovementPauseReason::DecommissionFailed); + } + if decommission_canceled { + reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled); + } + if rebalance_active { + reasons.push(ScannerDataMovementPauseReason::RebalanceActive); + } + let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0); + let duration_seconds = started_at + .and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok()) + .unwrap_or(0); + let paused = !reasons.is_empty(); + + ScannerDataMovementPauseStatus { + paused, + policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY, + reasons, + started_at_unix_secs, + duration_seconds, + operation_epoch: sequence.operation_epoch, + movement_generation: sequence.movement_generation, + movement_backlog_work_items, + movement_backlog_estimated: paused, + } +} + +fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) { + metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 }); + metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds)); + metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items)); +} + impl std::fmt::Debug for ECStore { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum(); @@ -300,6 +509,28 @@ impl ECStore { self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect() } + /// Erasure sets that may receive scanner pause-backlog replicas. + /// + /// An actively decommissioning or already decommissioned source pool is + /// excluded so an operational record acknowledged during movement always + /// has a copy on storage that remains in the cluster. The record is kept + /// separate from pool and rebalance metadata. + pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec> { + let pool_meta = self.pool_meta.read().await; + self.pools + .iter() + .enumerate() + .filter(|(pool_index, _)| { + !pool_meta.pools.get(*pool_index).is_some_and(|pool| { + pool.decommission + .as_ref() + .is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled) + }) + }) + .flat_map(|(_, pool)| pool.disk_set.iter().cloned()) + .collect() + } + /// Get server configuration (delegates to global) pub fn get_server_config(&self) -> Option { runtime_sources::server_config() @@ -454,14 +685,14 @@ impl ECStore { self.scanner_data_usage_publication_snapshot_blocked().await } + pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus { + let operation_gate = self.ctx.data_movement_operation_gate(); + let _operation_guard = operation_gate.read_owned().await; + self.scanner_data_movement_pause_snapshot().await + } + async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool { - if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() { - self.ctx.set_scanner_publication_state(true); - return true; - } - let (_, blocked) = self.scanner_data_movement_snapshot_locked().await; - self.ctx.set_scanner_publication_state(blocked); - blocked + self.scanner_data_movement_pause_snapshot().await.paused } async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) { @@ -481,19 +712,56 @@ impl ECStore { .as_ref() .is_some_and(|info| !info.queued && (info.failed || info.canceled)) }); - drop(pool_meta); - - let rebalance_active = self - .rebalance_meta - .read() - .await + let rebalance_meta = self.rebalance_meta.read().await; + let rebalance_active = rebalance_meta .as_ref() .is_some_and(is_rebalance_conflicting_with_decommission); + self.ctx + .observe_durable_data_movement_generation(durable_scanner_data_movement_generation( + &pool_meta, + rebalance_meta.as_ref(), + )); let blocked = decommission_active || decommission_terminal || rebalance_active; (decommission_active || rebalance_active, blocked) } + async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus { + let decommission_active = { + let decommission_cancelers = self.decommission_cancelers.read().await; + decommission_cancelers + .iter() + .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active)) + }; + let pool_meta = self.pool_meta.read().await.clone(); + let rebalance_meta = self.rebalance_meta.read().await.clone(); + self.ctx + .observe_durable_data_movement_generation(durable_scanner_data_movement_generation( + &pool_meta, + rebalance_meta.as_ref(), + )); + let status = resolve_scanner_data_movement_pause_status( + &pool_meta, + rebalance_meta.as_ref(), + decommission_active, + ScannerDataMovementSequenceState { + operation_epoch: self.ctx.data_movement_operation_epoch(), + operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(), + movement_generation: self.ctx.data_movement_generation(), + movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(), + }, + OffsetDateTime::now_utc(), + ); + self.ctx.set_scanner_publication_state(status.paused); + record_scanner_data_movement_pause_status(&status); + status + } + + #[cfg(test)] + pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus { + self.scanner_data_movement_pause_snapshot().await + } + /// Admit one short data-usage publication commit under the same /// per-instance gate used by decommission side effects and transitions. /// The epoch is sampled while the read guard is held, so a transition @@ -1196,7 +1464,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore { #[cfg(test)] mod tests { use super::*; - use crate::core::pools::{PoolDecommissionInfo, PoolStatus}; + use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus}; use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType}; use crate::object_api::ObjectOptions; use crate::runtime::global::reset_local_disk_test_state; @@ -1326,6 +1594,558 @@ mod tests { }) } + fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState { + ScannerDataMovementSequenceState { + operation_epoch, + operation_epoch_exhausted: false, + movement_generation, + movement_generation_exhausted: false, + } + } + + #[test] + fn scanner_pause_status_derives_restart_stable_decommission_fields() { + let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid"); + let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid"); + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: started_at, + decommission: Some(PoolDecommissionInfo { + start_time: Some(started_at), + queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + bucket: "bucket-a".to_string(), + ..Default::default() + }), + }], + ..Default::default() + }; + + let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now); + + assert!(status.paused); + assert_eq!(status.policy, "global_pause"); + assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]); + assert_eq!(status.started_at_unix_secs, 1_000); + assert_eq!(status.duration_seconds, 90); + assert_eq!(status.operation_epoch, 7); + assert_eq!(status.movement_generation, 11); + assert_eq!(status.movement_backlog_work_items, 2); + assert!(status.movement_backlog_estimated); + } + + #[test] + fn completed_decommission_restores_durable_movement_generation() { + let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid"); + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: completed_at, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None); + let ctx = InstanceContext::new(); + + ctx.observe_durable_data_movement_generation(durable_generation); + + assert_eq!(durable_generation, 1_100_000_000_000); + assert_eq!(ctx.data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn cleared_decommission_restores_durable_movement_generation_after_restart() { + let mut pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }], + ..Default::default() + }; + assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear")); + assert!( + pool_meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None); + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + + let status = restarted.scanner_data_movement_pause_status().await; + + assert_ne!(durable_generation, 0); + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid"); + let mut pool_meta = PoolMeta { + pools: vec![ + PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }, + PoolStatus { + id: 1, + cmd_line: "pool-1".to_string(), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + canceled: true, + ..Default::default() + }), + }, + ], + ..Default::default() + }; + + assert!( + pool_meta + .clear_decommission_at_for_test(0, same_tick, None) + .expect("first terminal decommission should clear") + ); + let first_generation = durable_scanner_data_movement_generation(&pool_meta, None); + assert_eq!( + first_generation, + scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1)) + ); + + assert!( + pool_meta + .clear_decommission_at_for_test(1, same_tick, None) + .expect("second terminal decommission should clear") + ); + let second_generation = durable_scanner_data_movement_generation(&pool_meta, None); + assert_eq!( + second_generation, + scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2)) + ); + assert!(second_generation > first_generation); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, second_generation); + assert_eq!(restarted.scanner_data_movement_generation(), second_generation); + } + + #[tokio::test] + async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid"); + let earlier_tick = same_tick - time::Duration::nanoseconds(10); + let rebalance_floor = same_tick + time::Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let active_decommission = |id| PoolStatus { + id, + cmd_line: format!("pool-{id}"), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + start_time: Some(same_tick), + ..Default::default() + }), + }; + let mut pool_meta = PoolMeta { + pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)], + ..Default::default() + }; + + assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance))); + assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1)); + + assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance))); + assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2)); + + assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance))); + assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3)); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance)); + assert_eq!( + durable_generation, + scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3)) + ); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = Some(rebalance); + let status = restarted.scanner_data_movement_pause_status().await; + + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + assert_eq!( + status.reasons, + vec![ + ScannerDataMovementPauseReason::DecommissionFailed, + ScannerDataMovementPauseReason::DecommissionCanceled + ] + ); + } + + #[tokio::test] + async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid"); + let earlier_tick = same_tick - time::Duration::nanoseconds(10); + let rebalance_floor = same_tick + time::Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let mut pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!( + pool_meta + .clear_decommission_at_for_test(0, same_tick, Some(&rebalance)) + .expect("failed decommission should clear") + ); + let cleared_at = rebalance_floor + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, cleared_at); + + pool_meta + .decommission_at_for_test( + 0, + PoolSpaceInfo { + total: 200, + free: 50, + used: 150, + }, + earlier_tick, + Some(&rebalance), + ) + .expect("decommission restart after clear should be allowed"); + let started_at = cleared_at + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, started_at); + assert_eq!( + pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), + Some(started_at) + ); + + assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance))); + let completed_at = started_at + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, completed_at); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance)); + assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at)); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = Some(rebalance); + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() { + let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid"); + let earlier_tick = terminal_at - time::Duration::nanoseconds(10); + let rebalance_floor = terminal_at + time::Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let mut pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: terminal_at, + decommission: Some(PoolDecommissionInfo { + start_time: Some(terminal_at), + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!( + pool_meta + .record_decommission_terminal_reload_failure_at_for_test( + 0, + "complete_decommission", + "peer reload failed".to_string(), + earlier_tick, + Some(&rebalance), + ) + .expect("reload failure should be recorded") + ); + let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, reload_failure_at); + let info = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission metadata should exist"); + assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at)); + assert_eq!( + info.terminal_reload_failures, + vec!["complete_decommission: peer reload failed".to_string()] + ); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance)); + assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at)); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = Some(rebalance); + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid"); + let earlier_tick = same_tick - time::Duration::nanoseconds(10); + let decommission_floor = same_tick + time::Duration::nanoseconds(5); + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + *store.pool_meta.write().await = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: decommission_floor, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + let started_at = store.next_scanner_data_movement_update(same_tick).await; + assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1)); + *store.rebalance_meta.write().await = Some(RebalanceMeta { + id: "rebalance-generation".to_string(), + pool_stats: vec![crate::services::rebalance::RebalanceStats { + participating: true, + info: crate::services::rebalance::RebalanceInfo { + start_time: Some(started_at), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }); + + let completed_at = store.next_scanner_data_movement_update(same_tick).await; + assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2)); + { + let mut rebalance_meta = store.rebalance_meta.write().await; + let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present"); + meta.pool_stats[0].info.status = RebalStatus::Completed; + meta.pool_stats[0].info.end_time = Some(completed_at); + } + + let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await; + assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3)); + { + let mut rebalance_meta = store.rebalance_meta.write().await; + let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present"); + meta.stopped_at = Some(stopped_at); + } + let pool_meta = store.pool_meta.read().await.clone(); + let rebalance_meta = store.rebalance_meta.read().await.clone(); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref()); + assert_eq!( + durable_generation, + scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3)) + ); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = rebalance_meta; + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[test] + fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() { + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX); + let exhausted_generation = + OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid"); + assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX); + } + + #[test] + fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() { + let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid"); + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + let rebalance_meta = RebalanceMeta { + id: "completed-rebalance".to_string(), + stopped_at: Some(valid_rebalance_at), + pool_stats: vec![crate::services::rebalance::RebalanceStats { + participating: true, + info: crate::services::rebalance::RebalanceInfo { + start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)), + end_time: Some(valid_rebalance_at), + status: RebalStatus::Completed, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX); + } + + #[test] + fn durable_movement_generation_without_records_is_zero() { + assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0); + assert_eq!( + durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())), + 0 + ); + } + + #[test] + fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() { + let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid"); + let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid"); + let failed = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }], + ..Default::default() + }; + let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now); + assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]); + assert_eq!(failed_status.started_at_unix_secs, 2_000); + assert_eq!(failed_status.duration_seconds, 30); + + let rebalance = RebalanceMeta { + pool_stats: vec![crate::services::rebalance::RebalanceStats { + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + participating: true, + info: crate::services::rebalance::RebalanceInfo { + start_time: Some(last_update), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + let rebalance_status = resolve_scanner_data_movement_pause_status( + &PoolMeta::default(), + Some(&rebalance), + false, + scanner_sequence_state(4, 13), + now, + ); + assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]); + assert_eq!(rebalance_status.movement_backlog_work_items, 2); + + let exhausted = resolve_scanner_data_movement_pause_status( + &PoolMeta::default(), + None, + false, + ScannerDataMovementSequenceState { + operation_epoch: u64::MAX, + operation_epoch_exhausted: true, + movement_generation: 14, + movement_generation_exhausted: false, + }, + now, + ); + assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]); + assert_eq!(exhausted.started_at_unix_secs, 0); + + let generation_exhausted = resolve_scanner_data_movement_pause_status( + &PoolMeta::default(), + None, + false, + ScannerDataMovementSequenceState { + operation_epoch: 5, + operation_epoch_exhausted: false, + movement_generation: u64::MAX, + movement_generation_exhausted: true, + }, + now, + ); + assert_eq!( + generation_exhausted.reasons, + vec![ScannerDataMovementPauseReason::MovementGenerationExhausted] + ); + + let idle = + resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now); + assert!(!idle.paused); + assert!(idle.reasons.is_empty()); + assert!(!idle.movement_backlog_estimated); + } + #[tokio::test] async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() { let store = build_store_with_ctx(Arc::new(InstanceContext::new())); diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 904fb0913..68617d6f6 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -106,7 +106,7 @@ hex-simd.workspace = true [dev-dependencies] tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] } serial_test = { workspace = true } -temp-env = { workspace = true } +temp-env = { workspace = true, features = ["async_closure"] } tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 3a8cb8d6f..e2992e95a 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -82,8 +82,10 @@ pub use remote_scanner::{ pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config}; pub use rustfs_scanner_contracts::last_minute; pub use scanner::{ - ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner, - reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest, + ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason, + ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, init_data_scanner, + reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_pause_backlog_status, + scanner_topology_digest, }; pub use scanner_io::{ ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 001ae6f8a..b79965d90 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -90,6 +90,144 @@ const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total"; const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60); + +#[cfg(test)] +static SCANNER_STARTUP_OBSERVED_PROBE: LazyLock>>> = + LazyLock::new(|| StdMutex::new(None)); + +#[cfg(test)] +struct ScannerStartupObservedProbeState { + observed: Notify, + resume: Notify, +} + +#[cfg(test)] +struct ScannerObservedProbeState { + store_key: usize, + paused: bool, + notify: Notify, +} + +#[cfg(test)] +pub(super) struct ScannerStartupObservedProbe { + state: Arc, +} + +#[cfg(test)] +static SCANNER_RUNTIME_OBSERVED_PROBE: LazyLock>>> = + LazyLock::new(|| StdMutex::new(None)); + +#[cfg(test)] +pub(super) struct ScannerRuntimeObservedProbe { + state: Arc, +} + +#[cfg(test)] +impl ScannerStartupObservedProbe { + pub(super) fn install() -> Self { + let state = Arc::new(ScannerStartupObservedProbeState { + observed: Notify::new(), + resume: Notify::new(), + }); + let mut probe = SCANNER_STARTUP_OBSERVED_PROBE + .lock() + .expect("scanner startup observed probe should not be poisoned"); + assert!(probe.is_none(), "scanner startup observed probe must be unique"); + *probe = Some(state.clone()); + Self { state } + } + + pub(super) async fn wait(&self) { + tokio::time::timeout(Duration::from_secs(5), self.state.observed.notified()) + .await + .expect("scanner should complete startup pause-backlog observation"); + } + + pub(super) fn resume(&self) { + self.state.resume.notify_one(); + } +} + +#[cfg(test)] +impl ScannerRuntimeObservedProbe { + pub(super) fn install(storeapi: &Arc, paused: bool) -> Self { + let state = Arc::new(ScannerObservedProbeState { + store_key: scanner_observed_probe_store_key(storeapi), + paused, + notify: Notify::new(), + }); + let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE + .lock() + .expect("scanner runtime observed probe should not be poisoned"); + assert!(probe.is_none(), "scanner runtime observed probe must be unique"); + *probe = Some(state.clone()); + Self { state } + } + + pub(super) async fn wait(&self) { + tokio::time::timeout(Duration::from_secs(10), self.state.notify.notified()) + .await + .expect("scanner should complete runtime pause-backlog observation"); + } +} + +#[cfg(test)] +impl Drop for ScannerStartupObservedProbe { + fn drop(&mut self) { + let mut probe = SCANNER_STARTUP_OBSERVED_PROBE + .lock() + .expect("scanner startup observed probe should not be poisoned"); + if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *probe = None; + } + self.state.resume.notify_one(); + } +} + +#[cfg(test)] +impl Drop for ScannerRuntimeObservedProbe { + fn drop(&mut self) { + let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE + .lock() + .expect("scanner runtime observed probe should not be poisoned"); + if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *probe = None; + } + } +} + +#[cfg(test)] +async fn notify_scanner_startup_observed_for_test() { + let probe = { + SCANNER_STARTUP_OBSERVED_PROBE + .lock() + .expect("scanner startup observed probe should not be poisoned") + .clone() + }; + if let Some(probe) = probe { + probe.observed.notify_one(); + probe.resume.notified().await; + } +} + +#[cfg(test)] +fn scanner_observed_probe_store_key(storeapi: &Arc) -> usize { + Arc::as_ptr(storeapi).cast::<()>() as usize +} + +#[cfg(test)] +fn notify_scanner_runtime_observed_for_test(storeapi: &Arc, observation: ScannerPauseBacklogObservation) { + if let Some(probe) = SCANNER_RUNTIME_OBSERVED_PROBE + .lock() + .expect("scanner runtime observed probe should not be poisoned") + .clone() + && probe.store_key == scanner_observed_probe_store_key(storeapi) + && probe.paused == observation.paused + { + probe.notify.notify_one(); + } +} + const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2; /// First-retry delay after a scanner cycle cannot publish authoritative usage. /// @@ -2190,6 +2328,74 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> run_data_scanner_with_maintenance_state(ctx, storeapi, maintenance_features, maintenance_generation).await } +async fn current_scanner_pause_backlog_observation(storeapi: &Arc) -> ScannerPauseBacklogObservation { + let now_unix_secs = scanner_pause_backlog_now(); + let pause = storeapi.scanner_data_movement_pause_status().await; + let metrics = global_metrics().report().await; + ScannerPauseBacklogObservation { + now_unix_secs, + paused: pause.paused, + movement_generation: pause.movement_generation, + movement_work_items: pause.movement_backlog_work_items, + pause_started_at_unix_secs: pause.started_at_unix_secs, + dirty_usage_buckets: metrics.usage_freshness.dirty_pending_buckets, + discovered_expiry_items: metrics + .lifecycle_expiry + .current_queued + .saturating_add(metrics.lifecycle_expiry.current_active), + discovered_transition_items: metrics + .lifecycle_transition + .current_queued + .saturating_add(metrics.lifecycle_transition.current_active) + .saturating_add(metrics.lifecycle_transition.compensation_pending) + .saturating_add(metrics.lifecycle_transition.compensation_running), + } +} + +async fn wait_for_scanner_data_movement_resume( + ctx: &CancellationToken, + storeapi: &Arc, + guard: &NamespaceLockGuard, + pause_backlog: &mut ScannerPauseBacklogController, +) -> bool { + loop { + let observation = current_scanner_pause_backlog_observation(storeapi).await; + pause_backlog.observe(observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(storeapi, observation); + if !observation.paused { + return !ctx.is_cancelled() && !guard.is_lock_lost(); + } + + let movement_changed = storeapi.scanner_data_movement_changed(); + if storeapi.scanner_data_movement_generation() != observation.movement_generation { + continue; + } + tokio::select! { + _ = ctx.cancelled() => return false, + _ = guard.lock_lost_notified() => return false, + _ = movement_changed.notified() => {}, + _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {}, + } + } +} + +async fn finish_scanner_pause_backlog_cycle( + pause_backlog: &mut ScannerPauseBacklogController, + storeapi: &Arc, + attempt: ScannerPauseBacklogAttemptDecision, + outcome: ScannerCycleOutcome, +) { + let observation = current_scanner_pause_backlog_observation(storeapi).await; + if let ScannerPauseBacklogAttemptDecision::Tracked(serial) = attempt { + pause_backlog.finish_attempt(serial, outcome, observation).await; + } else { + pause_backlog.observe_cycle_outcome(outcome, observation).await; + } + #[cfg(test)] + notify_scanner_runtime_observed_for_test(storeapi, observation); +} + async fn run_data_scanner_with_maintenance_state( ctx: CancellationToken, storeapi: Arc, @@ -2269,6 +2475,28 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } }; + let pause_backlog_now = scanner_pause_backlog_now(); + let mut pause_backlog = match ScannerPauseBacklogController::claim(storeapi.clone(), pause_backlog_now).await { + Ok(controller) => controller, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "pause_backlog_claim_failed", + error = %err, + "Scanner pause backlog persistence is unavailable" + ); + ScannerPauseBacklogController::unavailable(storeapi.clone(), err, pause_backlog_now) + } + }; + if !wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await { + global_metrics().set_cycle(None).await; + return Ok(()); + } + #[cfg(test)] + notify_scanner_startup_observed_for_test().await; let single_disk = storeapi.setup_is_erasure_sd().await; let erasure = storeapi.setup_is_erasure().await; let distributed = storeapi.setup_is_dist_erasure().await; @@ -2416,6 +2644,19 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } if !leadership_claimed { + let observation = current_scanner_pause_backlog_observation(&storeapi).await; + pause_backlog.observe(observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(&storeapi, observation); + if observation.paused { + if wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await { + return Err(ScannerError::Other( + "scanner startup was fenced by data movement; retrying from durable state".to_string(), + )); + } + global_metrics().set_cycle(None).await; + return Ok(()); + } error!( target: "rustfs::scanner", event = EVENT_SCANNER_LOCK_STATE, @@ -2448,7 +2689,13 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } - if !ctx.is_cancelled() { + let initial_pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await; + if !ctx.is_cancelled() + && matches!( + initial_pause_backlog_attempt, + ScannerPauseBacklogAttemptDecision::Untracked | ScannerPauseBacklogAttemptDecision::Tracked(_) + ) + { // Preserve previous behavior: run one cycle immediately after lock acquisition. let dirty_generation_before_cycle = dirty_usage_generation(); let dirty_usage_pending_before_cycle = dirty_usage_buckets_pending(); @@ -2506,6 +2753,7 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } }; + finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, initial_pause_backlog_attempt, initial_outcome).await; superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; @@ -2558,6 +2806,10 @@ async fn run_data_scanner_with_maintenance_state( break; } + let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await; + pause_backlog.observe(pause_backlog_observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation); let runtime_config = resolve_scanner_runtime_config(); if clean_idle_topology_supported && scanner_clean_idle_backoff_configured(&runtime_config) { let current_generation = scanner_maintenance_generation(); @@ -2594,11 +2846,16 @@ async fn run_data_scanner_with_maintenance_state( scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for); let superseded_retry_interval = superseded_backoff.retry_interval(runtime_config.cycle_interval); let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval); - let convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval); + let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval); if let Some(retry_interval) = convergence_retry_interval { wait_plan.effective_interval = retry_interval; wait_plan.delay = randomized_cycle_delay_for(retry_interval).min(retry_interval); } + if let Some(pause_backlog_delay) = pause_backlog.scheduling_delay(scanner_pause_backlog_now()) { + wait_plan.effective_interval = pause_backlog_delay.max(Duration::from_secs(1)); + wait_plan.delay = pause_backlog_delay; + convergence_retry_interval = Some(pause_backlog_delay.max(Duration::from_secs(1))); + } let dirty_generation_before_wait = dirty_usage_generation(); let dirty_usage_pending_before_wait = dirty_usage_buckets_pending(); let maintenance_generation_before_wait = scanner_maintenance_generation(); @@ -2723,6 +2980,20 @@ async fn run_data_scanner_with_maintenance_state( record_scanner_leader_lock_lost("Scanner leader lock lost before starting the next cycle").await; break; } + let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await; + pause_backlog.observe(pause_backlog_observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation); + if pause_backlog_observation.paused { + continue; + } + let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await; + if matches!( + pause_backlog_attempt, + ScannerPauseBacklogAttemptDecision::RateLimited | ScannerPauseBacklogAttemptDecision::PersistenceUnavailable + ) { + continue; + } let dirty_generation_before_cycle = dirty_usage_generation(); let cycle_ctx = ctx.child_token(); let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config()); @@ -2771,6 +3042,7 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } }; + finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, pause_backlog_attempt, outcome).await; superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; @@ -3088,12 +3360,14 @@ fn data_usage_reintroduces_missing_bucket(incoming: &DataUsageInfo, existing: Op /// Store data usage info in backend. Will store all objects sent on the receiver until closed. mod activity; +mod backlog; mod cycle_state; mod heal_info; mod leadership; mod usage_store; use activity::*; +use backlog::*; use cycle_state::*; use leadership::*; use usage_store::*; @@ -3104,6 +3378,10 @@ pub(crate) use activity::{ scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements, }; pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; +pub use backlog::{ + ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, + scanner_pause_backlog_status, +}; #[cfg(test)] pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test; pub use cycle_state::{ diff --git a/crates/scanner/src/scanner/backlog.rs b/crates/scanner/src/scanner/backlog.rs new file mode 100644 index 000000000..2f160a42a --- /dev/null +++ b/crates/scanner/src/scanner/backlog.rs @@ -0,0 +1,2377 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Durable operational accounting for scanner pauses and bounded catch-up. +//! +//! This ledger is replicated outside the authoritative data-usage publication +//! path so it can advance while that path is fenced by data movement. It never +//! grants publication admission; scanner usage still passes the storage-owned +//! movement epoch and final publication fences. + +use super::ScannerCycleOutcome; +use crate::data_usage_define::DataUsageCacheRevision; +use crate::storage_api::owner::ObjectIO as _; +use crate::{ + BUCKET_META_PREFIX, ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerObjectOptions, SetDisks, save_config_with_preconditions, +}; +use futures::future::join_all; +use http::HeaderMap; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, LazyLock, RwLock}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::io::AsyncReadExt; + +const SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION: u16 = 1; +const SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION: u16 = 1; +const SCANNER_PAUSE_BACKLOG_OBJECT: &str = ".scanner-pause-backlog.json"; +const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024; +const SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS: u64 = 5 * 60; +const SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS: u64 = 5 * 60; +const SCANNER_CATCH_UP_WINDOW_SECONDS: u64 = 60 * 60; +const SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW: u32 = 4; +const SCANNER_CATCH_UP_FAILURE_LIMIT: u32 = 5; +const SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS: u64 = 60 * 60; +const SCANNER_PAUSE_DURATION_ALERT_SECONDS: u64 = 24 * 60 * 60; +const SCANNER_PAUSE_DEFERRED_CYCLES_ALERT: u64 = 3; +const SCANNER_PAUSE_BACKLOG_ITEMS_ALERT: u64 = 10_000; + +const METRIC_SCANNER_PAUSE_BACKLOG_PHASE: &str = "rustfs_scanner_pause_backlog_phase"; +const METRIC_SCANNER_PAUSE_BACKLOG_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_pause_backlog_pause_duration_seconds"; +const METRIC_SCANNER_PAUSE_BACKLOG_PENDING_WORK_ITEMS: &str = "rustfs_scanner_pause_backlog_pending_work_items"; +const METRIC_SCANNER_PAUSE_BACKLOG_CONSECUTIVE_FAILURES: &str = "rustfs_scanner_pause_backlog_consecutive_failures"; +const METRIC_SCANNER_PAUSE_BACKLOG_RATE_LIMITED: &str = "rustfs_scanner_pause_backlog_rate_limited"; +const METRIC_SCANNER_PAUSE_BACKLOG_RETRY_EXHAUSTED: &str = "rustfs_scanner_pause_backlog_retry_exhausted"; +const METRIC_SCANNER_PAUSE_BACKLOG_ALERTING: &str = "rustfs_scanner_pause_backlog_alerting"; +const METRIC_SCANNER_PAUSE_BACKLOG_REPLICA_DEGRADED: &str = "rustfs_scanner_pause_backlog_replica_degraded"; + +static SCANNER_PAUSE_BACKLOG_PATH: LazyLock = + LazyLock::new(|| format!("{BUCKET_META_PREFIX}/{SCANNER_PAUSE_BACKLOG_OBJECT}")); +static SCANNER_PAUSE_BACKLOG_RUNTIME_ERROR: LazyLock>> = LazyLock::new(|| RwLock::new(None)); + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScannerPauseBacklogPhase { + #[default] + Idle, + Paused, + CatchingUp, + RetryExhausted, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScannerPauseBacklogAlertReason { + PauseDurationThreshold, + DeferredCyclesThreshold, + BacklogItemsThreshold, + RetryBudgetExhausted, + CounterExhausted, + ReplicaDegraded, + PersistenceUnavailable, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +pub struct ScannerPauseBacklogThresholds { + pub pause_duration_seconds: u64, + pub deferred_cycles: u64, + pub backlog_work_items: u64, + pub catch_up_min_interval_seconds: u64, + pub catch_up_window_seconds: u64, + pub catch_up_max_attempts_per_window: u32, + pub catch_up_failure_limit: u32, + pub retry_exhausted_probe_seconds: u64, +} + +impl Default for ScannerPauseBacklogThresholds { + fn default() -> Self { + Self { + pause_duration_seconds: SCANNER_PAUSE_DURATION_ALERT_SECONDS, + deferred_cycles: SCANNER_PAUSE_DEFERRED_CYCLES_ALERT, + backlog_work_items: SCANNER_PAUSE_BACKLOG_ITEMS_ALERT, + catch_up_min_interval_seconds: SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS, + catch_up_window_seconds: SCANNER_CATCH_UP_WINDOW_SECONDS, + catch_up_max_attempts_per_window: SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW, + catch_up_failure_limit: SCANNER_CATCH_UP_FAILURE_LIMIT, + retry_exhausted_probe_seconds: SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS, + } + } +} + +#[derive(Clone, Debug, Default, Serialize)] +pub struct ScannerPauseBacklogStatus { + pub path: String, + pub persistence_state: String, + pub durable: bool, + pub schema_version: u16, + pub generation: u64, + pub writer_epoch: u64, + pub phase: ScannerPauseBacklogPhase, + pub movement_generation: u64, + pub movement_work_items: u64, + pub pause_started_at_unix_secs: u64, + pub pause_ended_at_unix_secs: u64, + pub pause_duration_seconds: u64, + pub last_updated_at_unix_secs: u64, + pub deferred_cycles: u64, + pub pending_full_scan: bool, + pub dirty_usage_buckets: u64, + pub discovered_expiry_items: u64, + pub discovered_transition_items: u64, + pub pending_work_items: u64, + pub catch_up_attempts: u64, + pub consecutive_failures: u32, + pub attempts_in_current_window: u32, + pub current_window_started_at_unix_secs: u64, + pub last_attempt_at_unix_secs: u64, + pub next_attempt_at_unix_secs: u64, + pub rate_limited: bool, + pub retry_exhausted: bool, + pub replica_count: usize, + pub healthy_replicas: usize, + pub stale_or_unavailable_replicas: usize, + pub alerting: bool, + pub alert_reasons: Vec, + pub thresholds: ScannerPauseBacklogThresholds, + pub error: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +struct ScannerPauseBacklogLedger { + schema_version: u16, + generation: u64, + writer_epoch: u64, + phase: ScannerPauseBacklogPhase, + movement_generation: u64, + movement_work_items: u64, + pause_started_at_unix_secs: u64, + pause_ended_at_unix_secs: u64, + last_updated_at_unix_secs: u64, + deferred_cycles: u64, + pending_full_scan: bool, + dirty_usage_buckets: u64, + discovered_expiry_items: u64, + discovered_transition_items: u64, + catch_up_attempts: u64, + consecutive_failures: u32, + current_window_started_at_unix_secs: u64, + attempts_in_current_window: u32, + last_attempt_at_unix_secs: u64, + next_attempt_at_unix_secs: u64, + current_attempt_serial: u64, + last_finished_attempt_serial: u64, + counter_exhausted: bool, +} + +impl Default for ScannerPauseBacklogLedger { + fn default() -> Self { + Self { + schema_version: SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION, + generation: 0, + writer_epoch: 0, + phase: ScannerPauseBacklogPhase::Idle, + movement_generation: 0, + movement_work_items: 0, + pause_started_at_unix_secs: 0, + pause_ended_at_unix_secs: 0, + last_updated_at_unix_secs: 0, + deferred_cycles: 0, + pending_full_scan: false, + dirty_usage_buckets: 0, + discovered_expiry_items: 0, + discovered_transition_items: 0, + catch_up_attempts: 0, + consecutive_failures: 0, + current_window_started_at_unix_secs: 0, + attempts_in_current_window: 0, + last_attempt_at_unix_secs: 0, + next_attempt_at_unix_secs: 0, + current_attempt_serial: 0, + last_finished_attempt_serial: 0, + counter_exhausted: false, + } + } +} + +impl ScannerPauseBacklogLedger { + fn validate(&self) -> Result<(), String> { + if self.schema_version != SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION { + return Err(format!("unsupported scanner pause backlog schema {}", self.schema_version)); + } + if self.generation == 0 || self.writer_epoch == 0 || self.last_updated_at_unix_secs == 0 { + return Err("scanner pause backlog has an invalid durable fence".to_string()); + } + if self.last_finished_attempt_serial > self.current_attempt_serial { + return Err("scanner pause backlog finished attempt exceeds the current attempt".to_string()); + } + if self.attempts_in_current_window > SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW { + return Err("scanner pause backlog rate window exceeds its attempt limit".to_string()); + } + if self.phase != ScannerPauseBacklogPhase::Idle && self.last_attempt_at_unix_secs > self.next_attempt_at_unix_secs { + return Err("scanner pause backlog next attempt precedes its last attempt".to_string()); + } + if self.phase == ScannerPauseBacklogPhase::Paused && !self.pending_full_scan { + return Err("scanner pause backlog lost its required post-pause scan".to_string()); + } + if self.phase == ScannerPauseBacklogPhase::Paused && self.pause_started_at_unix_secs == 0 { + return Err("scanner pause backlog has no pause start time".to_string()); + } + if matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) && (self.pause_started_at_unix_secs == 0 || self.pause_ended_at_unix_secs == 0) + { + return Err("scanner pause backlog has incomplete catch-up timestamps".to_string()); + } + if self.pause_ended_at_unix_secs != 0 && self.pause_ended_at_unix_secs < self.pause_started_at_unix_secs { + return Err("scanner pause backlog ends before its pause start".to_string()); + } + if self.phase == ScannerPauseBacklogPhase::Idle && self.pending_full_scan { + return Err("idle scanner pause backlog still requires a full scan".to_string()); + } + Ok(()) + } + + fn pending_work_items(&self) -> u64 { + (if self.pending_full_scan { 1_u64 } else { 0 }) + .saturating_add(self.dirty_usage_buckets) + .saturating_add(self.discovered_expiry_items) + .saturating_add(self.discovered_transition_items) + } + + fn pause_duration_seconds(&self, now: u64) -> u64 { + if self.pause_started_at_unix_secs == 0 { + return 0; + } + let end = if self.phase == ScannerPauseBacklogPhase::Paused { + now + } else { + self.pause_ended_at_unix_secs + }; + end.saturating_sub(self.pause_started_at_unix_secs) + } + + fn has_unfinished_attempt(&self) -> bool { + self.current_attempt_serial > self.last_finished_attempt_serial + } + + fn rate_limit_floor(&self) -> u64 { + let mut floor = self.next_attempt_at_unix_secs; + if self.last_attempt_at_unix_secs > 0 { + floor = floor.max( + self.last_attempt_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS), + ); + } + if self.current_window_started_at_unix_secs > 0 + && self.attempts_in_current_window >= SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW + { + floor = floor.max( + self.current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS), + ); + } + floor + } + + fn claim_writer(&mut self, now: u64) -> Result<(), String> { + if self.has_unfinished_attempt() { + self.last_finished_attempt_serial = self.current_attempt_serial; + if matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) { + increment_u32(&mut self.consecutive_failures, &mut self.counter_exhausted); + self.exhaust_retry_budget(now); + } + } + self.writer_epoch = self + .writer_epoch + .checked_add(1) + .ok_or_else(|| "scanner pause backlog writer epoch is exhausted".to_string())?; + Ok(()) + } + + fn apply_observation(&mut self, observation: ScannerPauseBacklogObservation) { + if !observation.paused + && observation.movement_generation != 0 + && observation.movement_generation != self.movement_generation + { + self.apply_observation(ScannerPauseBacklogObservation { + paused: true, + pause_started_at_unix_secs: observation.now_unix_secs, + ..observation + }); + self.apply_observation(observation); + return; + } + if !observation.paused && self.phase == ScannerPauseBacklogPhase::Idle { + self.movement_generation = observation.movement_generation; + self.movement_work_items = 0; + return; + } + self.movement_work_items = observation.movement_work_items; + self.dirty_usage_buckets = observation.dirty_usage_buckets; + self.discovered_expiry_items = observation.discovered_expiry_items; + self.discovered_transition_items = observation.discovered_transition_items; + + if observation.paused { + let new_pause = + self.phase != ScannerPauseBacklogPhase::Paused || self.movement_generation != observation.movement_generation; + if new_pause { + increment_u64(&mut self.deferred_cycles, &mut self.counter_exhausted); + self.pause_started_at_unix_secs = if observation.pause_started_at_unix_secs == 0 { + observation.now_unix_secs + } else { + observation.pause_started_at_unix_secs + }; + self.pause_ended_at_unix_secs = 0; + self.consecutive_failures = 0; + } else if observation.pause_started_at_unix_secs > 0 { + self.pause_started_at_unix_secs = self.pause_started_at_unix_secs.min(observation.pause_started_at_unix_secs); + } + self.phase = ScannerPauseBacklogPhase::Paused; + self.movement_generation = observation.movement_generation; + self.pending_full_scan = true; + self.next_attempt_at_unix_secs = self.rate_limit_floor(); + return; + } + + self.movement_generation = observation.movement_generation; + if self.phase == ScannerPauseBacklogPhase::Paused { + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.pause_ended_at_unix_secs = observation.now_unix_secs.max(self.pause_started_at_unix_secs); + self.pending_full_scan = true; + self.consecutive_failures = 0; + self.next_attempt_at_unix_secs = self.next_attempt_at_unix_secs.max(observation.now_unix_secs); + } else if matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) && !self.pending_full_scan + && self.pending_work_items() == 0 + { + self.phase = ScannerPauseBacklogPhase::Idle; + self.deferred_cycles = 0; + self.consecutive_failures = 0; + self.next_attempt_at_unix_secs = 0; + } + } + + fn begin_attempt(&mut self, now: u64) -> ScannerPauseBacklogAttemptDecision { + if !matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) { + return ScannerPauseBacklogAttemptDecision::Untracked; + } + if !self.pending_full_scan && self.pending_work_items() == 0 { + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + return ScannerPauseBacklogAttemptDecision::RateLimited; + } + if self.has_unfinished_attempt() || now < self.next_attempt_at_unix_secs { + return ScannerPauseBacklogAttemptDecision::RateLimited; + } + + if self.phase == ScannerPauseBacklogPhase::CatchingUp { + let window_end = self + .current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS); + if self.current_window_started_at_unix_secs == 0 || now >= window_end { + self.current_window_started_at_unix_secs = now; + self.attempts_in_current_window = 0; + } + if self.attempts_in_current_window >= SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW { + self.next_attempt_at_unix_secs = self + .current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS); + return ScannerPauseBacklogAttemptDecision::RateLimited; + } + } + + let Some(serial) = self.current_attempt_serial.checked_add(1) else { + self.counter_exhausted = true; + self.phase = ScannerPauseBacklogPhase::RetryExhausted; + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS); + return ScannerPauseBacklogAttemptDecision::RateLimited; + }; + self.current_attempt_serial = serial; + increment_u64(&mut self.catch_up_attempts, &mut self.counter_exhausted); + self.last_attempt_at_unix_secs = now; + + if self.phase == ScannerPauseBacklogPhase::CatchingUp { + increment_u32(&mut self.attempts_in_current_window, &mut self.counter_exhausted); + self.next_attempt_at_unix_secs = if self.attempts_in_current_window >= SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW { + self.current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS) + } else { + now.saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS) + }; + } else { + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS); + } + self.exhaust_retry_budget(now); + + ScannerPauseBacklogAttemptDecision::Tracked(serial) + } + + fn finish_attempt( + &mut self, + serial: u64, + outcome: ScannerPauseBacklogCycleOutcome, + observation: ScannerPauseBacklogObservation, + ) { + if serial == 0 || serial != self.current_attempt_serial || serial <= self.last_finished_attempt_serial { + return; + } + let movement_generation_advanced = + observation.movement_generation != 0 && observation.movement_generation != self.movement_generation; + let retry_exhausted_probe = self.phase == ScannerPauseBacklogPhase::RetryExhausted; + let successful_probe = matches!( + outcome, + ScannerPauseBacklogCycleOutcome::Completed + | ScannerPauseBacklogCycleOutcome::PendingMaintenance + | ScannerPauseBacklogCycleOutcome::Progressed + ); + let scheduled_retry_at = self.rate_limit_floor(); + let attempt_retry_at = if movement_generation_advanced || outcome == ScannerPauseBacklogCycleOutcome::DataMovementDeferred + { + scheduled_retry_at.max( + observation + .now_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS), + ) + } else { + scheduled_retry_at + }; + self.last_finished_attempt_serial = serial; + self.observe_cycle_outcome(outcome, observation); + let recovered_probe = retry_exhausted_probe && successful_probe && !self.counter_exhausted; + if self.phase == ScannerPauseBacklogPhase::Paused { + if recovered_probe { + self.consecutive_failures = 0; + self.current_window_started_at_unix_secs = observation.now_unix_secs; + self.attempts_in_current_window = 0; + self.next_attempt_at_unix_secs = observation + .now_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS); + } else { + self.next_attempt_at_unix_secs = self.next_attempt_at_unix_secs.max(attempt_retry_at); + } + return; + } + + match outcome { + ScannerPauseBacklogCycleOutcome::Completed => { + self.consecutive_failures = 0; + if movement_generation_advanced { + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.pending_full_scan = true; + } else { + self.pending_full_scan = false; + if self.pending_work_items() == 0 { + self.phase = ScannerPauseBacklogPhase::Idle; + self.deferred_cycles = 0; + self.next_attempt_at_unix_secs = 0; + } else { + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.next_attempt_at_unix_secs = observation + .now_unix_secs + .saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + } + } + } + ScannerPauseBacklogCycleOutcome::Progressed => { + self.consecutive_failures = 0; + self.phase = ScannerPauseBacklogPhase::CatchingUp; + } + ScannerPauseBacklogCycleOutcome::PendingMaintenance => { + self.consecutive_failures = 0; + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.pending_full_scan = true; + } + ScannerPauseBacklogCycleOutcome::DataMovementDeferred => {} + ScannerPauseBacklogCycleOutcome::RetryableFailure => { + increment_u32(&mut self.consecutive_failures, &mut self.counter_exhausted); + self.exhaust_retry_budget(observation.now_unix_secs); + } + } + if retry_exhausted_probe && !recovered_probe { + self.phase = ScannerPauseBacklogPhase::RetryExhausted; + } + if self.phase != ScannerPauseBacklogPhase::Idle { + if recovered_probe { + self.current_window_started_at_unix_secs = observation.now_unix_secs; + self.attempts_in_current_window = 0; + self.next_attempt_at_unix_secs = observation + .now_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS); + } else { + self.next_attempt_at_unix_secs = self.next_attempt_at_unix_secs.max(attempt_retry_at); + } + } + } + + fn observe_cycle_outcome(&mut self, outcome: ScannerPauseBacklogCycleOutcome, observation: ScannerPauseBacklogObservation) { + if outcome == ScannerPauseBacklogCycleOutcome::DataMovementDeferred && !observation.paused { + self.apply_observation(ScannerPauseBacklogObservation { + paused: true, + pause_started_at_unix_secs: observation.now_unix_secs, + ..observation + }); + } + self.apply_observation(observation); + } + + fn exhaust_retry_budget(&mut self, now: u64) { + if self.counter_exhausted || self.consecutive_failures >= SCANNER_CATCH_UP_FAILURE_LIMIT { + self.phase = ScannerPauseBacklogPhase::RetryExhausted; + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS); + } + } + + fn alert_reasons( + &self, + now: u64, + replica_degraded: bool, + persistence_unavailable: bool, + ) -> Vec { + let mut reasons = Vec::new(); + if self.phase != ScannerPauseBacklogPhase::Idle + && self.pause_duration_seconds(now) >= SCANNER_PAUSE_DURATION_ALERT_SECONDS + { + reasons.push(ScannerPauseBacklogAlertReason::PauseDurationThreshold); + } + if self.phase != ScannerPauseBacklogPhase::Idle && self.deferred_cycles >= SCANNER_PAUSE_DEFERRED_CYCLES_ALERT { + reasons.push(ScannerPauseBacklogAlertReason::DeferredCyclesThreshold); + } + if self.pending_work_items() >= SCANNER_PAUSE_BACKLOG_ITEMS_ALERT { + reasons.push(ScannerPauseBacklogAlertReason::BacklogItemsThreshold); + } + if self.phase == ScannerPauseBacklogPhase::RetryExhausted { + reasons.push(ScannerPauseBacklogAlertReason::RetryBudgetExhausted); + } + if self.counter_exhausted { + reasons.push(ScannerPauseBacklogAlertReason::CounterExhausted); + } + if replica_degraded { + reasons.push(ScannerPauseBacklogAlertReason::ReplicaDegraded); + } + if persistence_unavailable { + reasons.push(ScannerPauseBacklogAlertReason::PersistenceUnavailable); + } + reasons + } +} + +fn increment_u64(value: &mut u64, exhausted: &mut bool) { + if let Some(next) = value.checked_add(1) { + *value = next; + } else { + *exhausted = true; + } +} + +fn increment_u32(value: &mut u32, exhausted: &mut bool) { + if let Some(next) = value.checked_add(1) { + *value = next; + } else { + *exhausted = true; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ScannerPauseBacklogObservation { + pub(super) now_unix_secs: u64, + pub(super) paused: bool, + pub(super) movement_generation: u64, + pub(super) movement_work_items: u64, + pub(super) pause_started_at_unix_secs: u64, + pub(super) dirty_usage_buckets: u64, + pub(super) discovered_expiry_items: u64, + pub(super) discovered_transition_items: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ScannerPauseBacklogCycleOutcome { + Completed, + PendingMaintenance, + Progressed, + DataMovementDeferred, + RetryableFailure, +} + +impl From for ScannerPauseBacklogCycleOutcome { + fn from(outcome: ScannerCycleOutcome) -> Self { + match outcome { + ScannerCycleOutcome::Completed => Self::Completed, + ScannerCycleOutcome::CompletedWithPendingMaintenance => Self::PendingMaintenance, + ScannerCycleOutcome::Partial => Self::Progressed, + ScannerCycleOutcome::Deferred(super::ScannerCycleDeferReason::DataMovement) => Self::DataMovementDeferred, + ScannerCycleOutcome::Superseded | ScannerCycleOutcome::Deferred(_) | ScannerCycleOutcome::Failed => { + Self::RetryableFailure + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ScannerPauseBacklogAttemptDecision { + Untracked, + RateLimited, + Tracked(u64), + PersistenceUnavailable, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +struct ScannerPauseBacklogReplicaId { + pool_index: usize, + set_index: usize, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +struct ScannerPauseBacklogCommitRecord { + ledger: ScannerPauseBacklogLedger, + replicas: Vec, +} + +impl ScannerPauseBacklogCommitRecord { + fn new(ledger: ScannerPauseBacklogLedger, replicas: Vec) -> Self { + Self { ledger, replicas } + } + + fn validate(&self) -> Result<(), String> { + self.ledger.validate()?; + if self.replicas.is_empty() || !self.replicas.windows(2).all(|pair| pair[0] < pair[1]) { + return Err("scanner pause backlog commit has invalid replica membership".to_string()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +struct ScannerPauseBacklogReplicaRecord { + replica_schema_version: u16, + /// Last generation known to be safe without consulting commit records. + stable: Option, + /// Candidate authority only when every surviving set stores it exactly. + committed: Option, +} + +impl ScannerPauseBacklogReplicaRecord { + fn new(stable: Option, committed: Option) -> Self { + Self { + replica_schema_version: SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION, + stable, + committed, + } + } + + fn validate(&self) -> Result<(), String> { + if self.replica_schema_version != SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION { + return Err(format!( + "unsupported scanner pause backlog replica schema {}", + self.replica_schema_version + )); + } + if self.stable.is_none() && self.committed.is_none() { + return Err("scanner pause backlog replica has no durable state".to_string()); + } + if let Some(stable) = &self.stable { + stable.validate()?; + } + if let Some(committed) = &self.committed { + committed.validate()?; + if let Some(stable) = &self.stable { + let stable_key = (stable.writer_epoch, stable.generation); + let committed_key = (committed.ledger.writer_epoch, committed.ledger.generation); + if committed_key < stable_key || (committed_key == stable_key && &committed.ledger != stable) { + return Err("scanner pause backlog commit precedes or diverges from its stable generation".to_string()); + } + } + } + Ok(()) + } +} + +#[derive(Clone)] +enum ScannerPauseBacklogReplicaState { + Missing, + Valid(Box), + Invalid(String), + FutureSchema(u64), + Unavailable(String), +} + +#[derive(Clone)] +struct ScannerPauseBacklogReplica { + id: ScannerPauseBacklogReplicaId, + revision: Option, + state: ScannerPauseBacklogReplicaState, +} + +#[derive(Clone)] +struct LoadedScannerPauseBacklog { + ledger: ScannerPauseBacklogLedger, + durable: bool, + persistence_state: String, + replicas: Vec, + replica_count: usize, + healthy_replicas: usize, + stale_or_unavailable_replicas: usize, + stable_matches_ledger: bool, + authoritative_commit: Option, + requires_reload: bool, +} + +impl LoadedScannerPauseBacklog { + fn status(&self, now: u64, error: Option) -> ScannerPauseBacklogStatus { + let persistence_unavailable = error.is_some(); + let replica_degraded = self.durable && self.stale_or_unavailable_replicas > 0; + status_from_ledger( + &self.ledger, + now, + self.persistence_state.clone(), + self.durable, + self.replica_count, + self.healthy_replicas, + self.stale_or_unavailable_replicas, + replica_degraded, + persistence_unavailable, + error, + ) + } +} + +#[allow(clippy::too_many_arguments)] +fn status_from_ledger( + ledger: &ScannerPauseBacklogLedger, + now: u64, + persistence_state: String, + durable: bool, + replica_count: usize, + healthy_replicas: usize, + stale_or_unavailable_replicas: usize, + replica_degraded: bool, + persistence_unavailable: bool, + error: Option, +) -> ScannerPauseBacklogStatus { + let alert_reasons = ledger.alert_reasons(now, replica_degraded, persistence_unavailable); + let status = ScannerPauseBacklogStatus { + path: SCANNER_PAUSE_BACKLOG_PATH.clone(), + persistence_state, + durable, + schema_version: ledger.schema_version, + generation: ledger.generation, + writer_epoch: ledger.writer_epoch, + phase: ledger.phase, + movement_generation: ledger.movement_generation, + movement_work_items: ledger.movement_work_items, + pause_started_at_unix_secs: ledger.pause_started_at_unix_secs, + pause_ended_at_unix_secs: ledger.pause_ended_at_unix_secs, + pause_duration_seconds: ledger.pause_duration_seconds(now), + last_updated_at_unix_secs: ledger.last_updated_at_unix_secs, + deferred_cycles: ledger.deferred_cycles, + pending_full_scan: ledger.pending_full_scan, + dirty_usage_buckets: ledger.dirty_usage_buckets, + discovered_expiry_items: ledger.discovered_expiry_items, + discovered_transition_items: ledger.discovered_transition_items, + pending_work_items: ledger.pending_work_items(), + catch_up_attempts: ledger.catch_up_attempts, + consecutive_failures: ledger.consecutive_failures, + attempts_in_current_window: ledger.attempts_in_current_window, + current_window_started_at_unix_secs: ledger.current_window_started_at_unix_secs, + last_attempt_at_unix_secs: ledger.last_attempt_at_unix_secs, + next_attempt_at_unix_secs: ledger.next_attempt_at_unix_secs, + rate_limited: ledger.has_unfinished_attempt() + || matches!( + ledger.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) && now < ledger.next_attempt_at_unix_secs, + retry_exhausted: ledger.phase == ScannerPauseBacklogPhase::RetryExhausted, + replica_count, + healthy_replicas, + stale_or_unavailable_replicas, + alerting: !alert_reasons.is_empty(), + alert_reasons, + thresholds: ScannerPauseBacklogThresholds::default(), + error, + }; + record_scanner_pause_backlog_status(&status); + status +} + +fn record_scanner_pause_backlog_status(status: &ScannerPauseBacklogStatus) { + let phase: u32 = match status.phase { + ScannerPauseBacklogPhase::Idle => 0, + ScannerPauseBacklogPhase::Paused => 1, + ScannerPauseBacklogPhase::CatchingUp => 2, + ScannerPauseBacklogPhase::RetryExhausted => 3, + }; + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_PHASE).set(f64::from(phase)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_PAUSE_DURATION_SECONDS).set(metric_u64(status.pause_duration_seconds)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_PENDING_WORK_ITEMS).set(metric_u64(status.pending_work_items)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_CONSECUTIVE_FAILURES).set(f64::from(status.consecutive_failures)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_RATE_LIMITED).set(bool_metric(status.rate_limited)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_RETRY_EXHAUSTED).set(bool_metric(status.retry_exhausted)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_ALERTING).set(bool_metric(status.alerting)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_REPLICA_DEGRADED) + .set(bool_metric(status.durable && status.stale_or_unavailable_replicas > 0)); +} + +fn metric_u64(value: u64) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +fn bool_metric(value: bool) -> f64 { + if value { 1.0 } else { 0.0 } +} + +fn unix_now() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() +} + +async fn read_scanner_pause_backlog_replica(store: Arc) -> ScannerPauseBacklogReplica { + let id = ScannerPauseBacklogReplicaId { + pool_index: store.pool_index, + set_index: store.set_index, + }; + let reader = match store + .get_object_reader( + RUSTFS_META_BUCKET, + SCANNER_PAUSE_BACKLOG_PATH.as_str(), + None, + HeaderMap::new(), + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader, + Err( + EcstoreError::ConfigNotFound + | EcstoreError::FileNotFound + | EcstoreError::VolumeNotFound + | EcstoreError::ObjectNotFound(_, _) + | EcstoreError::BucketNotFound(_), + ) => { + return ScannerPauseBacklogReplica { + id, + revision: Some(DataUsageCacheRevision::Missing), + state: ScannerPauseBacklogReplicaState::Missing, + }; + } + Err(err) => { + return ScannerPauseBacklogReplica { + id, + revision: None, + state: ScannerPauseBacklogReplicaState::Unavailable(err.to_string()), + }; + } + }; + + let revision = reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .map(DataUsageCacheRevision::Etag); + let max_size = i64::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(i64::MAX); + if revision.is_none() || reader.object_info.is_dir || reader.object_info.size < 0 || reader.object_info.size > max_size { + return ScannerPauseBacklogReplica { + id, + revision, + state: ScannerPauseBacklogReplicaState::Invalid( + "scanner pause backlog replica is oversized or has no revision".to_string(), + ), + }; + } + + let mut data = Vec::new(); + let max_len = usize::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(usize::MAX); + let read_result = reader + .take(MAX_SCANNER_PAUSE_BACKLOG_BYTES.saturating_add(1)) + .read_to_end(&mut data) + .await; + let state = match read_result { + Err(err) => ScannerPauseBacklogReplicaState::Unavailable(err.to_string()), + Ok(_) if data.len() > max_len => { + ScannerPauseBacklogReplicaState::Invalid("scanner pause backlog replica exceeds its size bound".to_string()) + } + Ok(_) => decode_scanner_pause_backlog_ledger(&data), + }; + ScannerPauseBacklogReplica { id, revision, state } +} + +fn decode_scanner_pause_backlog_ledger(data: &[u8]) -> ScannerPauseBacklogReplicaState { + let value = match serde_json::from_slice::(data) { + Ok(value) => value, + Err(err) => return ScannerPauseBacklogReplicaState::Invalid(err.to_string()), + }; + if let Some(version) = value.get("replica_schema_version").and_then(serde_json::Value::as_u64) { + if version > u64::from(SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION) { + return ScannerPauseBacklogReplicaState::FutureSchema(version); + } + let record = match serde_json::from_value::(value) { + Ok(record) => record, + Err(err) => return ScannerPauseBacklogReplicaState::Invalid(err.to_string()), + }; + return match record.validate() { + Ok(()) => ScannerPauseBacklogReplicaState::Valid(Box::new(record)), + Err(err) => ScannerPauseBacklogReplicaState::Invalid(err), + }; + } + + let version = value.get("schema_version").and_then(serde_json::Value::as_u64).unwrap_or(0); + if version > u64::from(SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION) { + return ScannerPauseBacklogReplicaState::FutureSchema(version); + } + let ledger = match serde_json::from_value::(value) { + Ok(ledger) => ledger, + Err(err) => return ScannerPauseBacklogReplicaState::Invalid(err.to_string()), + }; + match ledger.validate() { + Ok(()) => ScannerPauseBacklogReplicaState::Valid(Box::new(ScannerPauseBacklogReplicaRecord::new(Some(ledger), None))), + Err(err) => ScannerPauseBacklogReplicaState::Invalid(err), + } +} + +fn scanner_pause_backlog_consensus<'a, T: Clone + PartialEq + 'a>( + values: impl Iterator>, +) -> Result, ()> { + let mut values = values; + let first = values.next().ok_or(())?; + if values.all(|candidate| candidate == first) { + Ok(first.cloned()) + } else { + Err(()) + } +} + +fn scanner_pause_backlog_replica_ids(replicas: &[ScannerPauseBacklogReplica]) -> Vec { + let mut ids = replicas.iter().map(|replica| replica.id).collect::>(); + ids.sort_unstable(); + ids +} + +fn select_scanner_pause_backlog_commit( + replicas: &[ScannerPauseBacklogReplica], + replica_ids: &[ScannerPauseBacklogReplicaId], +) -> Result, String> { + let current_ids = replica_ids.iter().copied().collect::>(); + let replicas_by_id = replicas + .iter() + .map(|replica| (replica.id, replica)) + .collect::>(); + let mut valid = Vec::::new(); + + for committed in replicas.iter().filter_map(|replica| match &replica.state { + ScannerPauseBacklogReplicaState::Valid(record) => record.committed.as_ref(), + _ => None, + }) { + if valid.contains(committed) + || !committed.replicas.iter().all(|id| { + current_ids.contains(id) + && replicas_by_id.get(id).is_some_and(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(record) + if record.committed.as_ref() == Some(committed) + ) + }) + }) + { + continue; + } + valid.push(committed.clone()); + } + + let Some(max_membership_len) = valid.iter().map(|committed| committed.replicas.len()).max() else { + return Ok(None); + }; + let mut largest = valid + .into_iter() + .filter(|committed| committed.replicas.len() == max_membership_len); + let Some(mut selected) = largest.next() else { + return Ok(None); + }; + for committed in largest { + if committed.ledger != selected.ledger { + return Err("scanner pause backlog has conflicting maximum-membership commit proofs".to_string()); + } + if committed.replicas < selected.replicas { + selected = committed; + } + } + Ok(Some(selected)) +} + +fn select_scanner_pause_backlog_replicas(replicas: Vec) -> Result { + if replicas.is_empty() { + return Err("scanner pause backlog has no storage replicas".to_string()); + } + for replica in &replicas { + if let ScannerPauseBacklogReplicaState::FutureSchema(version) = &replica.state { + return Err(format!( + "scanner pause backlog pool {} set {} uses future schema {version}", + replica.id.pool_index, replica.id.set_index + )); + } + } + + let replica_ids = scanner_pause_backlog_replica_ids(&replicas); + let authoritative_commit = select_scanner_pause_backlog_commit(&replicas, &replica_ids)?; + let stable_consensus = if replicas.iter().all(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(_) | ScannerPauseBacklogReplicaState::Missing + ) + }) { + scanner_pause_backlog_consensus(replicas.iter().map(|replica| match &replica.state { + ScannerPauseBacklogReplicaState::Valid(record) => record.stable.as_ref(), + ScannerPauseBacklogReplicaState::Missing => None, + _ => unreachable!("replica states were checked above"), + })) + } else { + Err(()) + }; + + let selected = match &authoritative_commit { + Some(committed) => Some(committed.ledger.clone()), + None => { + if let Some(replica) = replicas.iter().find(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Invalid(_) | ScannerPauseBacklogReplicaState::Unavailable(_) + ) + }) { + let reason = match &replica.state { + ScannerPauseBacklogReplicaState::Invalid(reason) | ScannerPauseBacklogReplicaState::Unavailable(reason) => { + reason + } + _ => unreachable!("replica state was checked above"), + }; + return Err(format!( + "scanner pause backlog pool {} set {} is unavailable: {reason}", + replica.id.pool_index, replica.id.set_index + )); + } + match &stable_consensus { + Ok(stable) => stable.clone(), + Err(()) => { + return Err( + "scanner pause backlog has neither a surviving membership commit nor a stable rollback point".to_string(), + ); + } + } + } + }; + let Some(selected) = selected else { + let replica_count = replicas.len(); + return Ok(LoadedScannerPauseBacklog { + ledger: ScannerPauseBacklogLedger::default(), + durable: false, + persistence_state: "missing".to_string(), + replica_count, + healthy_replicas: 0, + stale_or_unavailable_replicas: replica_count, + stable_matches_ledger: true, + authoritative_commit: None, + requires_reload: false, + replicas, + }); + }; + + let stable_matches_ledger = matches!(&stable_consensus, Ok(Some(stable)) if stable == &selected); + let healthy_replicas = replicas + .iter() + .filter(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(record) + if record.stable.as_ref() == Some(&selected) + && match &authoritative_commit { + Some(committed) => record.committed.as_ref() == Some(committed), + None => record.committed.is_none(), + } + ) + }) + .count(); + let replica_count = replicas.len(); + let stale_or_unavailable_replicas = replica_count.saturating_sub(healthy_replicas); + let membership_repair_pending = authoritative_commit + .as_ref() + .is_some_and(|committed| committed.replicas.as_slice() != replica_ids.as_slice()); + Ok(LoadedScannerPauseBacklog { + ledger: selected, + durable: true, + persistence_state: if membership_repair_pending { + "membership_repair_pending" + } else if authoritative_commit.is_some() && !stable_matches_ledger { + "committed_pending_stabilization" + } else if stale_or_unavailable_replicas > 0 { + "rolled_back_partial_commit" + } else { + "healthy" + } + .to_string(), + replica_count, + replicas, + healthy_replicas, + stale_or_unavailable_replicas, + stable_matches_ledger, + authoritative_commit, + requires_reload: false, + }) +} + +async fn load_scanner_pause_backlog(storeapi: Arc) -> Result { + let writable = storeapi.scanner_pause_backlog_writable_set_disks().await; + if writable.is_empty() { + return Err("scanner pause backlog has no surviving storage replicas".to_string()); + } + let replicas = join_all(writable.into_iter().map(read_scanner_pause_backlog_replica)).await; + select_scanner_pause_backlog_replicas(replicas) +} + +async fn write_scanner_pause_backlog_record( + storeapi: Arc, + loaded: &LoadedScannerPauseBacklog, + record: ScannerPauseBacklogReplicaRecord, +) -> Result<(), String> { + let data = serde_json::to_vec(&record).map_err(|err| format!("failed to encode scanner pause backlog: {err}"))?; + if data.len() > usize::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(usize::MAX) { + return Err("scanner pause backlog exceeds its size bound".to_string()); + } + + let writable = storeapi.scanner_pause_backlog_writable_set_disks().await; + if writable.is_empty() { + return Err("scanner pause backlog has no surviving writable set".to_string()); + } + let loaded_ids = loaded.replicas.iter().map(|replica| replica.id).collect::>(); + let writable_ids = writable + .iter() + .map(|set| ScannerPauseBacklogReplicaId { + pool_index: set.pool_index, + set_index: set.set_index, + }) + .collect::>(); + if writable_ids != loaded_ids { + return Err("scanner pause backlog replica topology changed during commit".to_string()); + } + + let revisions = loaded + .replicas + .iter() + .filter_map(|replica| replica.revision.clone().map(|revision| (replica.id, revision))) + .collect::>(); + let results = join_all(writable.into_iter().map(|set| { + let id = ScannerPauseBacklogReplicaId { + pool_index: set.pool_index, + set_index: set.set_index, + }; + let revision = revisions.get(&id).cloned(); + let data = data.clone(); + async move { + let Some(revision) = revision else { + return (id, Err("replica revision is unavailable".to_string())); + }; + let result = save_config_with_preconditions(set, SCANNER_PAUSE_BACKLOG_PATH.as_str(), data, revision.preconditions()) + .await + .map(|_| ()) + .map_err(|err| err.to_string()); + (id, result) + } + })) + .await; + + let failures = results + .iter() + .filter_map(|(id, result)| { + result + .as_ref() + .err() + .map(|err| format!("pool {} set {}: {err}", id.pool_index, id.set_index)) + }) + .collect::>(); + if !failures.is_empty() { + return Err(format!( + "scanner pause backlog commit did not reach every surviving set ({})", + failures.join("; ") + )); + } + Ok(()) +} + +async fn stabilize_scanner_pause_backlog( + storeapi: Arc, + loaded: &LoadedScannerPauseBacklog, +) -> Result { + let committed = loaded.authoritative_commit.clone().unwrap_or_else(|| { + ScannerPauseBacklogCommitRecord::new(loaded.ledger.clone(), scanner_pause_backlog_replica_ids(&loaded.replicas)) + }); + let record = ScannerPauseBacklogReplicaRecord::new(Some(loaded.ledger.clone()), Some(committed)); + write_scanner_pause_backlog_record(storeapi.clone(), loaded, record).await?; + let stabilized = load_scanner_pause_backlog(storeapi).await?; + if stabilized.ledger != loaded.ledger || !stabilized.stable_matches_ledger { + return Err("scanner pause backlog failed to stabilize its last committed generation".to_string()); + } + Ok(stabilized) +} + +fn committed_scanner_pause_backlog_pending_reload( + ledger: ScannerPauseBacklogLedger, + replica_count: usize, +) -> LoadedScannerPauseBacklog { + LoadedScannerPauseBacklog { + ledger, + durable: true, + persistence_state: "committed_reload_pending".to_string(), + replicas: Vec::new(), + replica_count, + healthy_replicas: 0, + stale_or_unavailable_replicas: replica_count, + stable_matches_ledger: false, + authoritative_commit: None, + requires_reload: true, + } +} + +async fn persist_scanner_pause_backlog( + storeapi: Arc, + loaded: &LoadedScannerPauseBacklog, + ledger: ScannerPauseBacklogLedger, +) -> Result { + let mut base = if loaded.requires_reload { + let reloaded = load_scanner_pause_backlog(storeapi.clone()).await?; + if reloaded.ledger != loaded.ledger { + return Err(format!( + "scanner pause backlog reload advanced to writer epoch {} generation {}", + reloaded.ledger.writer_epoch, reloaded.ledger.generation + )); + } + reloaded + } else { + loaded.clone() + }; + if base.durable && !base.stable_matches_ledger { + base = stabilize_scanner_pause_backlog(storeapi.clone(), &base).await?; + } + + let replicas = scanner_pause_backlog_replica_ids(&base.replicas); + let committed = ScannerPauseBacklogCommitRecord::new(ledger.clone(), replicas); + let record = ScannerPauseBacklogReplicaRecord::new(base.durable.then_some(base.ledger.clone()), Some(committed)); + write_scanner_pause_backlog_record(storeapi.clone(), &base, record).await?; + + match load_scanner_pause_backlog(storeapi.clone()).await { + Ok(committed) if committed.ledger == ledger => match stabilize_scanner_pause_backlog(storeapi, &committed).await { + Ok(stabilized) => Ok(stabilized), + Err(_) => Ok(committed_scanner_pause_backlog_pending_reload(ledger, base.replica_count)), + }, + Ok(_) | Err(_) => Ok(committed_scanner_pause_backlog_pending_reload(ledger, base.replica_count)), + } +} + +pub(super) struct ScannerPauseBacklogController { + storeapi: Arc, + loaded: LoadedScannerPauseBacklog, + persistence_disabled: bool, + persistence_retry_at_unix_secs: u64, +} + +impl ScannerPauseBacklogController { + pub(super) async fn claim(storeapi: Arc, now: u64) -> Result { + let loaded = load_scanner_pause_backlog(storeapi.clone()).await?; + let mut ledger = loaded.ledger.clone(); + ledger.claim_writer(now)?; + prepare_scanner_pause_backlog_persist(&mut ledger, now)?; + let loaded = persist_scanner_pause_backlog(storeapi.clone(), &loaded, ledger).await?; + set_runtime_error(None); + let controller = Self { + storeapi, + loaded, + persistence_disabled: false, + persistence_retry_at_unix_secs: 0, + }; + controller.record_status(now); + Ok(controller) + } + + pub(super) fn unavailable(storeapi: Arc, error: String, now: u64) -> Self { + set_runtime_error(Some(error)); + let loaded = LoadedScannerPauseBacklog { + ledger: ScannerPauseBacklogLedger::default(), + durable: false, + persistence_state: "unavailable".to_string(), + replicas: Vec::new(), + replica_count: 0, + healthy_replicas: 0, + stale_or_unavailable_replicas: 0, + stable_matches_ledger: true, + authoritative_commit: None, + requires_reload: false, + }; + let controller = Self { + storeapi, + loaded, + persistence_disabled: true, + persistence_retry_at_unix_secs: now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS), + }; + controller.record_status(now); + controller + } + + pub(super) fn scheduling_delay(&self, now: u64) -> Option { + if self.persistence_disabled { + return Some(Duration::from_secs(self.persistence_retry_at_unix_secs.saturating_sub(now))); + } + match self.loaded.ledger.phase { + ScannerPauseBacklogPhase::Idle => None, + ScannerPauseBacklogPhase::Paused => Some(Duration::from_secs(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS)), + ScannerPauseBacklogPhase::CatchingUp => { + Some(Duration::from_secs(self.loaded.ledger.next_attempt_at_unix_secs.saturating_sub(now))) + } + ScannerPauseBacklogPhase::RetryExhausted => { + Some(Duration::from_secs(self.loaded.ledger.next_attempt_at_unix_secs.saturating_sub(now))) + } + } + } + + pub(super) async fn observe(&mut self, observation: ScannerPauseBacklogObservation) { + if !self.try_recover_persistence(observation.now_unix_secs).await { + return; + } + self.persist_mutation(observation.now_unix_secs, |ledger| ledger.apply_observation(observation)) + .await; + } + + pub(super) async fn begin_attempt(&mut self, now: u64) -> ScannerPauseBacklogAttemptDecision { + if self.persistence_disabled { + return ScannerPauseBacklogAttemptDecision::PersistenceUnavailable; + } + let mut candidate = self.loaded.ledger.clone(); + let decision = candidate.begin_attempt(now); + if candidate == self.loaded.ledger { + self.record_status(now); + return decision; + } + if let Err(err) = prepare_scanner_pause_backlog_persist(&mut candidate, now) { + self.disable_persistence(err, now); + return ScannerPauseBacklogAttemptDecision::PersistenceUnavailable; + } + match persist_scanner_pause_backlog(self.storeapi.clone(), &self.loaded, candidate).await { + Ok(loaded) => { + self.loaded = loaded; + set_runtime_error(None); + self.record_status(now); + decision + } + Err(err) => { + self.disable_persistence(err, now); + ScannerPauseBacklogAttemptDecision::PersistenceUnavailable + } + } + } + + pub(super) async fn finish_attempt( + &mut self, + serial: u64, + outcome: ScannerCycleOutcome, + observation: ScannerPauseBacklogObservation, + ) { + self.persist_mutation(observation.now_unix_secs, |ledger| { + ledger.finish_attempt(serial, outcome.into(), observation) + }) + .await; + } + + pub(super) async fn observe_cycle_outcome( + &mut self, + outcome: ScannerCycleOutcome, + observation: ScannerPauseBacklogObservation, + ) { + self.persist_mutation(observation.now_unix_secs, |ledger| { + ledger.observe_cycle_outcome(outcome.into(), observation) + }) + .await; + } + + async fn persist_mutation(&mut self, now: u64, mutate: impl FnOnce(&mut ScannerPauseBacklogLedger)) { + if self.persistence_disabled { + return; + } + let mut candidate = self.loaded.ledger.clone(); + mutate(&mut candidate); + if candidate == self.loaded.ledger { + self.record_status(now); + return; + } + if let Err(err) = prepare_scanner_pause_backlog_persist(&mut candidate, now) { + self.disable_persistence(err, now); + return; + } + match persist_scanner_pause_backlog(self.storeapi.clone(), &self.loaded, candidate).await { + Ok(loaded) => { + self.loaded = loaded; + set_runtime_error(None); + self.record_status(now); + } + Err(err) => self.disable_persistence(err, now), + } + } + + fn disable_persistence(&mut self, error: String, now: u64) { + self.persistence_disabled = true; + self.persistence_retry_at_unix_secs = now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + set_runtime_error(Some(error)); + self.record_status(now); + } + + async fn try_recover_persistence(&mut self, now: u64) -> bool { + if !self.persistence_disabled { + return true; + } + if now < self.persistence_retry_at_unix_secs { + self.record_status(now); + return false; + } + match Self::claim(self.storeapi.clone(), now).await { + Ok(controller) => { + *self = controller; + true + } + Err(error) => { + self.persistence_retry_at_unix_secs = now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + set_runtime_error(Some(error)); + self.record_status(now); + false + } + } + } + + fn record_status(&self, now: u64) { + let _ = self.loaded.status(now, runtime_error()); + } +} + +fn prepare_scanner_pause_backlog_persist(ledger: &mut ScannerPauseBacklogLedger, now: u64) -> Result<(), String> { + ledger.generation = ledger + .generation + .checked_add(1) + .ok_or_else(|| "scanner pause backlog generation is exhausted".to_string())?; + ledger.last_updated_at_unix_secs = now; + ledger.validate() +} + +fn set_runtime_error(error: Option) { + *SCANNER_PAUSE_BACKLOG_RUNTIME_ERROR + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = error; +} + +fn runtime_error() -> Option { + SCANNER_PAUSE_BACKLOG_RUNTIME_ERROR + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() +} + +pub async fn scanner_pause_backlog_status(storeapi: Arc) -> ScannerPauseBacklogStatus { + let now = unix_now(); + match load_scanner_pause_backlog(storeapi).await { + Ok(loaded) => loaded.status(now, runtime_error()), + Err(error) => { + let error = runtime_error().map_or(error.clone(), |runtime| format!("{error}; {runtime}")); + status_from_ledger( + &ScannerPauseBacklogLedger::default(), + now, + "unavailable".to_string(), + false, + 0, + 0, + 0, + false, + true, + Some(error), + ) + } + } +} + +pub(super) fn scanner_pause_backlog_now() -> u64 { + unix_now() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn observation(now: u64, paused: bool, pending: u64) -> ScannerPauseBacklogObservation { + ScannerPauseBacklogObservation { + now_unix_secs: now, + paused, + movement_generation: 7, + movement_work_items: if paused { 1 } else { 0 }, + pause_started_at_unix_secs: if paused { now } else { 0 }, + dirty_usage_buckets: pending, + discovered_expiry_items: 0, + discovered_transition_items: 0, + } + } + + fn durable_ledger(now: u64) -> ScannerPauseBacklogLedger { + let mut ledger = ScannerPauseBacklogLedger::default(); + ledger.claim_writer(now).expect("writer epoch should be available"); + prepare_scanner_pause_backlog_persist(&mut ledger, now).expect("ledger should become durable"); + ledger + } + + fn decode_valid_ledger(ledger: &ScannerPauseBacklogLedger) -> ScannerPauseBacklogLedger { + let encoded = serde_json::to_vec(ledger).expect("ledger should encode"); + let ScannerPauseBacklogReplicaState::Valid(decoded) = decode_scanner_pause_backlog_ledger(&encoded) else { + panic!("ledger should decode"); + }; + decoded.stable.expect("legacy ledger should become a stable replica") + } + + fn retry_exhausted_ledger() -> ScannerPauseBacklogLedger { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + for now in [120, 420, 720, 1020, 3720] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("failure attempt at {now} should be admitted"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::RetryableFailure, observation(now + 1, false, 0)); + } + ledger + } + + fn replica_id(pool_index: usize, set_index: usize) -> ScannerPauseBacklogReplicaId { + ScannerPauseBacklogReplicaId { pool_index, set_index } + } + + fn decoded_replica( + id: ScannerPauseBacklogReplicaId, + record: &ScannerPauseBacklogReplicaRecord, + ) -> ScannerPauseBacklogReplica { + let encoded = serde_json::to_vec(record).expect("replica should encode"); + ScannerPauseBacklogReplica { + id, + revision: Some(DataUsageCacheRevision::Etag(format!("revision-{}-{}", id.pool_index, id.set_index))), + state: decode_scanner_pause_backlog_ledger(&encoded), + } + } + + fn crash_reload_replicas(replicas: Vec) -> LoadedScannerPauseBacklog { + select_scanner_pause_backlog_replicas(replicas).expect("replicas should have an authoritative rollback point") + } + + fn crash_reload(records: &[ScannerPauseBacklogReplicaRecord]) -> LoadedScannerPauseBacklog { + let replicas = records + .iter() + .enumerate() + .map(|(set_index, record)| decoded_replica(replica_id(0, set_index), record)) + .collect(); + crash_reload_replicas(replicas) + } + + fn replica_record_for_members( + stable: &ScannerPauseBacklogLedger, + committed: &ScannerPauseBacklogLedger, + replicas: &[ScannerPauseBacklogReplicaId], + ) -> ScannerPauseBacklogReplicaRecord { + ScannerPauseBacklogReplicaRecord::new( + Some(stable.clone()), + Some(ScannerPauseBacklogCommitRecord::new(committed.clone(), replicas.to_vec())), + ) + } + + fn replica_record( + stable: &ScannerPauseBacklogLedger, + committed: &ScannerPauseBacklogLedger, + ) -> ScannerPauseBacklogReplicaRecord { + let replicas = (0..3) + .map(|set_index| ScannerPauseBacklogReplicaId { + pool_index: 0, + set_index, + }) + .collect::>(); + replica_record_for_members(stable, committed, &replicas) + } + + #[derive(Clone, Copy)] + enum RejoinedSourceState { + Missing, + OlderCommit, + NewerSmallCommit, + NewerUnprovenCommit, + StaleStable, + } + + fn assert_rejoined_source_is_safely_seeded(source_state: RejoinedSourceState) { + let source_id = replica_id(0, 0); + let target_ids = [replica_id(1, 0), replica_id(1, 1)]; + let full_ids = [source_id, target_ids[0], target_ids[1]]; + let old = durable_ledger(50); + let mut target = old.clone(); + target.claim_writer(100).expect("target membership epoch should advance"); + prepare_scanner_pause_backlog_persist(&mut target, 100).expect("target membership should persist"); + let target_record = replica_record_for_members(&target, &target, &target_ids); + let source = match source_state { + RejoinedSourceState::Missing => ScannerPauseBacklogReplica { + id: source_id, + revision: Some(DataUsageCacheRevision::Missing), + state: ScannerPauseBacklogReplicaState::Missing, + }, + RejoinedSourceState::OlderCommit => { + let stale_record = replica_record_for_members(&old, &old, &[source_id]); + decoded_replica(source_id, &stale_record) + } + RejoinedSourceState::NewerSmallCommit => { + let mut stale = target.clone(); + stale.claim_writer(110).expect("stale source epoch should advance"); + prepare_scanner_pause_backlog_persist(&mut stale, 110).expect("stale source should persist"); + stale.claim_writer(120).expect("stale source epoch should advance again"); + prepare_scanner_pause_backlog_persist(&mut stale, 120).expect("stale source should persist again"); + assert!(stale.writer_epoch > target.writer_epoch); + let stale_record = replica_record_for_members(&stale, &stale, &[source_id]); + decoded_replica(source_id, &stale_record) + } + RejoinedSourceState::NewerUnprovenCommit => { + let mut stale = target.clone(); + stale.claim_writer(110).expect("stale source epoch should advance"); + prepare_scanner_pause_backlog_persist(&mut stale, 110).expect("stale source should persist"); + stale.claim_writer(120).expect("stale source epoch should advance again"); + prepare_scanner_pause_backlog_persist(&mut stale, 120).expect("stale source should persist again"); + assert!(stale.writer_epoch > target.writer_epoch); + let stale_record = replica_record_for_members(&stale, &stale, &full_ids); + decoded_replica(source_id, &stale_record) + } + RejoinedSourceState::StaleStable => { + let stale_record = ScannerPauseBacklogReplicaRecord::new(Some(old), None); + decoded_replica(source_id, &stale_record) + } + }; + + let active = crash_reload_replicas(target_ids.iter().map(|id| decoded_replica(*id, &target_record)).collect()); + assert_eq!(active.ledger, target); + assert_eq!(active.persistence_state, "healthy"); + + let rejoined = crash_reload_replicas(vec![ + source, + decoded_replica(target_ids[0], &target_record), + decoded_replica(target_ids[1], &target_record), + ]); + assert_eq!(rejoined.ledger, target); + assert_eq!(rejoined.persistence_state, "membership_repair_pending"); + assert_eq!(rejoined.healthy_replicas, target_ids.len()); + assert_eq!(rejoined.stale_or_unavailable_replicas, 1); + assert_eq!( + rejoined + .authoritative_commit + .as_ref() + .expect("surviving target proof should remain authoritative") + .replicas + .as_slice(), + target_ids.as_slice() + ); + + let seeded_record = replica_record_for_members(&target, &target, &target_ids); + let seeded = crash_reload_replicas(full_ids.iter().map(|id| decoded_replica(*id, &seeded_record)).collect()); + assert_eq!(seeded.ledger, target); + assert!(seeded.stable_matches_ledger); + assert_eq!(seeded.persistence_state, "membership_repair_pending"); + + let mut replacement = seeded.ledger; + replacement + .claim_writer(200) + .expect("replacement node should claim the surviving ledger"); + prepare_scanner_pause_backlog_persist(&mut replacement, 200).expect("replacement node should persist"); + assert_eq!(replacement.writer_epoch, target.writer_epoch + 1); + let full_commit = replica_record_for_members(&target, &replacement, &full_ids); + + let source_first = crash_reload_replicas(vec![ + decoded_replica(source_id, &full_commit), + decoded_replica(target_ids[0], &seeded_record), + decoded_replica(target_ids[1], &seeded_record), + ]); + assert_eq!(source_first.ledger, target); + assert_eq!(source_first.persistence_state, "membership_repair_pending"); + + let target_first = crash_reload_replicas(vec![ + decoded_replica(source_id, &seeded_record), + decoded_replica(target_ids[0], &full_commit), + decoded_replica(target_ids[1], &seeded_record), + ]); + assert_eq!(target_first.ledger, target); + assert_eq!(target_first.persistence_state, "rolled_back_partial_commit"); + + let committed = crash_reload_replicas(full_ids.iter().map(|id| decoded_replica(*id, &full_commit)).collect()); + assert_eq!(committed.ledger, replacement); + assert_eq!(committed.persistence_state, "committed_pending_stabilization"); + + let stable_full_commit = replica_record_for_members(&replacement, &replacement, &full_ids); + let stable = crash_reload_replicas(full_ids.iter().map(|id| decoded_replica(*id, &stable_full_commit)).collect()); + assert_eq!(stable.ledger, replacement); + assert_eq!(stable.persistence_state, "healthy"); + assert_eq!(stable.healthy_replicas, full_ids.len()); + } + + #[test] + fn restart_recovers_paused_backlog_and_requires_one_full_catch_up_scan() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 3)); + prepare_scanner_pause_backlog_persist(&mut ledger, 110).expect("paused ledger should persist"); + let mut restarted = decode_valid_ledger(&ledger); + + restarted.claim_writer(120).expect("new process should claim a writer epoch"); + restarted.apply_observation(observation(120, false, 3)); + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(restarted.pending_full_scan); + assert_eq!(restarted.next_attempt_at_unix_secs, 120); + } + + #[test] + fn resume_after_clock_rollback_keeps_pause_timestamps_monotonic() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(ScannerPauseBacklogObservation { + now_unix_secs: 200, + pause_started_at_unix_secs: 200, + paused: true, + ..observation(200, true, 0) + }); + + ledger.apply_observation(observation(150, false, 0)); + + assert_eq!(ledger.pause_started_at_unix_secs, 200); + assert_eq!(ledger.pause_ended_at_unix_secs, 200); + prepare_scanner_pause_backlog_persist(&mut ledger, 150).expect("clock rollback should remain persistable"); + } + + #[test] + fn movement_that_starts_and_ends_inside_a_cycle_still_creates_catch_up_work() { + let mut ledger = durable_ledger(100); + ledger.observe_cycle_outcome( + ScannerPauseBacklogCycleOutcome::DataMovementDeferred, + ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(120, false, 0) + }, + ); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.pause_started_at_unix_secs, 120); + assert_eq!(ledger.pause_ended_at_unix_secs, 120); + } + + #[test] + fn restart_recovers_movement_that_completed_before_pause_was_persisted() { + let ledger = durable_ledger(100); + let mut restarted = decode_valid_ledger(&ledger); + + restarted.apply_observation(ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(120, false, 0) + }); + + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(restarted.pending_full_scan); + assert_eq!(restarted.pause_started_at_unix_secs, 120); + assert_eq!(restarted.pause_ended_at_unix_secs, 120); + } + + #[test] + fn generation_advance_reopens_full_scan_while_catch_up_has_only_known_work() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::Completed, observation(130, false, 1)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + + ledger.apply_observation(ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(140, false, 0) + }); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.movement_generation, 8); + assert_eq!(ledger.pause_started_at_unix_secs, 140); + assert_eq!(ledger.pause_ended_at_unix_secs, 140); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + } + + #[test] + fn completed_attempt_observing_generation_advance_keeps_full_scan_debt() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + + ledger.finish_attempt( + serial, + ScannerPauseBacklogCycleOutcome::Completed, + ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(130, false, 0) + }, + ); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.movement_generation, 8); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + prepare_scanner_pause_backlog_persist(&mut ledger, 130).expect("generation advance should remain persistable"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(restarted.pending_full_scan); + assert_eq!(restarted.movement_generation, 8); + } + + #[test] + fn generation_advance_preserves_attempt_rate_fence_for_deferred_and_progressed_cycles() { + for outcome in [ + ScannerPauseBacklogCycleOutcome::DataMovementDeferred, + ScannerPauseBacklogCycleOutcome::Progressed, + ] { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + assert_eq!(ledger.next_attempt_at_unix_secs, 420); + + ledger.finish_attempt( + serial, + outcome, + ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(130, false, 0) + }, + ); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + assert_eq!(ledger.current_window_started_at_unix_secs, 120); + assert_eq!(ledger.attempts_in_current_window, 1); + assert_eq!(ledger.last_attempt_at_unix_secs, 120); + prepare_scanner_pause_backlog_persist(&mut ledger, 130).expect("rate fence should remain persistable"); + let mut restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.next_attempt_at_unix_secs, 430); + assert_eq!(restarted.current_window_started_at_unix_secs, 120); + assert_eq!(restarted.attempts_in_current_window, 1); + assert_eq!(restarted.last_attempt_at_unix_secs, 120); + assert_eq!(restarted.begin_attempt(429), ScannerPauseBacklogAttemptDecision::RateLimited); + assert!(matches!(restarted.begin_attempt(430), ScannerPauseBacklogAttemptDecision::Tracked(_))); + } + } + + #[test] + fn remote_deferred_cycle_without_generation_change_keeps_durable_rate_state() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + assert_eq!(ledger.next_attempt_at_unix_secs, 420); + + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::DataMovementDeferred, observation(130, false, 0)); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.movement_generation, 7); + assert_eq!(ledger.current_window_started_at_unix_secs, 120); + assert_eq!(ledger.attempts_in_current_window, 1); + assert_eq!(ledger.last_attempt_at_unix_secs, 120); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + prepare_scanner_pause_backlog_persist(&mut ledger, 130).expect("remote defer fence should persist"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.current_window_started_at_unix_secs, 120); + assert_eq!(restarted.attempts_in_current_window, 1); + assert_eq!(restarted.last_attempt_at_unix_secs, 120); + assert_eq!(restarted.next_attempt_at_unix_secs, 430); + } + + #[test] + fn repeated_short_movements_cannot_reset_the_four_attempt_window() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(first) = ledger.begin_attempt(120) else { + panic!("first catch-up attempt should begin"); + }; + + for (serial, attempt_at, finished_at, movement_generation) in + [(first, 120, 130, 8), (0, 430, 440, 9), (0, 740, 750, 10), (0, 1050, 1060, 11)] + { + let serial = if serial == 0 { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(attempt_at) else { + panic!("short-movement attempt at {attempt_at} should be admitted"); + }; + serial + } else { + serial + }; + ledger.finish_attempt( + serial, + ScannerPauseBacklogCycleOutcome::Progressed, + ScannerPauseBacklogObservation { + movement_generation, + ..observation(finished_at, false, 0) + }, + ); + } + + assert_eq!(ledger.current_window_started_at_unix_secs, 120); + assert_eq!(ledger.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(ledger.last_attempt_at_unix_secs, 1050); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + assert_eq!(ledger.begin_attempt(1360), ScannerPauseBacklogAttemptDecision::RateLimited); + prepare_scanner_pause_backlog_persist(&mut ledger, 1360).expect("movement rate window should persist"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.current_window_started_at_unix_secs, 120); + assert_eq!(restarted.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(restarted.next_attempt_at_unix_secs, 3720); + } + + #[test] + fn node_offline_for_the_whole_movement_epoch_detects_generation_advance() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(ScannerPauseBacklogObservation { + movement_generation: 9, + ..observation(120, false, 0) + }); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + } + + #[test] + fn node_switch_selects_only_an_all_replica_writer_epoch() { + let old = durable_ledger(100); + let mut current = old.clone(); + current.claim_writer(110).expect("new node should claim a higher epoch"); + prepare_scanner_pause_backlog_persist(&mut current, 110).expect("new node claim should persist"); + let committed = crash_reload(&[ + replica_record(&old, ¤t), + replica_record(&old, ¤t), + replica_record(&old, ¤t), + ]); + assert_eq!(committed.ledger, current); + + let mut divergent = current.clone(); + divergent.deferred_cycles = 9; + let rolled_back = crash_reload(&[ + replica_record(&old, ¤t), + replica_record(&old, &divergent), + replica_record(&old, &old), + ]); + assert_eq!(rolled_back.ledger, old); + assert_eq!(rolled_back.persistence_state, "rolled_back_partial_commit"); + } + + #[test] + fn active_to_failed_rejoins_and_seeds_a_missing_source() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::Missing); + } + + #[test] + fn active_to_canceled_rejoins_from_an_older_source_commit() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::OlderCommit); + } + + #[test] + fn cleared_decommission_rejoins_and_repairs_a_stale_source_after_restart() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::StaleStable); + } + + #[test] + fn rejoined_source_cannot_reverse_overwrite_a_surviving_commit() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::NewerUnprovenCommit); + } + + #[test] + fn one_replica_high_epoch_commit_cannot_override_a_larger_surviving_commit() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::NewerSmallCommit); + } + + #[test] + fn partial_begin_attempt_commit_is_not_authoritative_after_crash() { + let mut stable = durable_ledger(100); + stable.apply_observation(observation(110, true, 0)); + stable.apply_observation(observation(120, false, 0)); + prepare_scanner_pause_backlog_persist(&mut stable, 120).expect("catch-up state should persist"); + + let mut begun = stable.clone(); + assert!(matches!(begun.begin_attempt(120), ScannerPauseBacklogAttemptDecision::Tracked(_))); + prepare_scanner_pause_backlog_persist(&mut begun, 120).expect("begun attempt should persist"); + let partial = crash_reload(&[ + replica_record(&stable, &begun), + replica_record(&stable, &stable), + replica_record(&stable, &stable), + ]); + + assert_eq!(partial.ledger, stable); + assert!(!partial.ledger.has_unfinished_attempt()); + let failed_member_absent = crash_reload(&[replica_record(&stable, &begun), replica_record(&stable, &begun)]); + assert_eq!(failed_member_absent.ledger, stable); + assert!(!failed_member_absent.ledger.has_unfinished_attempt()); + let committed = crash_reload(&[ + replica_record(&stable, &begun), + replica_record(&stable, &begun), + replica_record(&stable, &begun), + ]); + assert_eq!(committed.ledger, begun); + assert!(committed.ledger.has_unfinished_attempt()); + } + + #[test] + fn partial_idle_finish_is_rolled_back_and_interrupted_attempt_fails_closed() { + let mut stable = durable_ledger(100); + stable.apply_observation(observation(110, true, 0)); + stable.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = stable.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + prepare_scanner_pause_backlog_persist(&mut stable, 120).expect("begun attempt should persist"); + + let mut finished = stable.clone(); + finished.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::Completed, observation(130, false, 0)); + prepare_scanner_pause_backlog_persist(&mut finished, 130).expect("finished attempt should persist"); + assert_eq!(finished.phase, ScannerPauseBacklogPhase::Idle); + + let partial = crash_reload(&[ + replica_record(&stable, &finished), + replica_record(&stable, &stable), + replica_record(&stable, &stable), + ]); + assert_eq!(partial.ledger, stable); + assert_ne!(partial.ledger.phase, ScannerPauseBacklogPhase::Idle); + assert!(partial.ledger.has_unfinished_attempt()); + let mut replacement = partial.ledger; + replacement + .claim_writer(140) + .expect("replacement node should claim the interrupted attempt"); + assert_eq!(replacement.consecutive_failures, 1); + assert_ne!(replacement.phase, ScannerPauseBacklogPhase::Idle); + + let failed_member_absent = crash_reload(&[replica_record(&stable, &finished), replica_record(&stable, &finished)]); + assert_ne!(failed_member_absent.ledger.phase, ScannerPauseBacklogPhase::Idle); + assert!(failed_member_absent.ledger.has_unfinished_attempt()); + + let committed = crash_reload(&[ + replica_record(&stable, &finished), + replica_record(&stable, &finished), + replica_record(&stable, &finished), + ]); + assert_eq!(committed.ledger.phase, ScannerPauseBacklogPhase::Idle); + } + + #[test] + fn backlog_converges_only_after_full_scan_and_known_ilm_work_clear() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(first) = ledger.begin_attempt(120) else { + panic!("first catch-up attempt should be admitted"); + }; + let mut pending = observation(130, false, 0); + pending.discovered_expiry_items = 2; + ledger.finish_attempt(first, ScannerPauseBacklogCycleOutcome::Completed, pending); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + + ledger.apply_observation(observation(420, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(ledger.pending_work_items(), 0); + prepare_scanner_pause_backlog_persist(&mut ledger, 430).expect("converged ledger should persist"); + assert_eq!(decode_valid_ledger(&ledger).phase, ScannerPauseBacklogPhase::Idle); + } + + #[test] + fn fourth_completed_known_work_attempt_preserves_window_end_before_convergence() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(full_scan) = ledger.begin_attempt(120) else { + panic!("required full scan should begin"); + }; + let known_work = ScannerPauseBacklogObservation { + dirty_usage_buckets: 1, + discovered_expiry_items: 2, + discovered_transition_items: 3, + ..observation(130, false, 0) + }; + ledger.finish_attempt(full_scan, ScannerPauseBacklogCycleOutcome::Completed, known_work); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 6); + assert_eq!(ledger.begin_attempt(429), ScannerPauseBacklogAttemptDecision::RateLimited); + + for (attempt_at, finished_at, outcome) in [ + (430, 431, ScannerPauseBacklogCycleOutcome::Progressed), + (730, 731, ScannerPauseBacklogCycleOutcome::Progressed), + (1030, 1031, ScannerPauseBacklogCycleOutcome::Completed), + ] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(attempt_at) else { + panic!("known-work attempt at {attempt_at} should be admitted"); + }; + ledger.finish_attempt( + serial, + outcome, + ScannerPauseBacklogObservation { + now_unix_secs: finished_at, + ..known_work + }, + ); + } + assert_eq!(ledger.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(ledger.pending_work_items(), 6); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + assert_eq!(ledger.begin_attempt(1320), ScannerPauseBacklogAttemptDecision::RateLimited); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + + let ScannerPauseBacklogAttemptDecision::Tracked(final_attempt) = ledger.begin_attempt(3720) else { + panic!("next bounded window should admit known work"); + }; + ledger.finish_attempt(final_attempt, ScannerPauseBacklogCycleOutcome::Completed, observation(3721, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(ledger.pending_work_items(), 0); + prepare_scanner_pause_backlog_persist(&mut ledger, 3721).expect("converged known work should persist"); + assert_eq!(decode_valid_ledger(&ledger).phase, ScannerPauseBacklogPhase::Idle); + } + + #[test] + fn catch_up_rate_window_survives_restart() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + for now in [120, 420, 720, 1020] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("attempt at {now} should be admitted"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::RetryableFailure, observation(now + 1, false, 0)); + } + prepare_scanner_pause_backlog_persist(&mut ledger, 1021).expect("rate window should persist"); + let mut restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.begin_attempt(1320), ScannerPauseBacklogAttemptDecision::RateLimited); + assert_eq!(restarted.next_attempt_at_unix_secs, 3720); + } + + #[test] + fn bounded_partial_catch_up_progress_does_not_exhaust_the_retry_budget() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + for now in [120, 420, 720, 1020, 3720] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("partial attempt at {now} should be admitted"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::Progressed, observation(now + 1, false, 0)); + } + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.consecutive_failures, 0); + assert_eq!(ledger.next_attempt_at_unix_secs, 4020); + } + + #[test] + fn completed_usage_with_pending_maintenance_stays_durable_until_convergence() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + + let ScannerPauseBacklogAttemptDecision::Tracked(full_scan) = ledger.begin_attempt(120) else { + panic!("full catch-up scan should be admitted"); + }; + ledger.finish_attempt(full_scan, ScannerCycleOutcome::Completed.into(), observation(121, false, 1)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 1); + assert_eq!(ledger.next_attempt_at_unix_secs, 421); + + for (index, now) in [421, 721, 1021].into_iter().enumerate() { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("pending-maintenance attempt at {now} should be admitted"); + }; + ledger.finish_attempt( + serial, + ScannerCycleOutcome::CompletedWithPendingMaintenance.into(), + observation(now + 1, false, 0), + ); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 1); + if index == 0 { + assert_eq!(ledger.next_attempt_at_unix_secs, 721); + prepare_scanner_pause_backlog_persist(&mut ledger, 422).expect("pending maintenance should remain durable"); + ledger = decode_valid_ledger(&ledger); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.next_attempt_at_unix_secs, 721); + let status = status_from_ledger(&ledger, 422, "healthy".to_string(), true, 1, 1, 0, false, false, None); + assert_eq!(status.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(status.pending_full_scan); + assert_eq!(status.pending_work_items, 1); + assert!(status.rate_limited); + } + } + + assert_eq!(ledger.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + assert_eq!(ledger.begin_attempt(1320), ScannerPauseBacklogAttemptDecision::RateLimited); + + let ScannerPauseBacklogAttemptDecision::Tracked(final_attempt) = ledger.begin_attempt(3720) else { + panic!("converged maintenance attempt should be admitted in the next window"); + }; + ledger.finish_attempt(final_attempt, ScannerCycleOutcome::Completed.into(), observation(3721, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::Idle); + assert!(!ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 0); + } + + #[test] + fn retry_budget_exhaustion_uses_sparse_probe_and_alerts() { + let mut ledger = retry_exhausted_ledger(); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(ledger.next_attempt_at_unix_secs, 7321); + assert_eq!(ledger.begin_attempt(7000), ScannerPauseBacklogAttemptDecision::RateLimited); + assert!( + ledger + .alert_reasons(7000, false, false) + .contains(&ScannerPauseBacklogAlertReason::RetryBudgetExhausted) + ); + + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = ledger.begin_attempt(7321) else { + panic!("hourly recovery probe should be admitted"); + }; + ledger.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Progressed, observation(7322, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(ledger.consecutive_failures, 0); + assert_eq!(ledger.next_attempt_at_unix_secs, 7622); + prepare_scanner_pause_backlog_persist(&mut ledger, 7322).expect("recovered retry cadence should persist"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(restarted.current_window_started_at_unix_secs, 7322); + assert_eq!(restarted.attempts_in_current_window, 0); + assert_eq!(restarted.next_attempt_at_unix_secs, 7622); + } + + #[test] + fn retry_exhausted_probe_releases_hourly_floor_only_after_success() { + let exhausted = retry_exhausted_ledger(); + + let mut completed_with_work = exhausted.clone(); + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = completed_with_work.begin_attempt(7321) else { + panic!("completed recovery probe should be admitted"); + }; + completed_with_work.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Completed, observation(7322, false, 1)); + assert_eq!(completed_with_work.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(completed_with_work.current_window_started_at_unix_secs, 7322); + assert_eq!(completed_with_work.attempts_in_current_window, 0); + assert_eq!(completed_with_work.next_attempt_at_unix_secs, 7622); + + let mut completed = exhausted.clone(); + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = completed.begin_attempt(7321) else { + panic!("final recovery probe should be admitted"); + }; + completed.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Completed, observation(7322, false, 0)); + assert_eq!(completed.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(completed.next_attempt_at_unix_secs, 0); + + let mut deferred = exhausted.clone(); + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = deferred.begin_attempt(7321) else { + panic!("deferred probe should be admitted"); + }; + deferred.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::DataMovementDeferred, observation(7322, false, 0)); + assert_eq!(deferred.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(deferred.next_attempt_at_unix_secs, 10921); + + let mut failed = exhausted; + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = failed.begin_attempt(7321) else { + panic!("failed probe should be admitted"); + }; + failed.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::RetryableFailure, observation(7322, false, 0)); + assert_eq!(failed.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(failed.next_attempt_at_unix_secs, 10922); + + let mut counter_exhausted = retry_exhausted_ledger(); + counter_exhausted.counter_exhausted = true; + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = counter_exhausted.begin_attempt(7321) else { + panic!("counter-exhausted probe should be admitted"); + }; + counter_exhausted.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Progressed, observation(7322, false, 0)); + assert_eq!(counter_exhausted.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(counter_exhausted.next_attempt_at_unix_secs, 10921); + } + + #[test] + fn node_switch_counts_an_interrupted_attempt_and_keeps_the_rate_fence() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + assert!(matches!(ledger.begin_attempt(120), ScannerPauseBacklogAttemptDecision::Tracked(_))); + ledger.claim_writer(130).expect("replacement node should claim the ledger"); + + assert_eq!(ledger.current_attempt_serial, ledger.last_finished_attempt_serial); + assert_eq!(ledger.consecutive_failures, 1); + assert_eq!(ledger.begin_attempt(130), ScannerPauseBacklogAttemptDecision::RateLimited); + } + + #[test] + fn pause_and_backlog_thresholds_are_visible() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(ScannerPauseBacklogObservation { + now_unix_secs: 110, + paused: true, + movement_generation: 7, + movement_work_items: 1, + pause_started_at_unix_secs: 100, + dirty_usage_buckets: SCANNER_PAUSE_BACKLOG_ITEMS_ALERT, + discovered_expiry_items: 0, + discovered_transition_items: 0, + }); + ledger.deferred_cycles = SCANNER_PAUSE_DEFERRED_CYCLES_ALERT; + let alerts = ledger.alert_reasons(100 + SCANNER_PAUSE_DURATION_ALERT_SECONDS, true, false); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::PauseDurationThreshold)); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::DeferredCyclesThreshold)); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::BacklogItemsThreshold)); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::ReplicaDegraded)); + } +} diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 60dacc00f..1f2b217b4 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -39,30 +39,46 @@ async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc) { } async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool) -> (tempfile::TempDir, Arc) { + setup_scanner_cycle_store_with_pool_count(seed_usage_baseline, 1).await +} + +async fn setup_scanner_cycle_store_with_pool_count( + seed_usage_baseline: bool, + pool_count: usize, +) -> (tempfile::TempDir, Arc) { init_ecstore_config_for_scanner_tests(); let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created"); - let mut endpoints = Vec::new(); - for disk_index in 0..4 { - let disk_path = temp_dir.path().join(format!("disk{disk_index}")); - tokio::fs::create_dir_all(&disk_path) - .await - .expect("scanner cycle test disk should be created"); - let mut endpoint = - Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); - endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(disk_index); - endpoints.push(endpoint); + let mut pools = Vec::with_capacity(pool_count); + for pool_index in 0..pool_count { + let mut endpoints = Vec::new(); + for disk_index in 0..4 { + let disk_path = temp_dir.path().join(format!("pool{pool_index}/disk{disk_index}")); + tokio::fs::create_dir_all(&disk_path) + .await + .expect("scanner cycle test disk should be created"); + let mut endpoint = + Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(pool_index); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + pools.push(PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: if pool_count == 1 { + "scanner-cycle-metrics".to_string() + } else { + format!("scanner-cycle-metrics-pool-{pool_index}") + }, + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }); } - let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints { - legacy: false, - set_count: 1, - drives_per_set: 4, - endpoints: Endpoints::from(endpoints), - cmd_line: "scanner-cycle-metrics".to_string(), - platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), - }]); + let endpoint_pools = EndpointServerPools::from(pools); let instance_ctx = Arc::new(InstanceContext::new()); + instance_ctx.set_endpoints(endpoint_pools.clone()); init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) .await .expect("scanner cycle test disks should initialize"); @@ -89,6 +105,27 @@ async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool (temp_dir, store) } +async fn restart_scanner_cycle_store_from(store: &Arc) -> Arc { + let endpoint_pools = store + .instance_endpoints() + .expect("scanner restart test store should retain its endpoint topology"); + let instance_ctx = Arc::new(InstanceContext::new()); + instance_ctx.set_endpoints(endpoint_pools.clone()); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("scanner restart test disks should reinitialize"); + let restarted = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address should parse"), + endpoint_pools, + CancellationToken::new(), + instance_ctx, + ) + .await + .expect("restarted scanner cycle test ECStore should initialize"); + init_bucket_metadata_sys_for_scanner_tests(restarted.clone()).await; + restarted +} + fn assert_run_data_scanner_signature(_run: F) where F: Fn(CancellationToken, Arc) -> Fut, @@ -101,6 +138,190 @@ fn run_data_scanner_keeps_its_two_argument_api() { assert_run_data_scanner_signature(run_data_scanner); } +#[tokio::test] +async fn restarted_main_loop_completes_durable_pause_backlog_catch_up() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + global_metrics().set_cycle(None).await; + let (_temp_dir, store) = setup_scanner_cycle_store().await; + + let paused_at = scanner_pause_backlog_now(); + let mut seeded = ScannerPauseBacklogController::claim(store.clone(), paused_at) + .await + .expect("seed writer should claim the durable pause backlog"); + seeded + .observe(ScannerPauseBacklogObservation { + now_unix_secs: paused_at.saturating_add(1), + paused: true, + movement_generation: store.scanner_data_movement_generation().saturating_add(1), + movement_work_items: 1, + pause_started_at_unix_secs: paused_at.saturating_add(1), + dirty_usage_buckets: 0, + discovered_expiry_items: 0, + discovered_transition_items: 0, + }) + .await; + drop(seeded); + + let seeded_status = scanner_pause_backlog_status(store.clone()).await; + assert!(seeded_status.durable, "seeded pause backlog must be set-backed"); + assert_eq!(seeded_status.phase, ScannerPauseBacklogPhase::Paused); + assert!(seeded_status.pending_full_scan); + assert_eq!(seeded_status.catch_up_attempts, 0); + + let restarted = restart_scanner_cycle_store_from(&store).await; + assert!( + restarted.instance_endpoints().is_some(), + "restarted scanner store must retain instance endpoints" + ); + let restarted_status = scanner_pause_backlog_status(restarted.clone()).await; + assert_eq!(restarted_status.phase, ScannerPauseBacklogPhase::Paused); + assert_eq!(restarted_status.generation, seeded_status.generation); + + let ctx = CancellationToken::new(); + let scanner_ctx = ctx.clone(); + let scanner_store = restarted.clone(); + let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await }); + + let final_status = match tokio::time::timeout(Duration::from_secs(30), async { + loop { + let status = scanner_pause_backlog_status(restarted.clone()).await; + if status.phase == ScannerPauseBacklogPhase::Idle + && status.writer_epoch > seeded_status.writer_epoch + && status.catch_up_attempts > seeded_status.catch_up_attempts + { + break status; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + { + Ok(status) => status, + Err(err) => { + ctx.cancel(); + scanner_task.abort(); + panic!("restarted scanner did not complete durable catch-up through the main loop: {err}"); + } + }; + + ctx.cancel(); + tokio::time::timeout(Duration::from_secs(5), scanner_task) + .await + .expect("scanner loop should stop after cancellation") + .expect("scanner task should not panic") + .expect("scanner loop should exit cleanly"); + + assert!(final_status.durable); + assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle); + assert!(!final_status.pending_full_scan); + assert_eq!(final_status.pending_work_items, 0); + assert_eq!(final_status.consecutive_failures, 0); + assert!(final_status.pause_ended_at_unix_secs >= final_status.pause_started_at_unix_secs); + + let usage = read_config(restarted.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("the catch-up scanner cycle should leave an authoritative usage snapshot readable"); + let usage = serde_json::from_slice::(&usage).expect("authoritative usage snapshot should decode"); + assert!( + usage.is_complete_bucket_usage_snapshot(), + "durable catch-up must run a complete scanner cycle before clearing the backlog" + ); + + global_metrics().set_cycle(None).await; + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial_test::serial(scanner_runtime_env)] +async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() { + temp_env::async_with_vars([(ENV_SCANNER_CYCLE, Some("1")), (ENV_SCANNER_START_DELAY_SECS, Some("0"))], async { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + global_metrics().set_cycle(None).await; + let (_temp_dir, store) = setup_scanner_cycle_store_with_pool_count(true, 2).await; + + let ctx = CancellationToken::new(); + let scanner_ctx = ctx.clone(); + let scanner_store = store.clone(); + let startup_probe = ScannerStartupObservedProbe::install(); + let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await }); + startup_probe.wait().await; + let ready_probe = ScannerRuntimeObservedProbe::install(&store, false); + startup_probe.resume(); + drop(startup_probe); + ready_probe.wait().await; + drop(ready_probe); + + let paused_probe = ScannerRuntimeObservedProbe::install(&store, true); + let paused_at = time::OffsetDateTime::now_utc(); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].last_update = paused_at; + pool_meta.pools[0].decommission = Some(crate::storage_api::owner::EcstorePoolDecommissionInfo { + failed: true, + ..Default::default() + }); + } + let pause_status = store.scanner_data_movement_pause_status().await; + assert!(pause_status.paused); + paused_probe.wait().await; + drop(paused_probe); + + let paused_backlog = scanner_pause_backlog_status(store.clone()).await; + assert_eq!(paused_backlog.phase, ScannerPauseBacklogPhase::Paused); + assert!(paused_backlog.pending_full_scan); + + let resumed_probe = ScannerRuntimeObservedProbe::install(&store, false); + store + .clear_decommission(0) + .await + .expect("terminal decommission clear should publish a movement generation"); + resumed_probe.wait().await; + drop(resumed_probe); + + let final_status = match tokio::time::timeout(Duration::from_secs(30), async { + loop { + let status = scanner_pause_backlog_status(store.clone()).await; + if status.phase == ScannerPauseBacklogPhase::Idle + && status.writer_epoch == paused_backlog.writer_epoch + && status.catch_up_attempts > paused_backlog.catch_up_attempts + { + break status; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + { + Ok(status) => status, + Err(err) => { + ctx.cancel(); + scanner_task.abort(); + panic!("running scanner did not complete durable catch-up after a runtime movement clear: {err}"); + } + }; + + ctx.cancel(); + tokio::time::timeout(Duration::from_secs(5), scanner_task) + .await + .expect("scanner loop should stop after cancellation") + .expect("scanner task should not panic") + .expect("scanner loop should exit cleanly"); + + assert!(final_status.durable); + assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(final_status.writer_epoch, paused_backlog.writer_epoch); + assert!(!final_status.pending_full_scan); + assert_eq!(final_status.pending_work_items, 0); + assert_eq!(final_status.consecutive_failures, 0); + + global_metrics().set_cycle(None).await; + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + }) + .await; + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); +} + #[tokio::test] async fn scanner_cycle_lock_fence_cancels_cycle_context() { let cycle_ctx = CancellationToken::new(); diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index de54b8f40..b480921ca 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -139,6 +139,12 @@ objects: backoff state. - `metrics`: scanner work, pressure, checkpoint, lifecycle, replication, heal, bitrot, and alert counters. +- `data_movement_pause`: the global-pause policy, current movement reason, + operation epoch, start time, duration, and estimated movement work items. +- `pause_backlog`: the replicated durable pause ledger, post-pause catch-up + phase, rate window, retry state, thresholds, and active alert reasons. +- `catch_up_estimate`: movement work plus current dirty-usage and already + discovered lifecycle queues. Example fields to inspect: @@ -163,8 +169,97 @@ metrics.cycle_timeout_total metrics.cycle_last_progress_age metrics.leader_lease_without_progress metrics.cycle_recovery_required_total +data_movement_pause.paused +data_movement_pause.reasons +data_movement_pause.duration_seconds +data_movement_pause.operation_epoch +data_movement_pause.movement_generation +data_movement_pause.movement_backlog_work_items +pause_backlog.persistence_state +pause_backlog.phase +pause_backlog.pause_duration_seconds +pause_backlog.pending_full_scan +pause_backlog.pending_work_items +pause_backlog.next_attempt_at_unix_secs +pause_backlog.alert_reasons +catch_up_estimate.dirty_usage_buckets +catch_up_estimate.discovered_expiry_items +catch_up_estimate.discovered_transition_items ``` +## Data Movement Pauses + +RustFS currently uses a `global_pause` policy while pool decommission or +rebalance can hide scanner metadata. Usage publication, lifecycle discovery, +tier cleanup discovery, scanner-originated heal and bitrot checks, and +replication discovery are deferred together. A failed or canceled +decommission remains a publication barrier until an operator retries or clears +it. + +`data_movement_pause.reasons` combines the in-process decommission worker state +with the durable pool and rebalance operation metadata. Exhausted operation +epochs or movement generations also fail closed and appear as explicit pause +reasons. Its start time, duration, and movement backlog come from the durable +metadata; a worker-only or exhausted-counter snapshot can therefore report +`paused=true` with zero start time and backlog. +`movement_backlog_work_items` counts remaining movement bucket work units, not +expired objects. `catch_up_estimate` combines that estimate with dirty-usage +buckets and lifecycle items that were already discovered before or during the +pause. The API sets `undiscovered_ilm_items_known=false` because a global pause +cannot count newly expired objects without scanning the namespace. Use +`usage_baseline_unix_secs` to judge the age of that estimate. + +The same pause and estimate objects are included in +`GET /v3/ilm/expiry/status`. The gauges +`rustfs_scanner_data_movement_paused`, +`rustfs_scanner_data_movement_pause_duration_seconds`, and +`rustfs_scanner_data_movement_backlog_work_items` expose the local snapshot +without bucket-name labels. + +The scanner persists `.scanner-pause-backlog.json` independently on erasure +sets in every surviving pool. A generation becomes authoritative only after +the identical commit record reaches every set named by its membership marker. +When a failed, canceled, or cleared decommission source rejoins, the last +committed surviving-set ledger seeds it before a new full-membership commit is +allowed; a smaller stale source membership cannot override the largest valid +surviving-set proof, and a membership claim is valid only when every declared +member stores the same proof. This repair appears as +`membership_repair_pending`. A partial commit is +rolled back to the previous stable generation after a crash or leader switch. +The ledger never rewrites pool or rebalance movement state. A new scanner +leader recovers the committed writer epoch and generation, counts an +interrupted attempt as a failure, and requires one successful full namespace +scan after movement clears. Known dirty-usage, expiry, and transition queues +must also reach zero before the ledger returns to `idle`. If the ledger cannot +be read or updated, scanner cycles remain gated and persistence is retried +every five minutes; the management status reports `persistence_unavailable` +until recovery. + +Catch-up attempts remain subject to the normal cycle duration, object, +directory, sleeper, and foreground-read budgets. The additional durable rate +window admits at most four attempts per hour and no more than one attempt per +five minutes. Five consecutive failed or interrupted attempts move the ledger +to `retry_exhausted`; accelerated retries stop and a sparse hourly probe is +used instead. A successful probe can return to bounded catch-up. + +`pause_backlog.thresholds` reports the exact pause-duration, deferred-cycle, +backlog-size, rate, and failure limits used by the running binary. +`pause_backlog.alert_reasons` identifies exceeded thresholds, exhausted +counters or retries, replica degradation, and persistence failures. The +threshold alerts fire after a 24-hour pause, three movement deferrals in one +unconverged pause episode, or 10,000 known pending work items. The +corresponding unlabeled gauges are: + +- `rustfs_scanner_pause_backlog_phase` (`0` idle, `1` paused, `2` catching up, + `3` retry exhausted); +- `rustfs_scanner_pause_backlog_pause_duration_seconds`; +- `rustfs_scanner_pause_backlog_pending_work_items`; +- `rustfs_scanner_pause_backlog_consecutive_failures`; +- `rustfs_scanner_pause_backlog_rate_limited`; +- `rustfs_scanner_pause_backlog_retry_exhausted`; +- `rustfs_scanner_pause_backlog_alerting`; +- `rustfs_scanner_pause_backlog_replica_degraded`. + ## Reading Pacing Pressure `metrics.pacing_pressure.primary_pressure` summarizes the highest-priority diff --git a/rustfs/src/admin/handlers/scanner.rs b/rustfs/src/admin/handlers/scanner.rs index 52fdfc777..47815ae1e 100644 --- a/rustfs/src/admin/handlers/scanner.rs +++ b/rustfs/src/admin/handlers/scanner.rs @@ -18,6 +18,7 @@ use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ app_context_from_req, current_object_store_handle_for_context, current_scanner_metrics_report, }; +use crate::admin::storage_api::ScannerDataMovementPauseStatus; use crate::module_switches::{ENV_SCANNER_ENABLED, scanner_enabled_from_env}; use crate::server::ADMIN_PREFIX; use chrono::Utc; @@ -27,6 +28,8 @@ use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::Credentials; use rustfs_policy::policy::action::{Action, AdminAction}; +#[cfg(test)] +use rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot; use rustfs_scanner_contracts::metrics::{ ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport, }; @@ -46,6 +49,9 @@ struct ScannerStatusResponse { cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, cycle_recovery: rustfs_scanner::ScannerCycleRecoveryStatus, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, + catch_up_estimate: ScannerCatchUpEstimate, } #[derive(Debug, Deserialize)] @@ -62,6 +68,17 @@ struct ScannerFreshnessStatus { reason: Option<&'static str>, } +#[derive(Debug, Serialize)] +struct ScannerCatchUpEstimate { + estimated: bool, + movement_work_items: u64, + dirty_usage_buckets: u64, + discovered_expiry_items: u64, + discovered_transition_items: u64, + undiscovered_ilm_items_known: bool, + usage_baseline_unix_secs: u64, +} + #[derive(Debug, Serialize)] struct IlmExpiryStatusResponse { enabled: bool, @@ -71,6 +88,46 @@ struct IlmExpiryStatusResponse { maintenance_control: ScannerMaintenanceControlSnapshot, current_cycle_lifecycle_expiry_actions: u64, last_cycle_lifecycle_expiry_actions: u64, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, + catch_up_estimate: ScannerCatchUpEstimate, +} + +fn scanner_catch_up_estimate( + pause: &ScannerDataMovementPauseStatus, + backlog: &rustfs_scanner::ScannerPauseBacklogStatus, + metrics: &ScannerMetricsReport, +) -> ScannerCatchUpEstimate { + ScannerCatchUpEstimate { + estimated: pause.paused || backlog.phase != rustfs_scanner::ScannerPauseBacklogPhase::Idle, + movement_work_items: pause.movement_backlog_work_items.max(backlog.movement_work_items), + dirty_usage_buckets: metrics.usage_freshness.dirty_pending_buckets.max(backlog.dirty_usage_buckets), + discovered_expiry_items: metrics + .lifecycle_expiry + .current_queued + .saturating_add(metrics.lifecycle_expiry.current_active) + .max(backlog.discovered_expiry_items), + discovered_transition_items: metrics + .lifecycle_transition + .current_queued + .saturating_add(metrics.lifecycle_transition.current_active) + .saturating_add(metrics.lifecycle_transition.compensation_pending) + .saturating_add(metrics.lifecycle_transition.compensation_running) + .max(backlog.discovered_transition_items), + undiscovered_ilm_items_known: !pause.paused && !backlog.pending_full_scan, + usage_baseline_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs, + } +} + +fn unavailable_pause_backlog(error: &str) -> rustfs_scanner::ScannerPauseBacklogStatus { + rustfs_scanner::ScannerPauseBacklogStatus { + persistence_state: "unavailable".to_string(), + alerting: true, + alert_reasons: vec![rustfs_scanner::ScannerPauseBacklogAlertReason::PersistenceUnavailable], + thresholds: rustfs_scanner::ScannerPauseBacklogThresholds::default(), + error: Some(error.to_string()), + ..Default::default() + } } fn scanner_disabled_reason(enabled: bool) -> Option { @@ -122,8 +179,11 @@ fn scanner_status_response( metrics: ScannerMetricsReport, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, ) -> ScannerStatusResponse { let freshness = scanner_freshness_status(&metrics, &runtime_config, cycle_schedule.effective_interval_seconds()); + let catch_up_estimate = scanner_catch_up_estimate(&data_movement_pause, &pause_backlog, &metrics); ScannerStatusResponse { enabled, disabled_reason: scanner_disabled_reason(enabled), @@ -132,6 +192,9 @@ fn scanner_status_response( cycle_schedule, runtime_config, cycle_recovery: rustfs_scanner::scanner::scanner_cycle_recovery_status(), + data_movement_pause, + pause_backlog, + catch_up_estimate, } } @@ -140,8 +203,11 @@ fn ilm_expiry_status_response( metrics: ScannerMetricsReport, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, ) -> IlmExpiryStatusResponse { let freshness = scanner_freshness_status(&metrics, &runtime_config, cycle_schedule.effective_interval_seconds()); + let catch_up_estimate = scanner_catch_up_estimate(&data_movement_pause, &pause_backlog, &metrics); IlmExpiryStatusResponse { enabled, disabled_reason: scanner_disabled_reason(enabled), @@ -150,6 +216,9 @@ fn ilm_expiry_status_response( maintenance_control: metrics.maintenance_control, current_cycle_lifecycle_expiry_actions: metrics.current_cycle_lifecycle_expiry_actions, last_cycle_lifecycle_expiry_actions: metrics.last_cycle_lifecycle_expiry_actions, + data_movement_pause, + pause_backlog, + catch_up_estimate, } } @@ -208,7 +277,20 @@ impl Operation for ScannerStatusHandler { let metrics = current_scanner_metrics_report().await; let runtime_config = rustfs_scanner::scanner_runtime_config_status(); let cycle_schedule = rustfs_scanner::scanner_cycle_schedule_status(); - let response = scanner_status_response(enabled, metrics, runtime_config, cycle_schedule); + let store = + app_context_from_req(&req).and_then(|context| current_object_store_handle_for_context(Some(context.as_ref()))); + let (data_movement_pause, pause_backlog) = match store { + Some(store) => ( + store.scanner_data_movement_pause_status().await, + rustfs_scanner::scanner_pause_backlog_status(store).await, + ), + None => ( + ScannerDataMovementPauseStatus::default(), + unavailable_pause_backlog("storage layer not initialized"), + ), + }; + let response = + scanner_status_response(enabled, metrics, runtime_config, cycle_schedule, data_movement_pause, pause_backlog); let body = serde_json::to_vec(&response).map_err(|err| { S3Error::with_message(S3ErrorCode::InternalError, format!("failed to encode scanner status: {err}")) })?; @@ -258,7 +340,20 @@ impl Operation for IlmExpiryStatusHandler { let metrics = current_scanner_metrics_report().await; let runtime_config = rustfs_scanner::scanner_runtime_config_status(); let cycle_schedule = rustfs_scanner::scanner_cycle_schedule_status(); - let response = ilm_expiry_status_response(enabled, metrics, runtime_config, cycle_schedule); + let store = + app_context_from_req(&req).and_then(|context| current_object_store_handle_for_context(Some(context.as_ref()))); + let (data_movement_pause, pause_backlog) = match store { + Some(store) => ( + store.scanner_data_movement_pause_status().await, + rustfs_scanner::scanner_pause_backlog_status(store).await, + ), + None => ( + ScannerDataMovementPauseStatus::default(), + unavailable_pause_backlog("storage layer not initialized"), + ), + }; + let response = + ilm_expiry_status_response(enabled, metrics, runtime_config, cycle_schedule, data_movement_pause, pause_backlog); let body = serde_json::to_vec(&response).map_err(|err| { S3Error::with_message(S3ErrorCode::InternalError, format!("failed to encode ILM expiry status: {err}")) })?; @@ -388,6 +483,8 @@ mod tests { ScannerMetricsReport::default(), rustfs_scanner::scanner_runtime_config_status(), rustfs_scanner::ScannerCycleScheduleStatus::default(), + ScannerDataMovementPauseStatus::default(), + rustfs_scanner::ScannerPauseBacklogStatus::default(), ); let encoded = serde_json::to_value(response).expect("scanner status should serialize"); @@ -401,6 +498,23 @@ mod tests { encoded["cycle_recovery"]["quarantine_path"], rustfs_scanner::DATA_USAGE_BLOOM_RECOVERY_PATH.as_str() ); + assert_eq!(encoded["data_movement_pause"]["policy"], "global_pause"); + assert_eq!(encoded["data_movement_pause"]["paused"], false); + assert_eq!(encoded["catch_up_estimate"]["estimated"], false); + assert_eq!(encoded["catch_up_estimate"]["undiscovered_ilm_items_known"], true); + } + + #[test] + fn scanner_status_keeps_an_unavailable_storage_layer_observable() { + let backlog = unavailable_pause_backlog("storage layer not initialized"); + + assert_eq!(backlog.persistence_state, "unavailable"); + assert!(backlog.alerting); + assert_eq!( + backlog.alert_reasons, + vec![rustfs_scanner::ScannerPauseBacklogAlertReason::PersistenceUnavailable] + ); + assert_eq!(backlog.error.as_deref(), Some("storage layer not initialized")); } #[test] @@ -418,6 +532,13 @@ mod tests { scanner_not_enqueued: 13, delete_failed: 19, }, + lifecycle_transition: ScannerLifecycleTransitionSnapshot { + current_queued: 2, + current_active: 3, + compensation_pending: 5, + compensation_running: 7, + ..Default::default() + }, maintenance_control: ScannerMaintenanceControlSnapshot { primary_control: "expiry_backlog".to_string(), ..Default::default() @@ -431,6 +552,18 @@ mod tests { metrics, rustfs_scanner::scanner_runtime_config_status(), rustfs_scanner::ScannerCycleScheduleStatus::default(), + ScannerDataMovementPauseStatus { + paused: true, + movement_backlog_work_items: 31, + movement_backlog_estimated: true, + ..Default::default() + }, + rustfs_scanner::ScannerPauseBacklogStatus { + phase: rustfs_scanner::ScannerPauseBacklogPhase::Paused, + movement_work_items: 31, + pending_full_scan: true, + ..Default::default() + }, ); let encoded = serde_json::to_value(response).expect("ILM expiry status should serialize"); @@ -441,5 +574,11 @@ mod tests { assert_eq!(encoded["maintenance_control"]["primary_control"].as_str(), Some("expiry_backlog")); assert_eq!(encoded["current_cycle_lifecycle_expiry_actions"].as_u64(), Some(23)); assert_eq!(encoded["last_cycle_lifecycle_expiry_actions"].as_u64(), Some(29)); + assert_eq!(encoded["data_movement_pause"]["paused"], true); + assert_eq!(encoded["pause_backlog"]["phase"], "paused"); + assert_eq!(encoded["catch_up_estimate"]["movement_work_items"].as_u64(), Some(31)); + assert_eq!(encoded["catch_up_estimate"]["discovered_expiry_items"].as_u64(), Some(9)); + assert_eq!(encoded["catch_up_estimate"]["discovered_transition_items"].as_u64(), Some(17)); + assert_eq!(encoded["catch_up_estimate"]["undiscovered_ilm_items_known"], false); } } diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 5ea5b3f74..200bdc2be 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -85,7 +85,7 @@ mod ecstore_rpc { } mod ecstore_storage { - pub(crate) use crate::storage::storage_api::ecstore_storage::ECStore; + pub(crate) use crate::storage::storage_api::ecstore_storage::{ECStore, ScannerDataMovementPauseStatus}; } mod ecstore_tier { @@ -108,6 +108,7 @@ pub(crate) type RebalanceCleanupWarnings = ecstore_rebalance::RebalanceCleanupWa pub(crate) type RebalanceMeta = ecstore_rebalance::RebalanceMeta; pub(crate) type RebalanceStats = ecstore_rebalance::RebalanceStats; pub(crate) type RebalanceStopPropagationRecord = ecstore_rebalance::RebalanceStopPropagationRecord; +pub(crate) type ScannerDataMovementPauseStatus = ecstore_storage::ScannerDataMovementPauseStatus; pub(crate) type StorageError = ecstore_error::StorageError; pub(crate) type Error = StorageError; pub(crate) type Result = core::result::Result; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 14587b344..0288ac712 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -594,8 +594,9 @@ pub(crate) mod ecstore_storage { #[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks; pub(crate) use rustfs_ecstore::api::storage::{ - ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, - init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx, + ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path, + find_local_disk_by_ref, init_local_disks_with_instance_ctx, init_lock_clients, + prewarm_local_disk_id_map_with_instance_ctx, }; }