From 095bf340865c80edd0bf3fa9e94afab2fcf450a3 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 20 Aug 2026 19:12:27 +0800 Subject: [PATCH] refactor(scanner): split scanner.rs cycle/leadership/persist children (#6305) Split the 8178-line scanner.rs (48% inline tests) into a canonical scanner.rs + scanner/ module tree with zero behavior change: - scanner.rs (~2140): cycle constants, schedule status, budget/config helpers, startup, maintenance features, the two run loops, and cycle-result finalization - scanner/activity.rs (~770): wake/backoff policy and scanner activity observation (probing, generations, topology digest) - scanner/heal_info.rs (~110): the background-heal info object - scanner/cycle_state.rs (~500): cycle-state codec, persisted usage floors, and cycle-state persistence - scanner/leadership.rs (~360): leader-lock claiming, usage-epoch fencing, and lock-loss handling - scanner/usage_store.rs (~480): the CAS data-usage store pipeline and observed-snapshot cleanup - scanner/tests.rs (~3920): the inline test module as a child module All crate paths are unchanged: scanner::BackgroundHealInfo, scanner::read_background_heal_info, scanner::store_data_usage_in_backend, and scanner_topology_digest resolve through root re-exports with their original visibilities, and the pub(crate) surface used by scanner_io and remote_scanner re-exports at pub(crate). Cross-module items gain pub(super), whose scope equals the old single-module privacy domain. Code is moved verbatim apart from those markers, per-module import headers, and rustfmt re-wraps. Co-authored-by: heihutu --- crates/scanner/src/scanner.rs | 6080 +-------------------- crates/scanner/src/scanner/activity.rs | 773 +++ crates/scanner/src/scanner/cycle_state.rs | 498 ++ crates/scanner/src/scanner/heal_info.rs | 109 + crates/scanner/src/scanner/leadership.rs | 363 ++ crates/scanner/src/scanner/tests.rs | 3920 +++++++++++++ crates/scanner/src/scanner/usage_store.rs | 483 ++ 7 files changed, 6167 insertions(+), 6059 deletions(-) create mode 100644 crates/scanner/src/scanner/activity.rs create mode 100644 crates/scanner/src/scanner/cycle_state.rs create mode 100644 crates/scanner/src/scanner/heal_info.rs create mode 100644 crates/scanner/src/scanner/leadership.rs create mode 100644 crates/scanner/src/scanner/tests.rs create mode 100644 crates/scanner/src/scanner/usage_store.rs diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index eb2dc9d4b..4dabd40fe 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -158,22 +158,6 @@ fn notify_scanner_cycle_state_persist_test_hook(leader_epoch: u64) { } } -#[derive(Debug, thiserror::Error)] -enum ScannerCycleStateError { - #[error("failed to encode scanner cycle state: {0}")] - Encode(#[from] rmp_serde::encode::Error), - #[error("failed to decode scanner cycle state: {0}")] - Decode(#[from] rmp_serde::decode::Error), - #[error("{0}")] - InvalidData(&'static str), -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct PersistedUsageFloor { - next_cycle: u64, - leader_epoch: u64, -} - #[derive(Clone, Copy, Debug, Serialize)] #[non_exhaustive] pub struct ScannerCycleScheduleStatus { @@ -321,752 +305,6 @@ fn cap_clean_idle_cycle_delay(delay: Duration, max_interval: Duration, enabled: .max(Duration::from_secs(1)) } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ScannerCycleWakeReason { - Timer, - DirtyUsage, - ClusterActivity, - ClusterMaintenance, - ClusterActivityUnavailable, - RuntimeConfig, - MaintenanceConfig, - LeaderLockLost, - Cancelled, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum ScannerCycleOutcome { - Completed, - CompletedWithPendingMaintenance, - Partial, - Superseded, - Deferred(ScannerCycleDeferReason), - Failed, -} - -pub(crate) fn scanner_cycle_outcome_with_pending_maintenance( - outcome: ScannerCycleOutcome, - pending_maintenance_work: bool, -) -> ScannerCycleOutcome { - if outcome == ScannerCycleOutcome::Completed && pending_maintenance_work { - ScannerCycleOutcome::CompletedWithPendingMaintenance - } else { - outcome - } -} - -async fn remote_dirty_usage_acknowledgement_pending(cycle: u64, acknowledgement_count: usize, acknowledgement: F) -> bool -where - F: Future>, - E: std::fmt::Display, -{ - match acknowledgement.await { - Ok(dirty_usage_pending) => dirty_usage_pending, - Err(err) => { - warn!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - cycle, - acknowledgement_count, - error = %err, - state = "remote_dirty_usage_acknowledgement_pending", - "Scanner cycle left remote dirty usage acknowledgements pending" - ); - true - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct ScannerCleanIdleBackoff { - interval_multiplier: u32, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct ScannerRetryBackoff { - consecutive_cycles: u32, -} - -impl ScannerRetryBackoff { - fn record_retryable_cycle(&mut self, retryable: bool) { - if retryable { - self.consecutive_cycles = self.consecutive_cycles.saturating_add(1); - } else { - self.consecutive_cycles = 0; - } - } - - fn retry_interval(self, configured_interval: Duration) -> Option { - let exponent = self.consecutive_cycles.checked_sub(1)?.min(31); - let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); - let base_interval = configured_interval - .max(Duration::from_secs(1)) - .min(SCANNER_RETRY_BASE_INTERVAL); - let cap = SCANNER_RETRY_MAX_INTERVAL.max(configured_interval.max(Duration::from_secs(1))); - Some(base_interval.saturating_mul(multiplier).min(cap)) - } -} - -impl Default for ScannerCleanIdleBackoff { - fn default() -> Self { - Self { interval_multiplier: 1 } - } -} - -impl ScannerCleanIdleBackoff { - fn reset(&mut self) { - self.interval_multiplier = 1; - } - - fn effective_interval(self, base_interval: Duration, max_interval: Duration, enabled: bool) -> Duration { - let base_interval = base_interval.max(Duration::from_secs(1)); - if !enabled { - return base_interval; - } - - let max_interval = max_interval.max(base_interval); - base_interval.saturating_mul(self.interval_multiplier).min(max_interval) - } - - fn record_cycle( - &mut self, - base_interval: Duration, - max_interval: Duration, - enabled: bool, - wake_reason: ScannerCycleWakeReason, - outcome: ScannerCycleOutcome, - dirty_work_observed: bool, - ) { - if !enabled - || wake_reason != ScannerCycleWakeReason::Timer - || outcome != ScannerCycleOutcome::Completed - || dirty_work_observed - { - self.reset(); - return; - } - - let max_interval = max_interval.max(base_interval.max(Duration::from_secs(1))); - if self.effective_interval(base_interval, max_interval, true) < max_interval { - self.interval_multiplier = self.interval_multiplier.saturating_mul(CLEAN_IDLE_BACKOFF_FACTOR); - } - } -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct ScannerMaintenanceInspectionRetry { - consecutive_failures: u32, - retry_at: Option, -} - -impl ScannerMaintenanceInspectionRetry { - fn from_features(features: ScannerMaintenanceFeatures, now: Instant) -> Self { - let mut retry = Self::default(); - retry.record_inspection(features, now); - retry - } - - fn reset(&mut self) { - self.consecutive_failures = 0; - self.retry_at = None; - } - - fn retry_interval(self) -> Option { - if self.consecutive_failures == 0 { - return None; - } - - let exponent = self.consecutive_failures.saturating_sub(1).min(31); - let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); - Some( - MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL - .saturating_mul(multiplier) - .min(MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL), - ) - } - - fn record_inspection(&mut self, features: ScannerMaintenanceFeatures, now: Instant) { - if !features.inspection_failed { - self.reset(); - return; - } - - self.consecutive_failures = self.consecutive_failures.saturating_add(1); - self.retry_at = self.retry_interval().map(|interval| now + interval); - } - - fn retry_due(self, features: ScannerMaintenanceFeatures, wake_reason: ScannerCycleWakeReason, now: Instant) -> bool { - features.inspection_failed - && wake_reason == ScannerCycleWakeReason::Timer - && self.retry_at.is_some_and(|retry_at| now >= retry_at) - } -} - -fn scanner_cycle_observed_dirty_work( - pending_before_wait: bool, - generation_before_wait: u64, - generation_after_cycle: u64, -) -> bool { - pending_before_wait || generation_before_wait != generation_after_cycle -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct ScannerCycleWaitPlan { - effective_interval: Duration, - clean_idle_max_interval: Duration, - delay: Duration, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct ScannerCycleObservedGenerations { - dirty_usage: Option, - runtime_config: u64, - maintenance: u64, - defer_cluster_activity: bool, -} - -const LOCAL_SCANNER_ACTIVITY_NODE: &str = ""; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ScannerNodeActivity { - instance_id: String, - namespace_generation: u64, - maintenance_generation: u64, - protocol_version: u32, - topology_digest: [u8; 32], - data_movement_active: bool, - dirty_usage_generation: u64, - dirty_usage_pending: bool, -} - -pub(crate) type ScannerActivitySnapshot = BTreeMap; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct ScannerDirtyUsageAcknowledgement { - pub(crate) host: String, - pub(crate) instance_id: String, - pub(crate) generation: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ScannerActivityObservation { - NotRequired, - Unchanged, - Changed, - MaintenanceChanged, - Unverified, -} - -fn scanner_cycle_wait_plan( - runtime_config: &ScannerRuntimeConfig, - clean_idle_backoff: ScannerCleanIdleBackoff, - clean_idle_backoff_enabled: bool, - jitter: impl FnOnce(Duration) -> Duration, -) -> ScannerCycleWaitPlan { - let clean_idle_max_interval = scanner_clean_idle_max_interval(runtime_config.cycle_interval, runtime_config); - let effective_interval = - clean_idle_backoff.effective_interval(runtime_config.cycle_interval, clean_idle_max_interval, clean_idle_backoff_enabled); - let delay = cap_clean_idle_cycle_delay(jitter(effective_interval), clean_idle_max_interval, clean_idle_backoff_enabled); - - ScannerCycleWaitPlan { - effective_interval, - clean_idle_max_interval, - delay, - } -} - -fn record_scanner_cycle_result( - clean_idle_backoff: &mut ScannerCleanIdleBackoff, - runtime_config: &ScannerRuntimeConfig, - clean_idle_backoff_enabled: bool, - wake_reason: ScannerCycleWakeReason, - outcome: ScannerCycleOutcome, - dirty_work_observed: bool, -) { - clean_idle_backoff.record_cycle( - runtime_config.cycle_interval, - scanner_clean_idle_max_interval(runtime_config.cycle_interval, runtime_config), - clean_idle_backoff_enabled, - wake_reason, - outcome, - dirty_work_observed, - ); -} - -fn scanner_clean_idle_backoff_configured(runtime_config: &ScannerRuntimeConfig) -> bool { - let bitrot_cycle_allows_backoff = - runtime_config.bitrot_cycle.is_none() || runtime_config.bitrot_cycle_source == ScannerRuntimeConfigSource::Default; - runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default && bitrot_cycle_allows_backoff -} - -fn scanner_clean_idle_max_interval(base_interval: Duration, runtime_config: &ScannerRuntimeConfig) -> Duration { - let policy_max = CLEAN_IDLE_MAX_INTERVAL.max(base_interval); - let Some(bitrot_cycle) = runtime_config.bitrot_cycle else { - return policy_max; - }; - if runtime_config.bitrot_cycle_source != ScannerRuntimeConfigSource::Default { - return policy_max; - } - - let selection_window = heal_object_select_prob(); - if selection_window == 0 { - return policy_max; - } - - bitrot_cycle - .checked_div(selection_window) - .unwrap_or(base_interval) - .max(base_interval) - .min(policy_max) -} - -fn scanner_clean_idle_backoff_enabled( - topology_supported: bool, - cluster_activity_ready: bool, - features: ScannerMaintenanceFeatures, - runtime_config: &ScannerRuntimeConfig, -) -> bool { - topology_supported - && cluster_activity_ready - && !features.needs_regular_cycle() - && scanner_clean_idle_backoff_configured(runtime_config) -} - -fn scanner_activity_probe_required( - topology_supported: bool, - backoff_blocked: bool, - features: ScannerMaintenanceFeatures, - runtime_config: &ScannerRuntimeConfig, -) -> bool { - topology_supported - && !backoff_blocked - && !features.needs_regular_cycle() - && scanner_clean_idle_backoff_configured(runtime_config) -} - -fn scanner_activity_observed_work(observation: ScannerActivityObservation) -> bool { - matches!( - observation, - ScannerActivityObservation::Changed - | ScannerActivityObservation::MaintenanceChanged - | ScannerActivityObservation::Unverified - ) -} - -fn scanner_activity_backoff_blocked_after_wake(currently_blocked: bool, wake_reason: ScannerCycleWakeReason) -> bool { - match wake_reason { - ScannerCycleWakeReason::ClusterMaintenance => true, - ScannerCycleWakeReason::MaintenanceConfig => false, - _ => currently_blocked, - } -} - -async fn wait_for_next_scanner_cycle( - ctx: &CancellationToken, - delay: Duration, - dirty_usage_generation_seen: Option, - runtime_config_generation: u64, - maintenance_generation: u64, - is_lock_lost: F, -) -> ScannerCycleWakeReason -where - F: Fn() -> bool, -{ - let sleep = tokio::time::sleep(delay); - tokio::pin!(sleep); - let lock_poll = tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL); - tokio::pin!(lock_poll); - - loop { - if is_lock_lost() { - return ScannerCycleWakeReason::LeaderLockLost; - } - if scanner_runtime_config_generation() != runtime_config_generation { - return ScannerCycleWakeReason::RuntimeConfig; - } - if scanner_maintenance_generation() != maintenance_generation { - return ScannerCycleWakeReason::MaintenanceConfig; - } - if dirty_usage_generation_seen.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) { - return ScannerCycleWakeReason::DirtyUsage; - } - - tokio::select! { - _ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled, - _ = &mut sleep => return ScannerCycleWakeReason::Timer, - _ = &mut lock_poll => { - if is_lock_lost() { - return ScannerCycleWakeReason::LeaderLockLost; - } - lock_poll.as_mut().reset(Instant::now() + SCANNER_LEADER_LOCK_POLL_INTERVAL); - } - _ = dirty_usage_bucket_notified() => { - if scanner_runtime_config_generation() != runtime_config_generation { - return ScannerCycleWakeReason::RuntimeConfig; - } - if scanner_maintenance_generation() != maintenance_generation { - return ScannerCycleWakeReason::MaintenanceConfig; - } - if dirty_usage_generation_seen - .is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) - { - return ScannerCycleWakeReason::DirtyUsage; - } - } - _ = scanner_runtime_config_changed() => { - if scanner_runtime_config_generation() != runtime_config_generation { - return ScannerCycleWakeReason::RuntimeConfig; - } - } - _ = scanner_maintenance_changed() => { - if scanner_maintenance_generation() != maintenance_generation { - return ScannerCycleWakeReason::MaintenanceConfig; - } - } - } - } -} - -async fn wait_for_next_scanner_cycle_with_activity( - ctx: &CancellationToken, - delay: Duration, - activity_poll_interval: Option, - activity_seen: &mut Option, - generations: ScannerCycleObservedGenerations, - is_lock_lost: F, - mut probe_activity: Probe, -) -> ScannerCycleWakeReason -where - F: Fn() -> bool, - Probe: FnMut() -> ProbeFuture, - ProbeFuture: Future>, -{ - let deadline = Instant::now() + delay; - loop { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return ScannerCycleWakeReason::Timer; - } - let wait_slice = activity_poll_interval - .map(|interval| interval.max(Duration::from_secs(1)).min(remaining)) - .unwrap_or(remaining); - let wake_reason = wait_for_next_scanner_cycle( - ctx, - wait_slice, - generations.dirty_usage, - generations.runtime_config, - generations.maintenance, - &is_lock_lost, - ) - .await; - if wake_reason != ScannerCycleWakeReason::Timer || Instant::now() >= deadline { - return wake_reason; - } - - let Some(_) = activity_poll_interval else { - return ScannerCycleWakeReason::Timer; - }; - if is_lock_lost() { - return ScannerCycleWakeReason::LeaderLockLost; - } - - let probe = probe_activity(); - tokio::pin!(probe); - let lock_lost = async { - loop { - tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL).await; - if is_lock_lost() { - break; - } - } - }; - tokio::pin!(lock_lost); - let probe_result = tokio::select! { - result = &mut probe => result, - _ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled, - _ = &mut lock_lost => return ScannerCycleWakeReason::LeaderLockLost, - }; - - let had_baseline = activity_seen.is_some(); - let (observation, probe_error) = apply_scanner_activity_probe_result(activity_seen, probe_result); - if let Some(err) = probe_error { - log_scanner_activity_probe_error(had_baseline, &err); - } - match observation { - ScannerActivityObservation::Unchanged | ScannerActivityObservation::NotRequired => {} - ScannerActivityObservation::Changed if !generations.defer_cluster_activity => { - return ScannerCycleWakeReason::ClusterActivity; - } - ScannerActivityObservation::Changed => {} - ScannerActivityObservation::MaintenanceChanged => return ScannerCycleWakeReason::ClusterMaintenance, - ScannerActivityObservation::Unverified if !generations.defer_cluster_activity => { - return ScannerCycleWakeReason::ClusterActivityUnavailable; - } - ScannerActivityObservation::Unverified => {} - } - } -} - -fn log_scanner_activity_probe_error(had_baseline: bool, err: &str) { - if had_baseline { - warn!( - target: "rustfs::scanner", - event = EVENT_SCANNER_CYCLE_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - state = "cluster_activity_probe_failed", - error = %err, - "Scanner cluster activity probe failed; preserving the base cycle" - ); - } else { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_CYCLE_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - state = "cluster_activity_probe_unavailable", - error = %err, - "Scanner cluster activity probe remains unavailable" - ); - } -} - -fn compare_scanner_activity(previous: &ScannerActivitySnapshot, current: &ScannerActivitySnapshot) -> ScannerActivityObservation { - if previous == current { - return ScannerActivityObservation::Unchanged; - } - - for (host, current_activity) in current { - let Some(previous_activity) = previous.get(host) else { - continue; - }; - if host != LOCAL_SCANNER_ACTIVITY_NODE - && previous_activity.instance_id == current_activity.instance_id - && previous_activity.maintenance_generation != current_activity.maintenance_generation - { - return ScannerActivityObservation::MaintenanceChanged; - } - } - - ScannerActivityObservation::Changed -} - -fn apply_scanner_activity_probe_result( - activity_seen: &mut Option, - result: Result, -) -> (ScannerActivityObservation, Option) { - match result { - Ok(current) => { - let observation = match activity_seen.as_ref() { - Some(previous) => compare_scanner_activity(previous, ¤t), - None => ScannerActivityObservation::Unverified, - }; - *activity_seen = Some(current); - (observation, None) - } - Err(err) => { - *activity_seen = None; - (ScannerActivityObservation::Unverified, Some(err)) - } - } -} - -async fn observe_scanner_activity( - storeapi: &Arc, - distributed: bool, - activity_seen: &mut Option, -) -> ScannerActivityObservation { - let had_baseline = activity_seen.is_some(); - let (observation, probe_error) = - apply_scanner_activity_probe_result(activity_seen, probe_scanner_activity(storeapi, distributed).await); - if let Some(err) = probe_error { - log_scanner_activity_probe_error(had_baseline, &err); - } - observation -} - -pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { - let mut hasher = Sha256::new(); - hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); - for (host, activity) in snapshot { - let host = host.as_bytes(); - let instance_id = activity.instance_id.as_bytes(); - hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes()); - hasher.update(host); - hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes()); - hasher.update(instance_id); - hasher.update(activity.namespace_generation.to_be_bytes()); - hasher.update(activity.maintenance_generation.to_be_bytes()); - hasher.update(activity.protocol_version.to_be_bytes()); - hasher.update(activity.topology_digest); - hasher.update([u8::from(activity.data_movement_active)]); - hasher.update(activity.dirty_usage_generation.to_be_bytes()); - hasher.update([u8::from(activity.dirty_usage_pending)]); - } - hasher.finalize().into() -} - -pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool { - snapshot.values().all(|activity| !activity.data_movement_active) -} - -pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySnapshot) -> Vec { - snapshot - .iter() - .filter(|(host, activity)| host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE && activity.dirty_usage_pending) - .map(|(host, activity)| ScannerDirtyUsageAcknowledgement { - host: host.clone(), - instance_id: activity.instance_id.clone(), - generation: activity.dirty_usage_generation, - }) - .collect() -} - -pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] { - let endpoint_pools = storeapi.endpoints(); - let mut hasher = Sha256::new(); - hasher.update(u64::try_from(endpoint_pools.0.len()).unwrap_or(u64::MAX).to_be_bytes()); - for (pool_index, pool) in endpoint_pools.0.iter().enumerate() { - hasher.update(u64::try_from(pool_index).unwrap_or(u64::MAX).to_be_bytes()); - hasher.update(u64::try_from(pool.set_count).unwrap_or(u64::MAX).to_be_bytes()); - hasher.update(u64::try_from(pool.drives_per_set).unwrap_or(u64::MAX).to_be_bytes()); - let mut endpoints = pool.endpoints.as_ref().iter().collect::>(); - endpoints.sort_unstable_by(|left, right| { - (left.pool_idx, left.set_idx, left.disk_idx, left.url.as_str()).cmp(&( - right.pool_idx, - right.set_idx, - right.disk_idx, - right.url.as_str(), - )) - }); - hasher.update(u64::try_from(endpoints.len()).unwrap_or(u64::MAX).to_be_bytes()); - for endpoint in endpoints { - hasher.update(endpoint.pool_idx.to_be_bytes()); - hasher.update(endpoint.set_idx.to_be_bytes()); - hasher.update(endpoint.disk_idx.to_be_bytes()); - let url = endpoint.url.as_str().as_bytes(); - hasher.update(u64::try_from(url.len()).unwrap_or(u64::MAX).to_be_bytes()); - hasher.update(url); - } - } - hasher.finalize().into() -} - -fn record_scanner_activity_instance( - instance_hosts: &mut BTreeMap, - host: &str, - instance_id: &str, -) -> Result<(), String> { - if let Some(existing_host) = instance_hosts.insert(instance_id.to_string(), host.to_string()) { - return Err(format!( - "scanner activity peers {existing_host} and {host} report the same process instance" - )); - } - Ok(()) -} - -pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool) -> Result { - let topology_digest = scanner_topology_digest(storeapi); - let data_movement_active = storeapi.scanner_data_movement_active().await; - let namespace_generation = storeapi.scanner_namespace_mutation_generation(); - let maintenance_generation = scanner_maintenance_generation(); - let dirty_usage = scanner_dirty_usage_state(); - if namespace_generation == u64::MAX || maintenance_generation == u64::MAX || dirty_usage.generation == u64::MAX { - return Err("local scanner activity generation is exhausted".to_string()); - } - let local_instance_id = crate::scanner_io::scanner_activity_epoch().to_string(); - let mut instance_hosts = BTreeMap::from([(local_instance_id.clone(), LOCAL_SCANNER_ACTIVITY_NODE.to_string())]); - let mut snapshot = ScannerActivitySnapshot::from([( - LOCAL_SCANNER_ACTIVITY_NODE.to_string(), - ScannerNodeActivity { - instance_id: local_instance_id, - namespace_generation, - maintenance_generation, - protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, - topology_digest, - data_movement_active, - dirty_usage_generation: dirty_usage.generation, - dirty_usage_pending: dirty_usage.pending, - }, - )]); - if !distributed { - return Ok(snapshot); - } - - let notification_system = storeapi - .notification_system() - .ok_or_else(|| "notification system is not initialized".to_string())?; - let peers = notification_system - .scanner_activity_snapshots() - .await - .map_err(|err| err.to_string())?; - for (host, activity) in peers { - if activity.namespace_generation == u64::MAX || activity.maintenance_generation == u64::MAX { - return Err(format!("scanner activity peer {host} exhausted its activity generation")); - } - let (peer_topology_digest, peer_data_movement_active, peer_dirty_usage_generation, peer_dirty_usage_pending) = - match activity.protocol_version { - SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => { - return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing")); - } - SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { - return Err(format!( - "scanner activity peer {host} cannot safely share scanner cache locks with protocol {}", - SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION - )); - } - SCANNER_ACTIVITY_PROTOCOL_VERSION => ( - activity - .topology_digest - .ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?, - activity - .data_movement_active - .ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?, - activity - .dirty_usage_generation - .ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?, - activity - .dirty_usage_pending - .ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?, - ), - version => { - return Err(format!( - "scanner activity peer {host} uses protocol {version}, expected {}", - SCANNER_ACTIVITY_PROTOCOL_VERSION - )); - } - }; - if peer_dirty_usage_generation == u64::MAX { - return Err(format!("scanner activity peer {host} exhausted its dirty usage generation")); - } - if peer_topology_digest != topology_digest { - return Err(format!("scanner activity peer {host} has a different storage topology")); - } - record_scanner_activity_instance(&mut instance_hosts, &host, &activity.instance_id)?; - if snapshot - .insert( - host.clone(), - ScannerNodeActivity { - instance_id: activity.instance_id, - namespace_generation: activity.namespace_generation, - maintenance_generation: activity.maintenance_generation, - protocol_version: activity.protocol_version, - topology_digest: peer_topology_digest, - data_movement_active: peer_data_movement_active, - dirty_usage_generation: peer_dirty_usage_generation, - dirty_usage_pending: peer_dirty_usage_pending, - }, - ) - .is_some() - { - return Err(format!("duplicate scanner activity peer: {host}")); - } - } - Ok(snapshot) -} - fn initial_scanner_delay_for(start_delay_secs: Option) -> Duration { start_delay_secs .map(|secs| randomized_cycle_delay_for(Duration::from_secs(secs))) @@ -1734,100 +972,6 @@ fn retain_recent_cycle_completions(cycle_completed: &mut Vec>) { } } -/// Background healing information -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct BackgroundHealInfo { - /// Bitrot scan start time - pub bitrot_start_time: Option>, - /// Bitrot scan start cycle - pub bitrot_start_cycle: u64, - /// Current scan mode - pub current_scan_mode: HealScanMode, -} - -/// Read background healing information from storage -pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHealInfo { - // Skip for ErasureSD setup - if scanner_is_erasure_sd().await { - return BackgroundHealInfo::default(); - } - - // Get last healing information - match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await { - Ok(buf) => serde_json::from_slice::(&buf).unwrap_or_else(|e| { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, - path = %&*BACKGROUND_HEAL_INFO_PATH, - state = "decode_failed", - error = %e, - "Scanner background heal decode failed" - ); - BackgroundHealInfo::default() - }), - Err(e) => { - // Only log if it's not a ConfigNotFound error - if e != EcstoreError::ConfigNotFound { - warn!( - target: "rustfs::scanner", - event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, - path = %&*BACKGROUND_HEAL_INFO_PATH, - state = "read_failed", - error = %e, - "Scanner background heal read failed" - ); - } - BackgroundHealInfo::default() - } - } -} - -/// Save background healing information to storage -#[instrument(skip(storeapi))] -pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundHealInfo) { - // Skip for ErasureSD setup - if scanner_is_erasure_sd().await { - return; - } - - // Serialize to JSON - let data = match serde_json::to_vec(&info) { - Ok(data) => data, - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, - path = %&*BACKGROUND_HEAL_INFO_PATH, - state = "encode_failed", - error = %e, - "Scanner background heal encode failed" - ); - return; - } - }; - - // Save configuration - if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await { - warn!( - target: "rustfs::scanner", - event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, - path = %&*BACKGROUND_HEAL_INFO_PATH, - state = "save_failed", - error = %e, - "Scanner background heal save failed" - ); - } -} - /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds) /// Defaults to 5 seconds if not set or invalid /// For distributed environments with multiple nodes, a longer timeout may be needed @@ -1845,800 +989,6 @@ async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard cycle_metrics_guard.finish(cycle_info.clone()).await; } -fn encode_scanner_cycle_state(cycle_info: &CurrentCycle, leader_epoch: u64) -> Result, ScannerCycleStateError> { - if cycle_info.next == u64::MAX { - return Err(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted")); - } - let cycle_info_buf = rmp_serde::to_vec(cycle_info)?; - let mut buf = Vec::with_capacity(cycle_info_buf.len() + SCANNER_CYCLE_STATE_HEADER_LEN); - buf.extend_from_slice(&cycle_info.next.to_le_bytes()); - buf.extend_from_slice(SCANNER_CYCLE_STATE_MAGIC); - buf.extend_from_slice(&leader_epoch.to_le_bytes()); - buf.extend_from_slice(&cycle_info_buf); - Ok(buf) -} - -fn decode_scanner_cycle_state(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> { - if buf.len() < 8 { - return Err(ScannerCycleStateError::InvalidData("scanner cycle state is truncated")); - } - - let persisted_next = u64::from_le_bytes( - buf[0..8] - .try_into() - .map_err(|_| ScannerCycleStateError::InvalidData("scanner cycle counter is truncated"))?, - ); - if persisted_next == u64::MAX { - return Err(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted")); - } - if buf.len() == 8 { - return Ok(( - CurrentCycle { - next: persisted_next, - ..Default::default() - }, - 0, - )); - } - - let (leader_epoch, payload) = if buf.len() >= 16 && &buf[8..16] == SCANNER_CYCLE_STATE_MAGIC { - if buf.len() < SCANNER_CYCLE_STATE_HEADER_LEN { - return Err(ScannerCycleStateError::InvalidData("scanner cycle fencing header is truncated")); - } - let epoch = u64::from_le_bytes( - buf[16..24] - .try_into() - .map_err(|_| ScannerCycleStateError::InvalidData("scanner leader epoch is truncated"))?, - ); - if epoch == 0 { - return Err(ScannerCycleStateError::InvalidData("scanner leader epoch is zero")); - } - (epoch, &buf[SCANNER_CYCLE_STATE_HEADER_LEN..]) - } else { - (0, &buf[8..]) - }; - - let cycle_info = rmp_serde::from_slice::(payload)?; - if cycle_info.next != persisted_next { - return Err(ScannerCycleStateError::InvalidData("scanner cycle counter disagrees with encoded state")); - } - Ok((cycle_info, leader_epoch)) -} - -pub(crate) fn decode_persisted_scanner_cycle_fence(buf: &[u8]) -> Result<(u64, u64), ScannerError> { - decode_scanner_cycle_state(buf) - .map(|(cycle, leader_epoch)| (cycle.next, leader_epoch)) - .map_err(|err| ScannerError::Other(format!("persisted scanner cycle state is invalid: {err}"))) -} - -#[cfg(test)] -pub(crate) fn encode_scanner_cycle_fence_for_test(next_cycle: u64, leader_epoch: u64) -> Vec { - encode_scanner_cycle_state( - &CurrentCycle { - next: next_cycle, - ..Default::default() - }, - leader_epoch, - ) - .expect("test scanner cycle fence should encode") -} - -pub(crate) async fn current_scanner_leader_epoch() -> Result { - let store = crate::resolve_scanner_object_store_handle() - .ok_or_else(|| ScannerError::Other("scanner object layer is unavailable".to_string()))?; - match read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await { - Ok(buf) => { - let (_, leader_epoch) = decode_persisted_scanner_cycle_fence(&buf)?; - if leader_epoch == 0 { - return Err(ScannerError::Other("persisted scanner cycle state has no leader epoch".to_string())); - } - Ok(leader_epoch) - } - Err(err) => Err(ScannerError::Other(format!("failed to read persisted scanner leader epoch: {err}"))), - } -} - -fn decode_scanner_cycle_state_for_startup(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> { - if buf.is_empty() { - Ok((CurrentCycle::default(), 0)) - } else { - decode_scanner_cycle_state(buf) - } -} - -fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(), ScannerCycleStateError> { - let next = cycle_info - .next - .checked_add(1) - .filter(|next| *next < u64::MAX) - .ok_or(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted"))?; - cycle_info.next = next; - Ok(()) -} - -async fn persisted_usage_floor(storeapi: Arc) -> Result { - let mut floor = PersistedUsageFloor::default(); - let update_floor = |floor: &mut PersistedUsageFloor, usage: DataUsageInfo, path: &str| -> Result<(), ScannerError> { - floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); - if let Some(completed_cycle) = usage.scanner_cycle { - let next_cycle = completed_cycle - .checked_add(1) - .filter(|next| *next < u64::MAX) - .ok_or_else(|| ScannerError::Other(format!("persisted scanner usage cycle is exhausted in {path}")))?; - floor.next_cycle = floor.next_cycle.max(next_cycle); - } - Ok(()) - }; - for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] { - let backup_path = format!("{primary_path}.bkp"); - let mut pair_found = false; - for path in [primary_path, backup_path.as_str()] { - let data = match read_config(storeapi.clone(), path).await { - Ok(data) => { - pair_found = true; - data - } - Err(EcstoreError::ConfigNotFound) => continue, - Err(err) => { - return Err(ScannerError::Other(format!( - "failed to read scanner usage epoch floor from {path}: {err}" - ))); - } - }; - let usage = serde_json::from_slice::(&data) - .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage floor from {path}: {err}")))?; - update_floor(&mut floor, usage, path)?; - } - if pair_found { - break; - } - } - Ok(floor) -} - -fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_epoch: &mut u64, floor: PersistedUsageFloor) { - cycle_info.next = cycle_info.next.max(floor.next_cycle); - *leader_epoch = (*leader_epoch).max(floor.leader_epoch); -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ScannerLeadershipClaimReconcile { - Durable, - Changed, - Unchanged, -} - -async fn reconcile_scanner_leadership_claim( - storeapi: Arc, - attempted: &[u8], - previous_revision: &DataUsageCacheRevision, - claimed_epoch: u64, - cycle_info: &mut CurrentCycle, - revision: &mut DataUsageCacheRevision, - persisted_epoch: &mut u64, -) -> Result { - let (persisted, persisted_revision) = read_config_with_revision(storeapi, DATA_USAGE_BLOOM_NAME_PATH.as_str()) - .await - .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner leadership claim: {err}")))?; - let revision_changed = &persisted_revision != previous_revision; - *revision = persisted_revision; - - let Some(persisted) = persisted else { - *cycle_info = CurrentCycle::default(); - return Ok(if revision_changed { - ScannerLeadershipClaimReconcile::Changed - } else { - ScannerLeadershipClaimReconcile::Unchanged - }); - }; - if persisted == attempted { - *persisted_epoch = claimed_epoch; - return Ok(ScannerLeadershipClaimReconcile::Durable); - } - - let (current, epoch) = decode_scanner_cycle_state(&persisted) - .map_err(|err| ScannerError::Other(format!("scanner leadership conflict winner is invalid: {err}")))?; - *cycle_info = current; - *persisted_epoch = (*persisted_epoch).max(epoch); - Ok(if revision_changed { - ScannerLeadershipClaimReconcile::Changed - } else { - ScannerLeadershipClaimReconcile::Unchanged - }) -} - -fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result { - serde_json::from_slice(data) - .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}"))) -} - -async fn usage_snapshot_for_epoch_fence( - storeapi: Arc, - primary: Option<&[u8]>, -) -> Result { - if let Some(primary) = primary { - return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()); - } - - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - let (backup, _) = read_config_with_revision(storeapi.clone(), &backup_path) - .await - .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?; - if let Some(backup) = backup.as_deref() { - return decode_usage_snapshot_for_epoch_fence(backup, &backup_path); - } - - for path in [ - LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), - format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), - ] { - let (legacy, _) = read_config_with_revision(storeapi.clone(), &path) - .await - .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; - if let Some(legacy) = legacy.as_deref() { - return decode_usage_snapshot_for_epoch_fence(legacy, &path); - } - } - Ok(DataUsageInfo::default()) -} - -async fn fence_scanner_usage_epoch( - ctx: &CancellationToken, - storeapi: Arc, - claimed_epoch: u64, -) -> Result<(), ScannerError> { - for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { - if ctx.is_cancelled() { - return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); - } - - let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) - .await - .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?; - let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?; - match usage.scanner_epoch { - Some(epoch) if epoch > claimed_epoch => { - return Err(ScannerError::Other(format!( - "scanner usage epoch fence lost to newer leader: claimed={claimed_epoch}, persisted={epoch}" - ))); - } - Some(epoch) if epoch == claimed_epoch => return Ok(()), - Some(_) | None => {} - } - usage.scanner_epoch = Some(claimed_epoch); - let data = serde_json::to_vec(&usage) - .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; - - let save_result = - save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) - .await; - if save_result - .as_ref() - .ok() - .and_then(|object_info| object_info.etag.as_deref()) - .is_some_and(|etag| !etag.is_empty()) - { - return Ok(()); - } - - let (persisted, persisted_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) - .await - .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage epoch fence: {err}")))?; - if let Some(persisted) = persisted { - let persisted = decode_usage_snapshot_for_epoch_fence(&persisted, DATA_USAGE_OBJ_NAME_PATH.as_str())?; - match persisted.scanner_epoch { - Some(epoch) if epoch == claimed_epoch => return Ok(()), - Some(epoch) if epoch > claimed_epoch => { - return Err(ScannerError::Other(format!( - "scanner usage epoch fence lost to newer leader: claimed={claimed_epoch}, persisted={epoch}" - ))); - } - Some(_) | None => {} - } - } - - let precondition_failed = matches!(save_result, Err(EcstoreError::PreconditionFailed)); - if retry < SCANNER_PERSIST_CAS_RETRIES && (precondition_failed || persisted_revision != revision) { - continue; - } - return Err(ScannerError::Other(match save_result { - Ok(_) => "scanner usage epoch fence returned no ETag and could not be confirmed".to_string(), - Err(err) => format!("scanner usage epoch fence save failed: {err}"), - })); - } - - Err(ScannerError::Other("scanner usage epoch fence retries exhausted".to_string())) -} - -async fn complete_scanner_leadership_claim( - ctx: &CancellationToken, - storeapi: Arc, - claimed_epoch: u64, -) -> bool { - if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), - state = "usage_epoch_fence_failed", - claimed_epoch, - error = %err, - "Scanner leadership usage epoch fencing failed" - ); - return false; - } - !ctx.is_cancelled() -} - -async fn claim_scanner_leadership( - ctx: &CancellationToken, - storeapi: Arc, - cycle_info: &mut CurrentCycle, - revision: &mut DataUsageCacheRevision, - persisted_epoch: &mut u64, -) -> bool { - for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { - if ctx.is_cancelled() { - return false; - } - let Some(claimed_epoch) = persisted_epoch.checked_add(1) else { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "leader_epoch_exhausted", - "Scanner leadership epoch is exhausted" - ); - return false; - }; - let data = match encode_scanner_cycle_state(cycle_info, claimed_epoch) { - Ok(data) => data, - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "leader_claim_encode_failed", - error = %err, - "Scanner leadership claim encoding failed" - ); - return false; - } - }; - let previous_revision = revision.clone(); - - let save_result = - save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions()) - .await; - match save_result { - Ok(object_info) => { - if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) { - *revision = DataUsageCacheRevision::Etag(etag); - *persisted_epoch = claimed_epoch; - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; - } - - match reconcile_scanner_leadership_claim( - storeapi.clone(), - &data, - &previous_revision, - claimed_epoch, - cycle_info, - revision, - persisted_epoch, - ) - .await - { - Ok(ScannerLeadershipClaimReconcile::Durable) => { - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; - } - Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue, - Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "leader_claim_missing_revision", - "Scanner leadership claim returned no ETag and could not be confirmed" - ); - return false; - } - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "leader_claim_reconcile_failed", - error = %err, - "Scanner leadership claim read-back failed" - ); - return false; - } - } - } - Err(err) => { - let precondition_failed = matches!(err, EcstoreError::PreconditionFailed); - match reconcile_scanner_leadership_claim( - storeapi.clone(), - &data, - &previous_revision, - claimed_epoch, - cycle_info, - revision, - persisted_epoch, - ) - .await - { - Ok(ScannerLeadershipClaimReconcile::Durable) => { - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; - } - Ok(ScannerLeadershipClaimReconcile::Changed) - if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => - { - continue; - } - Ok(ScannerLeadershipClaimReconcile::Unchanged) - if precondition_failed && retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => - { - continue; - } - Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = if precondition_failed { - "leader_claim_conflicts_exhausted" - } else { - "leader_claim_failed" - }, - error = %err, - "Scanner leadership claim failed" - ); - return false; - } - Err(reconcile_err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "leader_claim_reload_failed", - error = %reconcile_err, - save_error = %err, - "Scanner leadership claim reconciliation failed" - ); - return false; - } - } - } - } - } - - false -} - -async fn persist_scanner_cycle_state( - ctx: &CancellationToken, - storeapi: Arc, - cycle_info: &mut CurrentCycle, - revision: &mut DataUsageCacheRevision, - leader_epoch: u64, -) -> bool { - let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) { - Ok(buf) => buf, - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "encode_failed", - error = %e, - "Scanner state encoding failed" - ); - return false; - } - }; - - for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { - if ctx.is_cancelled() { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "cancelled_before_save", - retry, - "Scanner state persistence cancelled by the leader fence" - ); - return false; - } - - #[cfg(test)] - notify_scanner_cycle_state_persist_test_hook(leader_epoch); - match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) - .await - { - Ok(object_info) => { - let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "missing_revision", - "Scanner state save returned no ETag" - ); - return false; - }; - *revision = DataUsageCacheRevision::Etag(etag); - if ctx.is_cancelled() { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "cancelled_after_save", - retry, - "Scanner state save completed after the leader fence was cancelled" - ); - return false; - } - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "saved", - "Scanner state saved" - ); - return true; - } - Err(EcstoreError::PreconditionFailed) => { - let (persisted, persisted_revision) = - match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await { - Ok(result) => result, - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "conflict_reload_failed", - error = %e, - "Scanner state conflict reconciliation failed" - ); - return false; - } - }; - *revision = persisted_revision; - if ctx.is_cancelled() { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "cancelled_after_conflict", - retry, - "Scanner state conflict reconciliation cancelled by the leader fence" - ); - return false; - } - - if let Some(persisted) = persisted { - if persisted.len() < 8 { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "conflict_state_invalid", - length = persisted.len(), - "Scanner state conflict winner is truncated" - ); - return false; - } - - let (persisted_cycle, persisted_epoch) = match decode_scanner_cycle_state(&persisted) { - Ok(state) => state, - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "conflict_state_decode_failed", - error = %e, - "Scanner state conflict winner could not be decoded" - ); - return false; - } - }; - if persisted_epoch != leader_epoch { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "leader_epoch_fenced", - expected_epoch = leader_epoch, - persisted_epoch, - "Scanner state save rejected by a newer leadership epoch" - ); - return false; - } - - if persisted_cycle.next >= cycle_info.next { - *cycle_info = persisted_cycle; - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "conflict_reconciled", - retry, - "Scanner state adopted the current persisted cycle" - ); - return true; - } - } - - if retry < SCANNER_PERSIST_CAS_RETRIES { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "conflict_retry", - retry = retry + 1, - "Scanner state CAS conflict will be retried" - ); - continue; - } - - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "conflict_retries_exhausted", - retries = SCANNER_PERSIST_CAS_RETRIES, - "Scanner state CAS conflict retries exhausted" - ); - return false; - } - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %&*DATA_USAGE_BLOOM_NAME_PATH, - state = "failed", - error = %e, - "Scanner state persistence failed" - ); - return false; - } - } - } - - false -} - -async fn finalize_partial_scan_cycle( - ctx: &CancellationToken, - storeapi: Arc, - cycle_info: &mut CurrentCycle, - revision: &mut DataUsageCacheRevision, - leader_epoch: u64, - cycle_metrics_guard: &mut ScannerCycleMetricsGuard, -) -> bool { - // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter - // must still advance (and persist) because per-bucket next_cycle is stamped from - // it and compacted folders are only rescanned when their hash matches - // next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle - // expiry and usage refresh on every folder outside the stuck window. - if let Err(err) = advance_scanner_cycle(cycle_info) { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - state = "cycle_counter_exhausted", - error = %err, - "Scanner partial cycle could not advance" - ); - mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await; - return false; - } - cycle_info.current = 0; - global_metrics().clear_current_scan_mode(); - let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; - cycle_metrics_guard.finish(cycle_info.clone()).await; - persisted -} - -async fn persist_required_scanner_cycle_floor( - ctx: &CancellationToken, - storeapi: Arc, - cycle_info: &mut CurrentCycle, - revision: &mut DataUsageCacheRevision, - leader_epoch: u64, - required_cycle: u64, - cycle_metrics_guard: &mut ScannerCycleMetricsGuard, -) -> bool { - if required_cycle <= cycle_info.current || required_cycle == u64::MAX { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - current_cycle = cycle_info.current, - required_cycle, - state = "invalid_cache_cycle_floor", - "Scanner cache cycle floor is invalid" - ); - mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await; - return false; - } - - cycle_info.next = cycle_info.next.max(required_cycle); - cycle_info.current = 0; - global_metrics().clear_current_scan_mode(); - let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; - cycle_metrics_guard.finish(cycle_info.clone()).await; - persisted -} - -async fn await_scanner_cycle_with_lock_fence( - cycle_ctx: &CancellationToken, - cycle: Cycle, - lock_lost: LockLost, -) -> Option -where - Cycle: Future, - LockLost: Future, -{ - tokio::pin!(cycle); - tokio::pin!(lock_lost); - tokio::select! { - biased; - _ = &mut lock_lost => { - cycle_ctx.cancel(); - tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await.ok() - } - output = &mut cycle => Some(output), - } -} - #[instrument(skip_all)] #[hotpath::measure] async fn run_data_scanner_cycle( @@ -3121,24 +1471,6 @@ impl Drop for ScannerCycleMetricsGuard { } } -async fn record_scanner_leader_lock_lost(message: &'static str) { - reset_scanner_cycle_schedule(); - record_scanner_leader_lock_state("lost"); - global_metrics() - .record_scanner_leader_liveness("lost", false, "leader lock refresh quorum lost") - .await; - warn!( - target: "rustfs::scanner", - event = EVENT_SCANNER_LOCK_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - lock_name = "leader.lock", - state = "lost", - reason = message, - "Scanner leader lock lost" - ); -} - pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> Result<(), ScannerError> { let (maintenance_features, maintenance_generation) = configure_scanner_defaults(&ctx, &storeapi).await; run_data_scanner_with_maintenance_state(ctx, storeapi, maintenance_features, maintenance_generation).await @@ -3669,55 +2001,6 @@ impl Drop for ScannerScanModeGuard { } } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -enum DataUsagePersistOutcome { - #[default] - NoUpdate, - Current, - AlreadyDurable, - PriorCycleDurable, - Saved, - Failed, -} - -#[derive(Clone, Debug)] -struct DataUsagePersistBaseline { - data: Option, - revision: DataUsageCacheRevision, -} - -#[derive(Debug)] -enum DataUsagePersistTaskResult { - Completed(DataUsagePersistOutcome), - Cancelled, - TimedOut, - JoinFailed(tokio::task::JoinError), -} - -async fn wait_for_data_usage_persist_task( - ctx: &CancellationToken, - task: &mut AbortOnDropHandle, - timeout: Duration, -) -> DataUsagePersistTaskResult { - tokio::select! { - biased; - result = &mut *task => match result { - Ok(outcome) => DataUsagePersistTaskResult::Completed(outcome), - Err(err) => DataUsagePersistTaskResult::JoinFailed(err), - }, - _ = ctx.cancelled() => { - task.abort(); - let _ = (&mut *task).await; - DataUsagePersistTaskResult::Cancelled - }, - _ = tokio::time::sleep(timeout) => { - task.abort(); - let _ = (&mut *task).await; - DataUsagePersistTaskResult::TimedOut - } - } -} - fn scanner_cycle_completion_outcome( scan_status: ScannerCycleStatus, usage_persist_outcome: DataUsagePersistOutcome, @@ -3830,4349 +2113,28 @@ fn data_usage_reintroduces_missing_bucket(incoming: &DataUsageInfo, existing: Op } /// Store data usage info in backend. Will store all objects sent on the receiver until closed. -#[instrument(skip(ctx, storeapi))] -pub async fn store_data_usage_in_backend( - ctx: CancellationToken, - storeapi: Arc, - receiver: mpsc::Receiver, -) { - let _ = store_data_usage_in_backend_with_outcome(ctx, storeapi, receiver).await; -} +mod activity; +mod cycle_state; +mod heal_info; +mod leadership; +mod usage_store; -async fn store_data_usage_in_backend_with_outcome( - ctx: CancellationToken, - storeapi: Arc, - receiver: mpsc::Receiver, -) -> DataUsagePersistOutcome { - store_data_usage_in_backend_with_outcome_for_epoch(ctx, storeapi, receiver, None).await -} +use activity::*; +use cycle_state::*; +use leadership::*; +use usage_store::*; -async fn store_data_usage_in_backend_with_outcome_for_epoch( - ctx: CancellationToken, - storeapi: Arc, - receiver: mpsc::Receiver, - leader_epoch: Option, -) -> DataUsagePersistOutcome { - store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(ctx, storeapi, receiver, leader_epoch, None).await -} - -async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline( - ctx: CancellationToken, - storeapi: Arc, - mut receiver: mpsc::Receiver, - leader_epoch: Option, - initial_baseline: Option, -) -> DataUsagePersistOutcome { - let mut outcome = DataUsagePersistOutcome::NoUpdate; - let mut next_baseline = initial_baseline; - - 'updates: while let Some(mut data_usage_info) = receiver.recv().await { - let _activity_guard = ScannerActivityGuard::new(); - if ctx.is_cancelled() { - break; - } - if let Some(leader_epoch) = leader_epoch { - data_usage_info.scanner_epoch = Some(leader_epoch); - } - let observational = data_usage_info.usage_snapshot_converged == Some(false); - let target_path = if observational { - DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() - } else { - DATA_USAGE_OBJ_NAME_PATH.as_str() - }; - - if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() { - let authoritative_data = match next_baseline.as_ref() { - Some(baseline) => baseline.data.clone(), - None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { - Ok((data, _)) => data.map(Bytes::from), - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), - state = "observed_baseline_load_failed", - error = %err, - "Scanner could not identify the authoritative baseline for an observation" - ); - outcome = DataUsagePersistOutcome::Failed; - continue; - } - }, - }; - let authoritative = match authoritative_data.as_deref() { - Some(data) => match serde_json::from_slice::(data) { - Ok(info) => info, - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), - state = "observed_baseline_decode_failed", - error = %err, - "Scanner refused to publish an observation from an invalid authoritative baseline" - ); - outcome = DataUsagePersistOutcome::Failed; - continue; - } - }, - None => DataUsageInfo::default(), - }; - data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity()); - } - - if !data_usage_info.is_complete_bucket_usage_snapshot() { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, - state = "reject_incomplete_snapshot", - "Scanner refused to persist an incomplete data usage snapshot" - ); - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); - outcome = DataUsagePersistOutcome::Failed; - continue; - } - - let data = match serde_json::to_vec(&data_usage_info) { - Ok(data) => data, - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, - state = "encode_failed", - error = %e, - "Scanner data usage encode failed" - ); - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::EncodeFailed); - outcome = DataUsagePersistOutcome::Failed; - continue; - } - }; - let sha256hex = (!data.is_empty()).then(|| hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower)); - let data = Bytes::from(data); - let backup_due = !observational && data_usage_backup_due(&data_usage_info); - let mut cas_retry = 0usize; - let save_outcome = loop { - if ctx.is_cancelled() { - break 'updates; - } - - let baseline = if !observational && cas_retry == 0 { - next_baseline.take() - } else { - None - }; - let (existing_data, revision) = match baseline { - Some(baseline) => (baseline.data, baseline.revision), - None => match read_config_with_revision(storeapi.clone(), target_path).await { - Ok((data, revision)) => (data.map(Bytes::from), revision), - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, - state = "revision_load_failed", - error = %e, - "Scanner data usage revision load failed" - ); - break DataUsagePersistOutcome::Failed; - } - }, - }; - let existing = existing_data - .as_deref() - .and_then(|buf| serde_json::from_slice::(buf).ok()); - if cas_retry > 0 && data_usage_reintroduces_missing_bucket(&data_usage_info, existing.as_ref()) { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, - incoming_scanner_epoch = ?data_usage_info.scanner_epoch, - incoming_scanner_cycle = ?data_usage_info.scanner_cycle, - state = "skip_deleted_bucket_reintroduction", - "Scanner usage update skipped after a concurrent bucket removal" - ); - break DataUsagePersistOutcome::Current; - } - if let Some(existing) = existing.as_ref() { - if existing == &data_usage_info { - break DataUsagePersistOutcome::AlreadyDurable; - } - if existing.scanner_epoch.is_some() - && existing.scanner_epoch == data_usage_info.scanner_epoch - && existing.scanner_cycle.is_some() - && existing.scanner_cycle == data_usage_info.scanner_cycle - { - break DataUsagePersistOutcome::PriorCycleDurable; - } - if let Some(reason) = stale_data_usage_update_reason(&data_usage_info, existing, std::time::SystemTime::now()) { - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, - incoming_scanner_epoch = ?data_usage_info.scanner_epoch, - existing_scanner_epoch = ?existing.scanner_epoch, - incoming_scanner_cycle = ?data_usage_info.scanner_cycle, - existing_scanner_cycle = ?existing.scanner_cycle, - incoming_last_update = ?data_usage_info.last_update, - existing_last_update = ?existing.last_update, - reason = reason, - state = "skip_stale_update", - "Scanner stale data usage update skipped" - ); - break DataUsagePersistOutcome::Current; - } - } - if ctx.is_cancelled() { - break 'updates; - } - - let done_save = Metrics::time(Metric::SaveUsage); - let save_result = save_config_shared_with_preconditions( - storeapi.clone(), - target_path, - data.clone(), - sha256hex.clone(), - revision.preconditions(), - ) - .await; - done_save(); - - match save_result { - Ok(object_info) => { - if !observational { - next_baseline = object_info - .etag - .filter(|etag| !etag.is_empty()) - .map(|etag| DataUsagePersistBaseline { - data: Some(data.clone()), - revision: DataUsageCacheRevision::Etag(etag), - }); - } - break DataUsagePersistOutcome::Saved; - } - Err(EcstoreError::PreconditionFailed) if cas_retry < SCANNER_PERSIST_CAS_RETRIES => { - cas_retry += 1; - debug!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, - state = "conflict_retry", - retry = cas_retry, - "Scanner data usage CAS conflict will be reconciled" - ); - } - Err(e) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, - state = if matches!(e, EcstoreError::PreconditionFailed) { - "conflict_retries_exhausted" - } else { - "save_failed" - }, - error = %e, - "Scanner data usage save failed" - ); - break DataUsagePersistOutcome::Failed; - } - } - }; - - match save_outcome { - DataUsagePersistOutcome::Current => { - if observational { - invalidate_admin_data_usage_snapshot_cache().await; - } else { - invalidate_data_usage_snapshot_cache().await; - } - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale); - outcome = DataUsagePersistOutcome::Current; - continue; - } - DataUsagePersistOutcome::AlreadyDurable => { - if observational { - invalidate_admin_data_usage_snapshot_cache().await; - } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; - invalidate_data_usage_snapshot_cache().await; - replace_bucket_usage_memory_from_info(&data_usage_info).await; - } - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); - outcome = DataUsagePersistOutcome::AlreadyDurable; - } - DataUsagePersistOutcome::PriorCycleDurable => { - if observational { - invalidate_admin_data_usage_snapshot_cache().await; - } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; - invalidate_data_usage_snapshot_cache().await; - } - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); - outcome = DataUsagePersistOutcome::PriorCycleDurable; - } - DataUsagePersistOutcome::Failed | DataUsagePersistOutcome::NoUpdate => { - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); - outcome = DataUsagePersistOutcome::Failed; - continue; - } - DataUsagePersistOutcome::Saved => { - if observational { - invalidate_admin_data_usage_snapshot_cache().await; - } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; - invalidate_data_usage_snapshot_cache().await; - replace_bucket_usage_memory_from_info(&data_usage_info).await; - } - global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); - outcome = DataUsagePersistOutcome::Saved; - } - } - - if backup_due { - let done_save = Metrics::time(Metric::SaveUsage); - if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await { - warn!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()), - state = "backup_save_failed", - error = %e, - "Scanner data usage backup save failed" - ); - } - done_save(); - } - } - - outcome -} - -async fn cleanup_observed_data_usage_snapshot( - storeapi: Arc, - authoritative: &DataUsageInfo, -) { - let (observed_data, revision) = - match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { - Ok((Some(data), revision)) => (data, revision), - Ok((None, _)) => return, - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), - state = "observed_cleanup_read_failed", - error = %err, - "Scanner could not inspect observational data usage snapshot before authoritative cleanup" - ); - return; - } - }; - let observed = match serde_json::from_slice::(&observed_data) { - Ok(observed) => observed, - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), - state = "observed_cleanup_decode_failed", - error = %err, - "Scanner refused to remove an invalid observational data usage snapshot after authoritative save" - ); - return; - } - }; - if observed_data_usage_is_newer(&observed, authoritative) { - return; - } - - let result = storeapi - .delete_config_object( - RUSTFS_META_BUCKET, - DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), - ScannerObjectOptions { - delete_prefix: true, - delete_prefix_object: true, - http_preconditions: Some(revision.preconditions()), - ..Default::default() - }, - ) - .await; - - match result { - Ok(_) - | Err( - EcstoreError::FileNotFound - | EcstoreError::ConfigNotFound - | EcstoreError::ObjectNotFound(_, _) - | EcstoreError::PreconditionFailed, - ) => {} - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), - state = "observed_cleanup_failed", - error = %err, - "Scanner could not remove stale observational data usage snapshot after authoritative save" - ); - } - } -} +pub use activity::scanner_topology_digest; +pub(crate) use activity::{ + ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication, + scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements, +}; +pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; +#[cfg(test)] +pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test; +pub(crate) use cycle_state::{current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence}; +pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info}; +pub use usage_store::store_data_usage_in_backend; #[cfg(test)] -mod tests { - use super::*; - use crate::EcstoreResult; - use crate::{ - Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader, - ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader, - init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, - }; - use serial_test::serial; - use std::collections::HashMap; - use std::io::Cursor; - use std::task::Poll; - use temp_env::{with_var, with_var_unset}; - use tokio::io::AsyncReadExt; - use tokio::sync::Mutex; - - const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60; - - async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc) { - init_ecstore_config_for_scanner_tests(); - let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created"); - let mut endpoints = Vec::new(); - for disk_index in 0..4 { - let disk_path = temp_dir.path().join(format!("disk{disk_index}")); - tokio::fs::create_dir_all(&disk_path) - .await - .expect("scanner cycle test disk should be created"); - let mut endpoint = - Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); - endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(disk_index); - endpoints.push(endpoint); - } - let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints { - legacy: false, - set_count: 1, - drives_per_set: 4, - endpoints: Endpoints::from(endpoints), - cmd_line: "scanner-cycle-metrics".to_string(), - platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), - }]); - let instance_ctx = Arc::new(InstanceContext::new()); - init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) - .await - .expect("scanner cycle test disks should initialize"); - let store = ECStore::new_with_instance_ctx( - "127.0.0.1:0".parse().expect("test address should parse"), - endpoint_pools, - CancellationToken::new(), - instance_ctx, - ) - .await - .expect("scanner cycle test ECStore should initialize"); - init_bucket_metadata_sys_for_scanner_tests(store.clone()).await; - - (temp_dir, store) - } - - fn assert_run_data_scanner_signature(_run: F) - where - F: Fn(CancellationToken, Arc) -> Fut, - Fut: Future>, - { - } - - #[test] - fn run_data_scanner_keeps_its_two_argument_api() { - assert_run_data_scanner_signature(run_data_scanner); - } - - #[tokio::test] - async fn scanner_cycle_lock_fence_cancels_cycle_context() { - let cycle_ctx = CancellationToken::new(); - let observed_ctx = cycle_ctx.clone(); - let output = await_scanner_cycle_with_lock_fence( - &cycle_ctx, - async move { - observed_ctx.cancelled().await; - observed_ctx.is_cancelled() - }, - std::future::ready(()), - ) - .await; - - assert_eq!(output, Some(true)); - assert!(cycle_ctx.is_cancelled()); - } - - #[tokio::test] - async fn scanner_cycle_lock_fence_preserves_completed_cycle() { - let cycle_ctx = CancellationToken::new(); - let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::ready(7_u8), std::future::pending()).await; - - assert_eq!(output, Some(7)); - assert!(!cycle_ctx.is_cancelled()); - } - - #[tokio::test] - async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() { - let cycle_ctx = CancellationToken::new(); - let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::pending::<()>(), std::future::ready(())).await; - - assert_eq!(output, None); - assert!(cycle_ctx.is_cancelled()); - } - - struct ScannerDefaultSpeedGuard; - - impl ScannerDefaultSpeedGuard { - fn set(speed: ScannerSpeed) -> Self { - set_scanner_default_speed(speed); - Self - } - } - - impl Drop for ScannerDefaultSpeedGuard { - fn drop(&mut self) { - set_scanner_default_speed(ScannerSpeed::Default); - } - } - - struct ScannerDefaultCycleGuard; - - impl ScannerDefaultCycleGuard { - fn set(secs: u64) -> Self { - set_scanner_default_cycle_secs(Some(secs)); - Self - } - } - - impl Drop for ScannerDefaultCycleGuard { - fn drop(&mut self) { - set_scanner_default_cycle_secs(None); - } - } - - #[derive(Debug, Default)] - struct MemoryConfigStore { - objects: Mutex>>, - revisions: Mutex>, - fail_put_number: Mutex>, - error_after_commit_put_number: Mutex>, - interleaving_puts: Mutex)>>, - cancel_after_interleaving_puts: Mutex>, - cancel_after_successful_puts: Mutex>, - replace_after_successful_puts: Mutex)>>, - put_counts: Mutex>, - } - - fn memory_config_key(bucket: &str, object: &str) -> String { - format!("{bucket}/{object}") - } - - #[async_trait::async_trait] - impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore { - type Error = EcstoreError; - type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec; - type HeaderMap = http::HeaderMap; - type ObjectOptions = ObjectOptions; - type ObjectInfo = ObjectInfo; - type GetObjectReader = GetObjectReader; - type PutObjectReader = PutObjReader; - - async fn get_object_reader( - &self, - bucket: &str, - object: &str, - _range: Option, - _h: http::HeaderMap, - _opts: &ObjectOptions, - ) -> EcstoreResult { - let key = memory_config_key(bucket, object); - let data = self - .objects - .lock() - .await - .get(&key) - .cloned() - .ok_or(EcstoreError::FileNotFound)?; - let revision = *self.revisions.lock().await.entry(key).or_insert(1); - - Ok(GetObjectReader { - stream: Box::new(Cursor::new(data)), - object_info: ObjectInfo { - etag: Some(format!("memory-{revision}")), - ..Default::default() - }, - buffered_body: None, - body_source: Default::default(), - }) - } - - async fn put_object( - &self, - bucket: &str, - object: &str, - data: &mut PutObjReader, - opts: &ObjectOptions, - ) -> EcstoreResult { - let mut buf = Vec::new(); - data.stream.read_to_end(&mut buf).await?; - let key = memory_config_key(bucket, object); - let put_count = { - let mut put_counts = self.put_counts.lock().await; - let put_count = put_counts.entry(key.clone()).or_insert(0); - *put_count += 1; - *put_count - }; - - if self.fail_put_number.lock().await.get(&key) == Some(&put_count) { - return Err(EcstoreError::other("injected put failure")); - } - - let interleaving_data = { - let mut interleaving_puts = self.interleaving_puts.lock().await; - if interleaving_puts - .get(&key) - .is_some_and(|(expected_put, _)| *expected_put == put_count) - { - interleaving_puts.remove(&key).map(|(_, data)| data) - } else { - None - } - }; - let cancel_after_interleaving = if interleaving_data.is_some() { - self.cancel_after_interleaving_puts.lock().await.remove(&key) - } else { - None - }; - let replacement = { - let mut replacements = self.replace_after_successful_puts.lock().await; - if replacements - .get(&key) - .is_some_and(|(expected_put, _)| *expected_put == put_count) - { - replacements.remove(&key).map(|(_, replacement)| replacement) - } else { - None - } - }; - let mut objects = self.objects.lock().await; - let mut revisions = self.revisions.lock().await; - if let Some(interleaving_data) = interleaving_data { - let revision = revisions.get(&key).copied().unwrap_or(0) + 1; - objects.insert(key.clone(), interleaving_data); - revisions.insert(key.clone(), revision); - if let Some(cancel) = cancel_after_interleaving { - cancel.cancel(); - } - } - let current_revision = objects.contains_key(&key).then(|| revisions.get(&key).copied().unwrap_or(1)); - if let Some(preconditions) = &opts.http_preconditions { - if preconditions - .if_none_match - .as_deref() - .is_some_and(|condition| !condition.trim().is_empty()) - && current_revision.is_some() - { - return Err(EcstoreError::PreconditionFailed); - } - if let Some(expected) = preconditions - .if_match - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - { - let actual = current_revision.map(|revision| format!("memory-{revision}")); - if actual.as_deref() != Some(expected.trim_matches('"')) { - return Err(EcstoreError::PreconditionFailed); - } - } - } - - let revision = current_revision.unwrap_or(0) + 1; - objects.insert(key.clone(), buf); - revisions.insert(key.clone(), revision); - if let Some(replacement) = replacement { - objects.insert(key.clone(), replacement); - revisions.insert(key.clone(), revision + 1); - } - drop(revisions); - drop(objects); - let cancel_after_success = { - let mut cancellations = self.cancel_after_successful_puts.lock().await; - if cancellations - .get(&key) - .is_some_and(|(expected_put, _)| *expected_put == put_count) - { - cancellations.remove(&key).map(|(_, cancel)| cancel) - } else { - None - } - }; - if let Some(cancel) = cancel_after_success { - cancel.cancel(); - } - if self.error_after_commit_put_number.lock().await.get(&key) == Some(&put_count) { - return Err(EcstoreError::other("injected post-commit put failure")); - } - Ok(ObjectInfo { - etag: Some(format!("memory-{revision}")), - ..Default::default() - }) - } - } - - fn with_unset_scanner_timing_env(f: impl FnOnce()) { - with_var_unset(ENV_SCANNER_SPEED, || { - with_var_unset("MINIO_SCANNER_SPEED", || { - with_var_unset(ENV_SCANNER_CYCLE, || { - with_var_unset("MINIO_SCANNER_CYCLE", || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, f); - }); - }); - }); - }); - }); - } - - #[test] - fn test_randomized_cycle_delay_keeps_configured_start_delay() { - // 120s with ±10% jitter should stay clearly above the historic 30s cap. - let delay = randomized_cycle_delay_for(Duration::from_secs(120)); - assert!(delay > Duration::from_secs(30), "expected delay > 30s, got {delay:?}"); - // Jitter window should stay within configured bounds. - assert!(delay >= Duration::from_secs(108)); - assert!(delay <= Duration::from_secs(132)); - } - - #[test] - fn test_randomized_cycle_delay_bounds_extreme_interval() { - let delay = randomized_cycle_delay_for(Duration::MAX); - - assert!(delay >= MAX_SCANNER_SCHEDULE_DELAY.mul_f64(0.9)); - assert!(delay <= MAX_SCANNER_SCHEDULE_DELAY); - } - - #[test] - fn test_initial_scanner_delay_uses_configured_start_delay() { - let delay = initial_scanner_delay_for(Some(120)); - assert!(delay >= Duration::from_secs(108)); - assert!(delay <= Duration::from_secs(132)); - } - - #[test] - #[serial] - fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() { - with_var(ENV_SCANNER_CYCLE, Some("120"), || { - crate::runtime_config::refresh_scanner_runtime_config_for_tests(); - let delay = initial_scanner_delay_for(None); - assert!(delay >= Duration::from_secs(108)); - assert!(delay <= Duration::from_secs(132)); - }); - crate::runtime_config::refresh_scanner_runtime_config_for_tests(); - } - - #[test] - fn test_initial_scanner_delay_skips_for_cold_usage_cache_with_buckets() { - let delay = initial_scanner_delay_for_startup(Some(120), true, true, false); - assert_eq!(delay, Duration::ZERO); - } - - #[test] - fn test_initial_scanner_delay_keeps_configured_delay_for_warm_usage_cache_no_replication() { - let delay = initial_scanner_delay_for_startup(Some(120), false, true, false); - assert!(delay >= Duration::from_secs(108)); - assert!(delay <= Duration::from_secs(132)); - } - - #[test] - fn test_initial_scanner_delay_skips_for_cold_usage_cache_without_buckets() { - let delay = initial_scanner_delay_for_startup(Some(120), true, false, false); - assert_eq!(delay, Duration::ZERO); - } - - #[test] - fn test_initial_scanner_delay_skips_for_active_replication_warm_cache() { - // Warm cache + active replication rules → skip startup delay so that FAILED-status objects - // from a crash are healed on the first cycle, not after a 27-33 min sleep. - let delay = initial_scanner_delay_for_startup(Some(120), false, true, true); - assert_eq!(delay, Duration::ZERO); - } - - #[test] - fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() { - // Active replication but no buckets → no objects to scan, keep normal delay. - let delay = initial_scanner_delay_for_startup(Some(120), false, false, true); - assert!(delay >= Duration::from_secs(108)); - assert!(delay <= Duration::from_secs(132)); - } - - #[test] - #[serial] - fn test_scanner_cycle_max_duration_uses_env() { - with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || { - assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42))); - }); - } - - #[test] - #[serial] - fn test_scanner_cycle_max_duration_default_is_disabled() { - with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { - assert_eq!(scanner_cycle_max_duration(), None); - }); - } - - #[tokio::test] - async fn test_scanner_cycle_budget_cancels_after_duration() { - let parent = CancellationToken::new(); - let budget = ScannerCycleBudget::new( - &parent, - ScannerCycleBudgetConfig { - max_duration: Some(Duration::from_millis(1)), - ..Default::default() - }, - ); - - tokio::time::timeout(Duration::from_secs(5), budget.token().cancelled()) - .await - .expect("scanner cycle budget should cancel after max duration"); - - assert!(budget.budget_elapsed()); - assert!(budget.token().is_cancelled()); - } - - #[tokio::test] - async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() { - let parent = CancellationToken::new(); - let budget = ScannerCycleBudget::new( - &parent, - ScannerCycleBudgetConfig { - max_duration: Some(Duration::from_secs(60)), - ..Default::default() - }, - ); - let token = budget.token(); - - drop(budget); - - assert!(token.is_cancelled()); - } - - #[test] - #[serial] - fn test_scanner_cycle_budget_config_uses_work_budget_env() { - with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || { - with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || { - let config = scanner_cycle_budget_config(); - assert_eq!(config.max_objects, Some(100)); - assert_eq!(config.max_directories, Some(25)); - }); - }); - } - - #[test] - #[serial] - fn test_scanner_cycle_budget_config_disables_zero_work_budgets() { - with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || { - with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || { - let config = scanner_cycle_budget_config(); - assert_eq!(config.max_objects, None); - assert_eq!(config.max_directories, None); - }); - }); - } - - #[test] - fn test_scan_cycle_partial_reason_maps_budget_reason() { - assert_eq!( - scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Runtime)), - ScanCyclePartialReason::Runtime - ); - assert_eq!( - scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Objects)), - ScanCyclePartialReason::Objects - ); - assert_eq!( - scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Directories)), - ScanCyclePartialReason::Directories - ); - assert_eq!(scan_cycle_partial_reason(None), ScanCyclePartialReason::Unknown); - } - - #[test] - fn test_scan_cycle_partial_source_maps_budget_reason() { - assert_eq!(scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Runtime)), None); - assert_eq!( - scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Objects)), - Some(ScannerWorkSource::Usage) - ); - assert_eq!( - scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Directories)), - Some(ScannerWorkSource::Usage) - ); - assert_eq!(scan_cycle_partial_source(None), None); - } - - #[tokio::test] - #[serial] - async fn test_mark_scan_cycle_idle_clears_published_cycle_state() { - let mut cycle_info = CurrentCycle { - current: 12, - next: 13, - cycle_completed: vec![Utc::now()], - started: Utc::now(), - }; - - global_metrics().set_current_scan_mode(HealScanMode::Deep); - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - - mark_scan_cycle_idle(&mut cycle_info, &mut cycle_metrics_guard).await; - - let published = global_metrics() - .get_cycle() - .await - .expect("scanner cycle state should remain published"); - - assert_eq!(cycle_info.current, 0); - assert_eq!(cycle_info.next, 13); - assert_eq!(published.current, 0); - assert_eq!(published.next, 13); - assert_eq!(global_metrics().current_scan_mode(), HealScanMode::Unknown); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() { - let cycle_started = Utc::now() - chrono::Duration::seconds(5); - let mut cycle_info = CurrentCycle { - current: 0, - next: 1, - started: cycle_started, - ..Default::default() - }; - let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - let setup_report = global_metrics().report().await; - assert!(setup_report.current_cycle_active); - assert_eq!(setup_report.current_cycle, 0); - assert_eq!(setup_report.current_started.as_second(), cycle_started.timestamp()); - assert_eq!( - setup_report.current_started.subsec_nanosecond(), - i32::try_from(cycle_started.timestamp_subsec_nanos()).expect("chrono nanoseconds fit in i32") - ); - - mark_scan_cycle_idle(&mut cycle_info, &mut guard).await; - let idle_report = global_metrics().report().await; - assert!(!idle_report.current_cycle_active); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() { - let mut cycle_info = CurrentCycle { - current: 12, - next: 13, - started: Utc::now(), - ..Default::default() - }; - let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - - cycle_info.current = 0; - tokio::task::yield_now().await; - let finalizing_report = global_metrics().report().await; - assert!(finalizing_report.current_cycle_active); - assert_eq!(finalizing_report.current_cycle, 12); - - guard.finish(cycle_info).await; - let idle_report = global_metrics().report().await; - assert!(!idle_report.current_cycle_active); - assert_eq!(idle_report.current_cycle, 0); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn scanner_cycle_metrics_guard_drop_clears_activity() { - let guard = ScannerCycleMetricsGuard::new(CurrentCycle { - current: 12, - next: 13, - started: Utc::now(), - ..Default::default() - }) - .await; - assert!(global_metrics().report().await.current_cycle_active); - - drop(guard); - - assert!(!global_metrics().report().await.current_cycle_active); - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { - let (_temp_dir, store) = setup_scanner_cycle_store().await; - let ctx = CancellationToken::new(); - let mut cycle_info = CurrentCycle::default(); - let mut revision = DataUsageCacheRevision::Missing; - let leader_epoch = u64::MAX - 1; - let state_persist_reached = Arc::new(Notify::new()); - let _state_persist_hook = set_scanner_cycle_state_persist_test_hook(leader_epoch, state_persist_reached.clone()); - let state_lock = store - .new_ns_lock(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()) - .await - .expect("scanner cycle state lock should be created"); - let state_guard = state_lock - .get_write_lock(Duration::from_secs(1)) - .await - .expect("scanner cycle state lock should be acquired"); - let mut cycle = Box::pin(run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch)); - let waker = std::task::Waker::noop(); - let mut context = std::task::Context::from_waker(waker); - - assert!(cycle.as_mut().poll(&mut context).is_pending()); - let active = global_metrics().report().await; - assert!(active.current_cycle_active); - assert_eq!(active.current_cycle, 0); - - tokio::time::timeout(Duration::from_secs(30), async { - tokio::select! { - outcome = &mut cycle => panic!("scanner cycle finished before state persistence was released: {outcome:?}"), - _ = state_persist_reached.notified() => {} - } - }) - .await - .expect("scanner cycle should reach state persistence"); - let finalizing = global_metrics().report().await; - assert!(finalizing.current_cycle_active); - - drop(state_guard); - let outcome = tokio::time::timeout(Duration::from_secs(30), cycle) - .await - .expect("scanner cycle should finish"); - assert!(matches!( - outcome, - ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance - )); - assert!(!global_metrics().report().await.current_cycle_active); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle_info = CurrentCycle { - current: 12, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - - assert!( - finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await - ); - - assert_eq!(cycle_info.next, 13); - assert_eq!(cycle_info.current, 0); - assert!(cycle_info.cycle_completed.is_empty()); - assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1")); - - let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("cycle state should be persisted after a partial cycle"); - assert_eq!( - u64::from_le_bytes(buf[0..8].try_into().expect("persisted state should start with the counter")), - 13 - ); - let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle info should decode"); - assert_eq!(decoded.next, 13); - assert_eq!(decoded.current, 0); - assert_eq!(epoch, 1); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn scanner_cycle_recovers_to_newer_durable_cache_floor() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle_info = CurrentCycle { - current: 12, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - - assert!( - persist_required_scanner_cycle_floor( - &ctx, - store.clone(), - &mut cycle_info, - &mut revision, - 7, - 19, - &mut cycle_metrics_guard, - ) - .await - ); - assert_eq!(cycle_info.current, 0); - assert_eq!(cycle_info.next, 19); - - let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("recovered cycle floor should be persisted"); - let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("recovered cycle state should decode"); - assert_eq!(decoded.current, 0); - assert_eq!(decoded.next, 19); - assert_eq!(epoch, 7); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn scanner_cycle_rejects_invalid_cache_floor() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle_info = CurrentCycle { - current: 12, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - - assert!( - !persist_required_scanner_cycle_floor( - &ctx, - store.clone(), - &mut cycle_info, - &mut revision, - 7, - 12, - &mut cycle_metrics_guard, - ) - .await - ); - assert_eq!(cycle_info.next, 12); - assert_eq!(revision, DataUsageCacheRevision::Missing); - let mut max_cycle_info = CurrentCycle { - current: 12, - next: 12, - ..Default::default() - }; - let mut max_cycle_metrics_guard = ScannerCycleMetricsGuard::new(max_cycle_info.clone()).await; - assert!( - !persist_required_scanner_cycle_floor( - &ctx, - store.clone(), - &mut max_cycle_info, - &mut revision, - 7, - u64::MAX, - &mut max_cycle_metrics_guard, - ) - .await - ); - assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); - - global_metrics().set_cycle(None).await; - } - - #[test] - fn scanner_cycle_state_decodes_legacy_and_fenced_formats() { - let cycle = CurrentCycle { - current: 12, - next: 13, - cycle_completed: vec![], - started: Utc::now(), - }; - let mut legacy = cycle.next.to_le_bytes().to_vec(); - legacy.extend(cycle.marshal().expect("legacy cycle state should encode")); - - let (legacy_cycle, legacy_epoch) = - decode_scanner_cycle_state(&legacy).expect("legacy cycle state should remain readable"); - assert_eq!(legacy_cycle.next, 13); - assert_eq!(legacy_epoch, 0); - - let fenced = encode_scanner_cycle_state(&cycle, 7).expect("fenced cycle state should encode"); - let (fenced_cycle, fenced_epoch) = decode_scanner_cycle_state(&fenced).expect("fenced cycle state should decode"); - assert_eq!(fenced_cycle.next, 13); - assert_eq!(fenced_epoch, 7); - } - - #[test] - fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() { - assert_eq!( - decode_scanner_cycle_state_for_startup(&[]) - .expect("missing cycle state should use defaults") - .1, - 0 - ); - assert!(decode_scanner_cycle_state_for_startup(&[1]).is_err()); - - let mut corrupt_fenced = 13_u64.to_le_bytes().to_vec(); - corrupt_fenced.extend_from_slice(SCANNER_CYCLE_STATE_MAGIC); - corrupt_fenced.extend_from_slice(&7_u64.to_le_bytes()); - corrupt_fenced.extend_from_slice(b"not-msgpack"); - assert!(decode_scanner_cycle_state_for_startup(&corrupt_fenced).is_err()); - assert!(decode_scanner_cycle_state_for_startup(&u64::MAX.to_le_bytes()).is_err()); - - let exhausted = CurrentCycle { - next: u64::MAX, - ..Default::default() - }; - assert!(encode_scanner_cycle_state(&exhausted, 7).is_err()); - } - - #[tokio::test] - async fn scanner_startup_uses_primary_and_backup_usage_floor() { - let store = Arc::new(MemoryConfigStore::default()); - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] { - store.objects.lock().await.insert( - memory_config_key(RUSTFS_META_BUCKET, path), - serde_json::to_vec(&DataUsageInfo { - scanner_epoch: Some(epoch), - scanner_cycle: Some(cycle), - ..Default::default() - }) - .expect("usage snapshot should encode"), - ); - } - - let floor = persisted_usage_floor(store).await.expect("usage floor should load"); - assert_eq!( - floor, - PersistedUsageFloor { - next_cycle: 104, - leader_epoch: 11, - } - ); - - let mut cycle = CurrentCycle::default(); - let mut epoch = 0; - apply_persisted_usage_floor(&mut cycle, &mut epoch, floor); - assert_eq!(cycle.next, 104); - assert_eq!(epoch, 11); - } - - #[test] - fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() { - let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1); - legacy.usage_snapshot_complete = false; - - assert!(data_usage_info_is_cold(&legacy)); - assert!(!data_usage_info_is_cold(&complete_usage_with_bucket_count( - Some(std::time::SystemTime::now()), - 1, - ))); - assert!(!data_usage_info_is_cold(&DataUsageInfo { - last_update: Some(std::time::SystemTime::now()), - usage_snapshot_complete: true, - ..Default::default() - })); - } - - #[test] - fn scanner_startup_prompts_only_for_a_newer_valid_observation() { - let authoritative = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH), - scanner_epoch: Some(4), - scanner_cycle: Some(10), - ..complete_usage_with_bucket_count(None, 0) - }; - let observed = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)), - scanner_epoch: Some(4), - scanner_cycle: Some(11), - usage_snapshot_converged: Some(false), - usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()), - ..complete_usage_with_bucket_count(None, 0) - }; - - assert!(usage_cache_needs_prompt_scan(&authoritative, Some(&observed))); - assert!(!usage_cache_needs_prompt_scan(&authoritative, None)); - - let mut converged = observed.clone(); - converged.usage_snapshot_converged = Some(true); - assert!(!usage_cache_needs_prompt_scan(&authoritative, Some(&converged))); - - let mut legacy_observation = observed; - legacy_observation.usage_snapshot_converged = None; - assert!(!usage_cache_needs_prompt_scan(&authoritative, Some(&legacy_observation))); - } - - #[tokio::test] - async fn scanner_startup_prefers_v2_over_legacy_usage() { - let store = Arc::new(MemoryConfigStore::default()); - let legacy = DataUsageInfo { - scanner_epoch: Some(19), - scanner_cycle: Some(41), - last_update: Some(std::time::SystemTime::now()), - ..Default::default() - }; - let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"); - store.objects.lock().await.insert( - memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), - legacy_data.clone(), - ); - - assert_eq!( - read_data_usage_config_for_startup(&store) - .await - .expect("legacy startup usage should load"), - Some(legacy_data) - ); - assert_eq!( - persisted_usage_floor(store.clone()) - .await - .expect("legacy usage floor should seed the upgrade"), - PersistedUsageFloor { - next_cycle: 42, - leader_epoch: 19, - } - ); - - let authoritative = DataUsageInfo { - scanner_epoch: Some(23), - scanner_cycle: Some(51), - last_update: Some(std::time::SystemTime::now()), - usage_snapshot_complete: true, - ..Default::default() - }; - let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode"); - store.objects.lock().await.insert( - memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), - authoritative_data.clone(), - ); - - assert_eq!( - read_data_usage_config_for_startup(&store) - .await - .expect("v2 startup usage should load"), - Some(authoritative_data) - ); - assert_eq!( - persisted_usage_floor(store.clone()) - .await - .expect("v2 usage floor should be authoritative"), - PersistedUsageFloor { - next_cycle: 52, - leader_epoch: 23, - } - ); - - store.objects.lock().await.insert( - memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), - b"corrupt-v2".to_vec(), - ); - assert_eq!( - read_data_usage_config_for_startup(&store) - .await - .expect("startup inspection should preserve authoritative bytes"), - Some(b"corrupt-v2".to_vec()) - ); - assert!( - persisted_usage_floor(store).await.is_err(), - "corrupt v2 state must not fall back to a legacy writer" - ); - } - - #[tokio::test] - async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() { - let store = Arc::new(MemoryConfigStore::default()); - store.objects.lock().await.insert( - memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), - b"not-json".to_vec(), - ); - - assert!(persisted_usage_floor(store.clone()).await.is_err()); - - store.objects.lock().await.insert( - memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), - serde_json::to_vec(&DataUsageInfo { - scanner_cycle: Some(u64::MAX - 1), - ..Default::default() - }) - .expect("usage snapshot should encode"), - ); - assert!(persisted_usage_floor(store).await.is_err()); - } - - #[tokio::test] - #[serial] - async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - - for cycle in [9, 10] { - let (sender, receiver) = mpsc::channel(1); - sender - .send(DataUsageInfo { - scanner_epoch: Some(1), - scanner_cycle: Some(cycle), - last_update: Some(std::time::SystemTime::now()), - ..complete_usage_with_bucket_count(None, 0) - }) - .await - .expect("usage update should queue"); - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(ctx.clone(), store.clone(), receiver).await, - DataUsagePersistOutcome::Saved - ); - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - let backup = read_config(store.clone(), &backup_path).await; - if cycle == 9 { - assert!(matches!(backup, Err(EcstoreError::ConfigNotFound))); - } else { - let saved = serde_json::from_slice::( - &backup.expect("the tenth durable scanner cycle should create a backup"), - ) - .expect("backup usage snapshot should decode"); - assert_eq!(saved.scanner_cycle, Some(10)); - assert_eq!(saved.scanner_epoch, Some(1)); - } - } - } - - #[async_trait::async_trait] - impl crate::ScannerConfigObjectDelete for MemoryConfigStore { - async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult { - let key = memory_config_key(bucket, object); - let mut objects = self.objects.lock().await; - if !objects.contains_key(&key) { - return Err(EcstoreError::FileNotFound); - } - let mut revisions = self.revisions.lock().await; - if let Some(expected) = opts - .http_preconditions - .as_ref() - .and_then(|preconditions| preconditions.if_match.as_deref()) - { - let actual = revisions.get(&key).map(|revision| format!("memory-{revision}")); - if actual.as_deref() != Some(expected.trim_matches('"')) { - return Err(EcstoreError::PreconditionFailed); - } - } - objects.remove(&key); - revisions.remove(&key); - Ok(ObjectInfo::default()) - } - } - - #[test] - fn scanner_cycle_advance_fails_before_reserved_exhausted_value() { - let mut cycle = CurrentCycle { - next: u64::MAX - 2, - ..Default::default() - }; - advance_scanner_cycle(&mut cycle).expect("last persistable scanner cycle should remain valid"); - assert_eq!(cycle.next, u64::MAX - 1); - assert!(advance_scanner_cycle(&mut cycle).is_err()); - assert_eq!(cycle.next, u64::MAX - 1); - } - - #[tokio::test] - #[serial] - async fn test_finalize_partial_scan_cycle_reports_persist_failure() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); - store.fail_put_number.lock().await.insert(key, 1); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle_info = CurrentCycle { - current: 12, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - - assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await); - assert_eq!(cycle_info.next, 13); - assert_eq!(cycle_info.current, 0); - assert_eq!(revision, DataUsageCacheRevision::Missing); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - #[serial] - async fn test_persist_scanner_cycle_state_reconciles_newer_winner() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut initial_revision = DataUsageCacheRevision::Missing; - let mut initial = CurrentCycle { - current: 0, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await); - - let mut current_revision = initial_revision.clone(); - let mut stale_revision = initial_revision; - let mut current = CurrentCycle { - next: 14, - ..initial.clone() - }; - let mut stale = CurrentCycle { next: 13, ..initial }; - - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut current_revision, 1).await); - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut stale, &mut stale_revision, 1).await); - - let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("new leader cycle state should remain persisted"); - let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); - assert_eq!(decoded.next, 14); - assert_eq!(epoch, 1); - assert_eq!(stale.next, 14); - assert!(matches!(current_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); - assert!(matches!(stale_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); - - global_metrics().set_cycle(None).await; - } - - #[tokio::test] - async fn test_persist_scanner_cycle_state_retries_after_stale_winner() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut initial_revision = DataUsageCacheRevision::Missing; - let mut initial = CurrentCycle { - current: 0, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await); - - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); - let stale = CurrentCycle { - next: 13, - ..initial.clone() - }; - let stale_buf = encode_scanner_cycle_state(&stale, 1).expect("stale cycle state should encode"); - store.interleaving_puts.lock().await.insert(key, (2, stale_buf)); - - let mut current = CurrentCycle { next: 14, ..initial }; - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut initial_revision, 1).await); - - let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("newer cycle state should replace the stale conflict winner"); - let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); - assert_eq!(decoded.next, 14); - assert_eq!(epoch, 1); - assert_eq!(current.next, 14); - assert!(matches!(initial_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-3")); - } - - #[tokio::test] - async fn test_persist_scanner_cycle_state_stops_retry_after_leader_fence() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut revision = DataUsageCacheRevision::Missing; - let mut initial = CurrentCycle { - current: 0, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut revision, 1).await); - - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); - let replacement = CurrentCycle { - next: 13, - ..initial.clone() - }; - let replacement_buf = encode_scanner_cycle_state(&replacement, 2).expect("replacement cycle state should encode"); - store.interleaving_puts.lock().await.insert(key.clone(), (2, replacement_buf)); - store - .cancel_after_interleaving_puts - .lock() - .await - .insert(key.clone(), ctx.clone()); - - let mut stale_leader = CurrentCycle { next: 14, ..initial }; - assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut stale_leader, &mut revision, 1).await); - - let buf = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("replacement leader cycle state should remain persisted"); - let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); - assert_eq!(decoded.next, 13); - assert_eq!(epoch, 2); - assert_eq!(stale_leader.next, 14); - assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); - assert_eq!(store.put_counts.lock().await.get(&key), Some(&2)); - } - - #[tokio::test] - async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conflict() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle = CurrentCycle { - current: 0, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); - - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); - let old_epoch_commit = CurrentCycle { - next: 14, - ..cycle.clone() - }; - store.interleaving_puts.lock().await.insert( - key.clone(), - ( - 2, - encode_scanner_cycle_state(&old_epoch_commit, 1).expect("old-epoch cycle state should encode"), - ), - ); - - let mut persisted_epoch = 8; - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); - - let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("new leadership claim should remain persisted"); - let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode"); - assert_eq!(claimed_cycle.next, 14); - assert_eq!(claimed_epoch, 9); - assert_eq!(persisted_epoch, 9); - assert_eq!(store.put_counts.lock().await.get(&key), Some(&3)); - } - - #[tokio::test] - async fn test_leadership_claim_confirms_commit_after_returned_error() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); - let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - store.error_after_commit_put_number.lock().await.insert(key.clone(), 1); - store.error_after_commit_put_number.lock().await.insert(usage_key.clone(), 1); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle = CurrentCycle { - current: 0, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - let mut persisted_epoch = 0; - - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); - - let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("ambiguous leadership claim should be durable"); - let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode"); - assert_eq!(claimed_cycle.next, 12); - assert_eq!(claimed_epoch, 1); - assert_eq!(persisted_epoch, 1); - assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1")); - assert_eq!(store.put_counts.lock().await.get(&key), Some(&1)); - let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) - .await - .expect("ambiguous usage epoch fence should be durable"); - assert_eq!( - serde_json::from_slice::(&usage) - .expect("usage epoch fence should decode") - .scanner_epoch, - Some(1) - ); - assert_eq!(store.put_counts.lock().await.get(&usage_key), Some(&1)); - } - - #[tokio::test] - async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { - let store = Arc::new(MemoryConfigStore::default()); - let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let mut old_usage = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), - scanner_epoch: Some(4), - scanner_cycle: Some(11), - ..Default::default() - }; - old_usage.buckets_usage.insert( - "bucket-a".to_string(), - rustfs_data_usage::BucketUsageInfo { - objects_count: 2, - size: 84, - ..Default::default() - }, - ); - old_usage.buckets_count = 1; - old_usage.calculate_totals(); - let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode"); - store.objects.lock().await.insert(usage_key.clone(), old_data.clone()); - store.revisions.lock().await.insert(usage_key, 1); - - let ctx = CancellationToken::new(); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle = CurrentCycle { - next: 12, - started: Utc::now(), - ..Default::default() - }; - let mut persisted_epoch = 4; - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); - - let (fenced_data, fenced_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) - .await - .expect("fenced usage snapshot should load"); - let fenced = serde_json::from_slice::(fenced_data.as_deref().expect("fenced usage snapshot should exist")) - .expect("fenced usage snapshot should decode"); - assert_eq!(fenced.scanner_epoch, Some(5)); - assert_eq!(fenced.objects_total_count, 2); - assert_eq!(fenced.buckets_usage.get("bucket-a").map(|usage| usage.size), Some(84)); - assert!(matches!(fenced_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); - - let stale_save = save_config_with_preconditions( - store, - DATA_USAGE_OBJ_NAME_PATH.as_str(), - old_data, - DataUsageCacheRevision::Etag("memory-1".to_string()).preconditions(), - ) - .await; - assert!(matches!(stale_save, Err(EcstoreError::PreconditionFailed))); - } - - #[tokio::test] - async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() { - let store = Arc::new(MemoryConfigStore::default()); - let ctx = CancellationToken::new(); - let mut revision = DataUsageCacheRevision::Missing; - let mut cycle = CurrentCycle { - current: 0, - next: 12, - cycle_completed: vec![], - started: Utc::now(), - }; - assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); - - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); - store - .cancel_after_successful_puts - .lock() - .await - .insert(key.clone(), (2, ctx.clone())); - cycle.next = 14; - assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); - - let (persisted, persisted_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) - .await - .expect("committed old-epoch state should load"); - let mut replacement_cycle = decode_scanner_cycle_state( - persisted - .as_deref() - .expect("old-epoch state should have committed before cancellation"), - ) - .expect("old-epoch state should decode") - .0; - let mut replacement_revision = persisted_revision; - let mut replacement_epoch = 1; - let replacement_ctx = CancellationToken::new(); - assert!( - claim_scanner_leadership( - &replacement_ctx, - store.clone(), - &mut replacement_cycle, - &mut replacement_revision, - &mut replacement_epoch, - ) - .await - ); - - let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) - .await - .expect("replacement leadership claim should persist"); - let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode"); - assert_eq!(claimed_cycle.next, 14); - assert_eq!(claimed_epoch, 2); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_preserves_newer_snapshot() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(2); - let ctx = CancellationToken::new(); - - let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2); - let older = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1); - - sender.send(newer).await.expect("newer usage snapshot should enqueue"); - sender.send(older).await.expect("older usage snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; - - let objects = store.objects.lock().await; - let saved = objects - .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str())) - .expect("data usage config should be saved"); - let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); - - assert_eq!(saved.buckets_count, 2); - assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20))); - assert_eq!(outcome, DataUsagePersistOutcome::Current); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(1); - let ctx = CancellationToken::new(); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2); - let stale = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1); - store - .interleaving_puts - .lock() - .await - .insert(key.clone(), (1, serde_json::to_vec(&newer).expect("newer usage snapshot should encode"))); - - sender.send(stale).await.expect("stale usage snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; - - let objects = store.objects.lock().await; - let saved = objects - .get(&key) - .expect("interleaving newer usage snapshot should remain saved"); - let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); - assert_eq!(saved.buckets_count, 2); - assert_eq!(saved.last_update, newer.last_update); - assert_eq!(outcome, DataUsagePersistOutcome::Current); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_does_not_resurrect_deleted_bucket_after_conflict() { - let store = Arc::new(MemoryConfigStore::default()); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let mut initial = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), - scanner_epoch: Some(8), - scanner_cycle: Some(12), - ..Default::default() - }; - initial.buckets_usage.insert( - "bucket-a".to_string(), - rustfs_data_usage::BucketUsageInfo { - objects_count: 2, - size: 84, - ..Default::default() - }, - ); - initial.bucket_sizes.insert("bucket-a".to_string(), 84); - initial.buckets_count = 1; - initial.calculate_totals(); - mark_usage_snapshot_complete(&mut initial); - let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode"); - store.objects.lock().await.insert(key.clone(), initial_data.clone()); - store.revisions.lock().await.insert(key.clone(), 1); - - let mut deleted = initial.clone(); - deleted.buckets_usage.clear(); - deleted.bucket_sizes.clear(); - deleted.buckets_count = 0; - deleted.calculate_totals(); - mark_usage_snapshot_complete(&mut deleted); - store - .interleaving_puts - .lock() - .await - .insert(key.clone(), (1, serde_json::to_vec(&deleted).expect("deleted snapshot should encode"))); - - let mut incoming = initial; - incoming.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)); - incoming.scanner_cycle = Some(13); - let (sender, receiver) = mpsc::channel(1); - sender.send(incoming).await.expect("stale scanner snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline( - CancellationToken::new(), - store.clone(), - receiver, - Some(8), - Some(DataUsagePersistBaseline { - data: Some(Bytes::from(initial_data)), - revision: DataUsageCacheRevision::Etag("memory-1".to_string()), - }), - ) - .await; - - assert_eq!(outcome, DataUsagePersistOutcome::Current); - let saved = store - .objects - .lock() - .await - .get(&key) - .cloned() - .expect("deleted usage snapshot should remain"); - let saved = serde_json::from_slice::(&saved).expect("deleted usage snapshot should decode"); - assert!(!saved.buckets_usage.contains_key("bucket-a")); - assert!(!saved.bucket_sizes.contains_key("bucket-a")); - assert_eq!(store.put_counts.lock().await.get(&key), Some(&1)); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_updates_backup_with_new_bucket() { - let store = Arc::new(MemoryConfigStore::default()); - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); - let deleted = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), - scanner_epoch: Some(8), - scanner_cycle: Some(1), - ..complete_usage_with_bucket_count(None, 0) - }; - store.objects.lock().await.insert( - backup_key.clone(), - serde_json::to_vec(&deleted).expect("deleted backup snapshot should encode"), - ); - store.revisions.lock().await.insert(backup_key.clone(), 1); - - let (sender, receiver) = mpsc::channel(11); - for cycle in 2_u64..=12 { - let mut incoming = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20 + cycle)), - scanner_epoch: Some(8), - scanner_cycle: Some(cycle), - ..Default::default() - }; - incoming.buckets_usage.insert( - "bucket-a".to_string(), - rustfs_data_usage::BucketUsageInfo { - objects_count: 2, - size: 84, - ..Default::default() - }, - ); - incoming.bucket_sizes.insert("bucket-a".to_string(), 84); - incoming.buckets_count = 1; - incoming.calculate_totals(); - mark_usage_snapshot_complete(&mut incoming); - sender.send(incoming).await.expect("usage snapshot should enqueue"); - } - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, - DataUsagePersistOutcome::Saved - ); - - let saved = store - .objects - .lock() - .await - .get(&backup_key) - .cloned() - .expect("deleted backup snapshot should remain"); - let saved = serde_json::from_slice::(&saved).expect("backup snapshot should decode"); - assert!(saved.buckets_usage.contains_key("bucket-a")); - assert!(saved.bucket_sizes.contains_key("bucket-a")); - assert_eq!(saved.scanner_cycle, Some(10)); - assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1)); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_repairs_backup_after_primary_only_commit() { - let store = Arc::new(MemoryConfigStore::default()); - let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); - let durable = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), - scanner_epoch: Some(8), - scanner_cycle: Some(10), - ..complete_usage_with_bucket_count(None, 0) - }; - let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode"); - store.objects.lock().await.insert(main_key.clone(), encoded.clone()); - store.revisions.lock().await.insert(main_key.clone(), 1); - - let (sender, receiver) = mpsc::channel(1); - sender.send(durable).await.expect("usage snapshot should enqueue"); - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, - DataUsagePersistOutcome::AlreadyDurable - ); - assert_eq!(store.objects.lock().await.get(&backup_key), Some(&encoded)); - assert_eq!(store.put_counts.lock().await.get(&main_key), None); - assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1)); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_copies_concurrent_bucket_removal_to_backup() { - let store = Arc::new(MemoryConfigStore::default()); - let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); - - let mut incoming = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), - scanner_epoch: Some(8), - scanner_cycle: Some(10), - ..Default::default() - }; - incoming.buckets_usage.insert( - "bucket-a".to_string(), - rustfs_data_usage::BucketUsageInfo { - objects_count: 2, - size: 84, - ..Default::default() - }, - ); - incoming.bucket_sizes.insert("bucket-a".to_string(), 84); - incoming.buckets_count = 1; - incoming.calculate_totals(); - mark_usage_snapshot_complete(&mut incoming); - - let mut deleted = incoming.clone(); - deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31)); - deleted.buckets_usage.clear(); - deleted.bucket_sizes.clear(); - deleted.buckets_count = 0; - deleted.calculate_totals(); - mark_usage_snapshot_complete(&mut deleted); - store.replace_after_successful_puts.lock().await.insert( - main_key.clone(), - (1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")), - ); - store.objects.lock().await.insert( - backup_key.clone(), - serde_json::to_vec(&incoming).expect("existing backup snapshot should encode"), - ); - store.revisions.lock().await.insert(backup_key.clone(), 1); - - let (sender, receiver) = mpsc::channel(1); - sender.send(incoming).await.expect("usage snapshot should enqueue"); - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, - DataUsagePersistOutcome::Saved - ); - - for key in [main_key, backup_key] { - let saved = store - .objects - .lock() - .await - .get(&key) - .cloned() - .expect("usage snapshot should remain"); - let saved = serde_json::from_slice::(&saved).expect("usage snapshot should decode"); - assert!(!saved.buckets_usage.contains_key("bucket-a")); - assert!(!saved.bucket_sizes.contains_key("bucket-a")); - } - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_retries_after_stale_interleaving_writer() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(1); - let ctx = CancellationToken::new(); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let initial = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 3); - let stale_winner = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 3); - let current = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), 3); - store - .objects - .lock() - .await - .insert(key.clone(), serde_json::to_vec(&initial).expect("initial usage snapshot should encode")); - store.revisions.lock().await.insert(key.clone(), 1); - store.interleaving_puts.lock().await.insert( - key.clone(), - (1, serde_json::to_vec(&stale_winner).expect("stale usage snapshot should encode")), - ); - - sender - .send(current.clone()) - .await - .expect("current usage snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; - - let objects = store.objects.lock().await; - let saved = objects - .get(&key) - .expect("current usage snapshot should replace the stale conflict winner"); - let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); - assert_eq!(saved.buckets_count, 3); - assert_eq!(saved.last_update, current.last_update); - assert_eq!(outcome, DataUsagePersistOutcome::Saved); - drop(objects); - assert_eq!(store.put_counts.lock().await.get(&key), Some(&2)); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapshot() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(2); - let ctx = CancellationToken::new(); - - let timestamped = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2); - let untimestamped = complete_usage_with_bucket_count(None, 1); - - sender - .send(timestamped) - .await - .expect("timestamped usage snapshot should enqueue"); - sender - .send(untimestamped) - .await - .expect("untimestamped usage snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; - - let objects = store.objects.lock().await; - let saved = objects - .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str())) - .expect("data usage config should be saved"); - let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); - - assert_eq!(saved.buckets_count, 2); - assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20))); - assert_eq!(outcome, DataUsagePersistOutcome::Failed); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_recognizes_already_durable_snapshot() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(1); - let ctx = CancellationToken::new(); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let snapshot = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), - scanner_cycle: Some(12), - ..complete_usage_with_bucket_count(None, 2) - }; - store - .objects - .lock() - .await - .insert(key.clone(), serde_json::to_vec(&snapshot).expect("durable usage snapshot should encode")); - store.revisions.lock().await.insert(key.clone(), 1); - - sender - .send(snapshot) - .await - .expect("ambiguous committed snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; - - assert_eq!(outcome, DataUsagePersistOutcome::AlreadyDurable); - assert_eq!(store.put_counts.lock().await.get(&key), None); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_advances_past_changed_same_epoch_cycle() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(1); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let durable = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), - scanner_epoch: Some(8), - scanner_cycle: Some(12), - ..complete_usage_with_bucket_count(None, 2) - }; - store - .objects - .lock() - .await - .insert(key.clone(), serde_json::to_vec(&durable).expect("durable usage snapshot should encode")); - store.revisions.lock().await.insert(key.clone(), 1); - - sender - .send(DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), - scanner_epoch: Some(8), - scanner_cycle: Some(12), - ..complete_usage_with_bucket_count(None, 3) - }) - .await - .expect("changed retry snapshot should enqueue"); - drop(sender); - - let outcome = - store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), receiver, Some(8)).await; - - assert_eq!(outcome, DataUsagePersistOutcome::PriorCycleDurable); - assert_eq!(store.put_counts.lock().await.get(&key), None); - let saved = store - .objects - .lock() - .await - .get(&key) - .cloned() - .expect("first snapshot should remain durable"); - assert_eq!( - serde_json::from_slice::(&saved) - .expect("durable usage snapshot should decode") - .buckets_count, - 2 - ); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_orders_scanner_cycles_before_wall_clock() { - let store = Arc::new(MemoryConfigStore::default()); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let existing = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)), - scanner_cycle: Some(12), - ..complete_usage_with_bucket_count(None, 2) - }; - store - .objects - .lock() - .await - .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); - store.revisions.lock().await.insert(key.clone(), 1); - - let (older_sender, older_receiver) = mpsc::channel(1); - let older = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), - scanner_cycle: Some(11), - ..complete_usage_with_bucket_count(None, 1) - }; - older_sender.send(older).await.expect("older-cycle snapshot should enqueue"); - drop(older_sender); - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await, - DataUsagePersistOutcome::Current - ); - - let (newer_sender, newer_receiver) = mpsc::channel(1); - let newer = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), - scanner_cycle: Some(13), - ..complete_usage_with_bucket_count(None, 3) - }; - newer_sender - .send(newer.clone()) - .await - .expect("newer-cycle snapshot should enqueue"); - drop(newer_sender); - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), newer_receiver).await, - DataUsagePersistOutcome::Saved - ); - - let saved = store - .objects - .lock() - .await - .get(&key) - .cloned() - .expect("newer scanner cycle should be persisted"); - assert_eq!( - serde_json::from_slice::(&saved) - .expect("persisted usage snapshot should decode") - .scanner_cycle, - Some(13) - ); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_orders_leader_epochs_before_cycles() { - let store = Arc::new(MemoryConfigStore::default()); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let existing = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)), - scanner_epoch: Some(8), - scanner_cycle: Some(12), - ..complete_usage_with_bucket_count(None, 2) - }; - store - .objects - .lock() - .await - .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); - store.revisions.lock().await.insert(key.clone(), 1); - - let (older_sender, older_receiver) = mpsc::channel(1); - older_sender - .send(DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), - scanner_epoch: Some(7), - scanner_cycle: Some(99), - ..complete_usage_with_bucket_count(None, 1) - }) - .await - .expect("old-epoch snapshot should enqueue"); - drop(older_sender); - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await, - DataUsagePersistOutcome::Current - ); - - let (newer_sender, newer_receiver) = mpsc::channel(1); - newer_sender - .send(DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), - scanner_epoch: None, - scanner_cycle: Some(1), - ..complete_usage_with_bucket_count(None, 3) - }) - .await - .expect("replacement-epoch snapshot should enqueue"); - drop(newer_sender); - assert_eq!( - store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), newer_receiver, Some(9),) - .await, - DataUsagePersistOutcome::Saved - ); - - let saved = store - .objects - .lock() - .await - .get(&key) - .cloned() - .expect("replacement leader snapshot should persist"); - let saved = serde_json::from_slice::(&saved).expect("persisted usage snapshot should decode"); - assert_eq!(saved.scanner_epoch, Some(9)); - assert_eq!(saved.scanner_cycle, Some(1)); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_keeps_first_same_cycle_snapshot() { - let store = Arc::new(MemoryConfigStore::default()); - let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let existing = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), - scanner_cycle: Some(12), - ..complete_usage_with_bucket_count(None, 2) - }; - store - .objects - .lock() - .await - .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); - store.revisions.lock().await.insert(key.clone(), 1); - - let (sender, receiver) = mpsc::channel(1); - sender - .send(DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), - scanner_cycle: Some(12), - ..complete_usage_with_bucket_count(None, 3) - }) - .await - .expect("conflicting same-cycle snapshot should enqueue"); - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, - DataUsagePersistOutcome::Current - ); - let saved = store - .objects - .lock() - .await - .get(&key) - .cloned() - .expect("first same-cycle snapshot should remain persisted"); - assert_eq!( - serde_json::from_slice::(&saved) - .expect("persisted usage snapshot should decode") - .buckets_count, - 2 - ); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(2); - let complete_update = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10); - - sender - .send(complete_usage_with_bucket_count(Some(complete_update), 1)) - .await - .expect("complete usage snapshot should enqueue"); - sender - .send(DataUsageInfo { - last_update: Some(complete_update + Duration::from_secs(1)), - buckets_count: 1, - ..Default::default() - }) - .await - .expect("incomplete usage snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await; - - let objects = store.objects.lock().await; - let saved = objects - .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str())) - .expect("complete data usage snapshot should remain saved"); - let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); - assert_eq!(saved.last_update, Some(complete_update)); - assert!(saved.is_complete_bucket_usage_snapshot()); - assert_eq!(outcome, DataUsagePersistOutcome::Failed); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_preserves_superseded_status() { - let store = Arc::new(MemoryConfigStore::default()); - let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let authoritative = DataUsageInfo { - scanner_epoch: Some(7), - scanner_cycle: Some(10), - ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 1) - }; - let authoritative_bytes = serde_json::to_vec(&authoritative).expect("authoritative snapshot should encode"); - store - .objects - .lock() - .await - .insert(authoritative_key.clone(), authoritative_bytes.clone()); - store.revisions.lock().await.insert(authoritative_key.clone(), 1); - - let (sender, receiver) = mpsc::channel(1); - sender - .send(DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)), - scanner_epoch: Some(7), - scanner_cycle: Some(11), - usage_snapshot_converged: Some(false), - ..complete_usage_with_bucket_count(None, 1) - }) - .await - .expect("superseded usage snapshot should enqueue"); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await; - let saved = store - .objects - .lock() - .await - .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())) - .cloned() - .expect("superseded usage snapshot should persist"); - let saved = serde_json::from_slice::(&saved).expect("persisted usage snapshot should decode"); - - assert_eq!(outcome, DataUsagePersistOutcome::Saved); - assert!(saved.is_complete_bucket_usage_snapshot()); - assert_eq!(saved.usage_snapshot_converged, Some(false)); - assert_eq!(saved.usage_snapshot_authoritative_baseline, Some(authoritative.snapshot_identity())); - assert_eq!( - store.objects.lock().await.get(&authoritative_key), - Some(&authoritative_bytes), - "an observation must never lower the quota-authoritative snapshot" - ); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_removes_observed_after_authoritative_save() { - let store = Arc::new(MemoryConfigStore::default()); - let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let authoritative = DataUsageInfo { - scanner_epoch: Some(7), - scanner_cycle: Some(10), - ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 1) - }; - store.objects.lock().await.insert( - authoritative_key.clone(), - serde_json::to_vec(&authoritative).expect("authoritative snapshot should encode"), - ); - store.revisions.lock().await.insert(authoritative_key, 1); - - let (sender, receiver) = mpsc::channel(1); - sender - .send(DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)), - scanner_epoch: Some(7), - scanner_cycle: Some(11), - usage_snapshot_converged: Some(false), - ..complete_usage_with_bucket_count(None, 1) - }) - .await - .expect("superseded usage snapshot should enqueue"); - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, - DataUsagePersistOutcome::Saved - ); - let observed_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()); - assert!(store.objects.lock().await.contains_key(&observed_key)); - - let (sender, receiver) = mpsc::channel(1); - sender - .send(DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(2)), - scanner_epoch: Some(7), - scanner_cycle: Some(12), - usage_snapshot_converged: Some(true), - ..complete_usage_with_bucket_count(None, 1) - }) - .await - .expect("authoritative usage snapshot should enqueue"); - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, - DataUsagePersistOutcome::Saved - ); - assert!( - !store.objects.lock().await.contains_key(&observed_key), - "an authoritative snapshot should retire stale observations" - ); - - let next_authoritative = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(3)), - scanner_epoch: Some(7), - scanner_cycle: Some(13), - usage_snapshot_converged: Some(true), - ..complete_usage_with_bucket_count(None, 1) - }; - let newer_observed = DataUsageInfo { - last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(4)), - scanner_epoch: Some(7), - scanner_cycle: Some(14), - usage_snapshot_converged: Some(false), - usage_snapshot_authoritative_baseline: Some(next_authoritative.snapshot_identity()), - ..complete_usage_with_bucket_count(None, 1) - }; - store.objects.lock().await.insert( - observed_key.clone(), - serde_json::to_vec(&newer_observed).expect("newer observed snapshot should encode"), - ); - store.revisions.lock().await.insert(observed_key.clone(), 3); - - let (sender, receiver) = mpsc::channel(1); - sender - .send(next_authoritative) - .await - .expect("next authoritative usage snapshot should enqueue"); - drop(sender); - - assert_eq!( - store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, - DataUsagePersistOutcome::Saved - ); - assert!( - store.objects.lock().await.contains_key(&observed_key), - "a newer observation must survive stale authoritative cleanup" - ); - } - - fn mark_usage_snapshot_complete(info: &mut DataUsageInfo) { - info.usage_snapshot_complete = true; - } - - fn complete_usage_with_bucket_count(last_update: Option, buckets_count: u64) -> DataUsageInfo { - let mut info = DataUsageInfo { - last_update, - buckets_count, - usage_snapshot_complete: true, - ..Default::default() - }; - for index in 0..buckets_count { - let bucket = format!("bucket-{index}"); - info.buckets_usage.insert(bucket.clone(), Default::default()); - info.bucket_sizes.insert(bucket, 0); - } - info - } - - fn usage_with_last_update(last_update: Option) -> DataUsageInfo { - complete_usage_with_bucket_count(last_update, 0) - } - - #[test] - fn test_stale_data_usage_update_reason_allows_newer_incoming() { - let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); - let incoming = usage_with_last_update(Some(now)); - let existing = usage_with_last_update(Some(now - Duration::from_secs(60))); - assert_eq!(stale_data_usage_update_reason(&incoming, &existing, now), None); - } - - #[test] - fn test_stale_data_usage_update_reason_skips_older_or_equal_incoming() { - let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); - let existing = usage_with_last_update(Some(now - Duration::from_secs(60))); - - let older = usage_with_last_update(Some(now - Duration::from_secs(120))); - assert_eq!(stale_data_usage_update_reason(&older, &existing, now), Some("older_or_equal_last_update")); - - let equal = usage_with_last_update(existing.last_update); - assert_eq!(stale_data_usage_update_reason(&equal, &existing, now), Some("older_or_equal_last_update")); - } - - #[test] - fn test_stale_data_usage_update_reason_allows_save_when_existing_is_future_dated() { - // Existing snapshot timestamp beyond the clock tolerance is untrustworthy - // (clock step-back / slower-clock leader): the save must be allowed even - // though incoming <= existing, otherwise usage stats freeze forever. - let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); - let existing = - usage_with_last_update(Some(now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(1))); - let incoming = usage_with_last_update(Some(now)); - assert_eq!(stale_data_usage_update_reason(&incoming, &existing, now), None); - } - - #[test] - fn test_stale_data_usage_update_reason_skips_at_exact_tolerance_boundary() { - // Exactly at now + tolerance is still within the trusted window. - let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); - let existing = usage_with_last_update(Some(now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE)); - let incoming = usage_with_last_update(Some(now)); - assert_eq!( - stale_data_usage_update_reason(&incoming, &existing, now), - Some("older_or_equal_last_update") - ); - } - - #[test] - fn test_stale_data_usage_update_reason_preserves_none_handling() { - let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); - - let incoming_none = usage_with_last_update(None); - let existing_some = usage_with_last_update(Some(now - Duration::from_secs(60))); - assert_eq!( - stale_data_usage_update_reason(&incoming_none, &existing_some, now), - Some("missing_incoming_last_update") - ); - - let incoming_some = usage_with_last_update(Some(now)); - let existing_none = usage_with_last_update(None); - assert_eq!(stale_data_usage_update_reason(&incoming_some, &existing_none, now), None); - - let both_none = usage_with_last_update(None); - assert_eq!(stale_data_usage_update_reason(&both_none, &usage_with_last_update(None), now), None); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_keeps_backup_when_primary_save_fails() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(11); - let ctx = CancellationToken::new(); - - let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); - let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); - let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); - let old_backup = b"old-backup".to_vec(); - - store.objects.lock().await.insert(backup_key.clone(), old_backup.clone()); - store.fail_put_number.lock().await.insert(main_key.clone(), 11); - - for idx in 1_u64..=11 { - sender - .send(complete_usage_with_bucket_count( - Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)), - idx, - )) - .await - .expect("usage snapshot should enqueue"); - } - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; - - let objects = store.objects.lock().await; - assert_eq!( - objects.get(&backup_key), - Some(&old_backup), - "primary save failure must not overwrite the previous backup" - ); - let saved = objects - .get(&main_key) - .expect("last successful primary usage snapshot should remain saved"); - let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); - assert_eq!(saved.buckets_count, 10); - assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10))); - assert_eq!(outcome, DataUsagePersistOutcome::Failed); - } - - #[tokio::test] - async fn test_store_data_usage_in_backend_reports_missing_snapshot() { - let store = Arc::new(MemoryConfigStore::default()); - let (sender, receiver) = mpsc::channel(1); - let ctx = CancellationToken::new(); - drop(sender); - - let outcome = store_data_usage_in_backend_with_outcome(ctx, store, receiver).await; - - assert_eq!(outcome, DataUsagePersistOutcome::NoUpdate); - } - - #[test] - fn test_scanner_cycle_completion_prioritizes_persist_failure() { - assert_eq!( - scanner_cycle_completion_outcome( - ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable), - DataUsagePersistOutcome::NoUpdate, - false, - false, - ), - ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable) - ); - assert_eq!( - scanner_cycle_completion_outcome( - ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), - DataUsagePersistOutcome::Saved, - false, - false, - ), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome( - ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), - DataUsagePersistOutcome::NoUpdate, - true, - false, - ), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome( - ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), - DataUsagePersistOutcome::Failed, - false, - false, - ), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, false, false), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Failed, true, true), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, true, true), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, false), - ScannerCycleOutcome::Partial - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, true), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Saved, true, false), - ScannerCycleOutcome::Completed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::AlreadyDurable, true, false,), - ScannerCycleOutcome::Completed - ); - assert_eq!( - scanner_cycle_completion_outcome( - ScannerCycleStatus::Complete, - DataUsagePersistOutcome::PriorCycleDurable, - true, - false, - ), - ScannerCycleOutcome::Completed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, false, false), - ScannerCycleOutcome::Completed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, true, false), - ScannerCycleOutcome::Failed - ); - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::NoUpdate, false, false), - ScannerCycleOutcome::Failed - ); - for persist_outcome in [ - DataUsagePersistOutcome::NoUpdate, - DataUsagePersistOutcome::Current, - DataUsagePersistOutcome::Saved, - ] { - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, persist_outcome, true, false), - ScannerCycleOutcome::Superseded - ); - } - assert_eq!( - scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, DataUsagePersistOutcome::Saved, true, true), - ScannerCycleOutcome::Failed - ); - } - - #[test] - #[serial] - fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() { - crate::scanner_io::clear_dirty_usage_bucket("photos"); - crate::scanner_io::record_dirty_usage_bucket("photos"); - let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); - - let remote_acknowledgement = ScannerDirtyUsageAcknowledgement { - host: "node-2".to_string(), - instance_id: "0123456789abcdef0123456789abcdef".to_string(), - generation: 11, - }; - let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone())) - .with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]); - let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate); - assert_eq!(outcome, ScannerCycleOutcome::Failed); - assert!(acknowledgements.is_empty()); - assert!(crate::scanner_io::dirty_usage_buckets_pending()); - - let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)) - .with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]); - let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved); - assert_eq!(outcome, ScannerCycleOutcome::Completed); - assert_eq!(acknowledgements, vec![remote_acknowledgement]); - assert!(!crate::scanner_io::dirty_usage_buckets_pending()); - } - - #[tokio::test] - async fn scanner_cycle_keeps_remote_pending_acknowledgement() { - let pending = - remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::(true))).await; - assert_eq!( - scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending), - ScannerCycleOutcome::CompletedWithPendingMaintenance - ); - - let cleared = - remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::(false))).await; - assert_eq!( - scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, cleared), - ScannerCycleOutcome::Completed - ); - - let failed = remote_dirty_usage_acknowledgement_pending( - 7, - 1, - std::future::ready(Err::(std::io::Error::other("injected acknowledgement failure"))), - ) - .await; - assert_eq!( - scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, failed), - ScannerCycleOutcome::CompletedWithPendingMaintenance - ); - } - - #[test] - #[serial] - fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() { - crate::scanner_io::clear_dirty_usage_bucket("photos"); - crate::scanner_io::record_dirty_usage_bucket("photos"); - let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); - - let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); - let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable); - - assert_eq!(outcome, ScannerCycleOutcome::Completed); - assert!(acknowledgements.is_empty()); - assert!(!crate::scanner_io::dirty_usage_buckets_pending()); - } - - #[test] - #[serial] - fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() { - crate::scanner_io::clear_dirty_usage_bucket("photos"); - crate::scanner_io::record_dirty_usage_bucket("photos"); - let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); - - let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); - let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable); - - assert_eq!(outcome, ScannerCycleOutcome::Completed); - assert!(acknowledgements.is_empty()); - assert!(crate::scanner_io::dirty_usage_buckets_pending()); - crate::scanner_io::clear_dirty_usage_bucket("photos"); - } - - #[test] - #[serial] - fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() { - crate::scanner_io::clear_dirty_usage_bucket("photos"); - crate::scanner_io::record_dirty_usage_bucket("photos"); - let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); - - let superseded = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Superseded, Some(dirty_snapshot)); - let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::Saved); - - assert_eq!(outcome, ScannerCycleOutcome::Superseded); - assert!(acknowledgements.is_empty()); - assert!(crate::scanner_io::dirty_usage_buckets_pending()); - crate::scanner_io::clear_dirty_usage_bucket("photos"); - } - - #[test] - #[serial] - fn data_usage_persist_wait_covers_cache_retries_and_backup() { - with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || { - crate::runtime_config::refresh_scanner_runtime_config_for_tests(); - assert_eq!(data_usage_persist_timeout(), Duration::from_millis(31_350)); - }); - crate::runtime_config::refresh_scanner_runtime_config_for_tests(); - } - - #[tokio::test] - async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() { - let ctx = CancellationToken::new(); - let mut task = AbortOnDropHandle::new(tokio::spawn(async { - std::future::pending::<()>().await; - DataUsagePersistOutcome::Saved - })); - ctx.cancel(); - - let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(60)).await; - - assert!(matches!(result, DataUsagePersistTaskResult::Cancelled)); - assert!(task.is_finished()); - } - - #[tokio::test(start_paused = true)] - async fn data_usage_persist_wait_aborts_after_timeout() { - let ctx = CancellationToken::new(); - let mut task = AbortOnDropHandle::new(tokio::spawn(async { - std::future::pending::<()>().await; - DataUsagePersistOutcome::Saved - })); - - let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(30)).await; - - assert!(matches!(result, DataUsagePersistTaskResult::TimedOut)); - assert!(task.is_finished()); - } - - #[tokio::test(start_paused = true)] - async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() { - let ctx = CancellationToken::new(); - - let result = wait_for_maintenance_feature_inspection( - &ctx, - std::future::pending::(), - Duration::from_secs(30), - ) - .await; - - assert_eq!(result, MaintenanceInspectionAttempt::TimedOut); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() { - let ctx = CancellationToken::new(); - - let (features, generation) = detect_stable_scanner_maintenance_features_with( - &ctx, - std::future::pending::, - Duration::from_secs(30), - ) - .await - .expect("timeout should preserve the scanner rather than stop it"); - - assert!(features.inspection_failed); - assert_eq!(generation, scanner_maintenance_generation()); - assert!(!scanner_clean_idle_backoff_enabled( - true, - true, - features, - &ScannerRuntimeConfig::default() - )); - } - - #[tokio::test(start_paused = true)] - async fn failed_maintenance_inspection_uses_bounded_retry_backoff() { - let failed = ScannerMaintenanceFeatures { - inspection_failed: true, - ..Default::default() - }; - let mut retry = ScannerMaintenanceInspectionRetry::from_features(failed, Instant::now()); - - assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL)); - assert!(!retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now())); - tokio::time::advance(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL).await; - assert!(retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now())); - assert!(!retry.retry_due(failed, ScannerCycleWakeReason::DirtyUsage, Instant::now())); - - retry.record_inspection(failed, Instant::now()); - assert_eq!( - retry.retry_interval(), - Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL.saturating_mul(2)) - ); - for _ in 0..8 { - retry.record_inspection(failed, Instant::now()); - } - assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL)); - - retry.record_inspection(ScannerMaintenanceFeatures::default(), Instant::now()); - assert_eq!(retry, ScannerMaintenanceInspectionRetry::default()); - } - - #[tokio::test] - async fn maintenance_feature_inspection_stops_on_cancellation() { - let ctx = CancellationToken::new(); - ctx.cancel(); - - let result = wait_for_maintenance_feature_inspection( - &ctx, - std::future::pending::(), - Duration::from_secs(30), - ) - .await; - - assert_eq!(result, MaintenanceInspectionAttempt::Cancelled); - } - - #[test] - #[serial] - fn test_cycle_interval_prefers_explicit_cycle_override() { - with_var(ENV_SCANNER_SPEED, Some("slowest"), || { - with_var(ENV_SCANNER_CYCLE, Some("42"), || { - assert_eq!(cycle_interval(), Duration::from_secs(42)); - }); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() { - let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); - - with_var(ENV_SCANNER_CYCLE, Some("42"), || { - assert_eq!(cycle_interval(), Duration::from_secs(42)); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() { - let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest); - - with_unset_scanner_timing_env(|| { - assert_eq!(cycle_interval(), Duration::from_secs(30 * 60)); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() { - let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest); - - with_var_unset(ENV_SCANNER_CYCLE, || { - with_var_unset("MINIO_SCANNER_CYCLE", || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || { - with_var(ENV_SCANNER_SPEED, Some("fastest"), || { - assert_eq!(cycle_interval(), Duration::from_secs(1)); - }); - }); - }); - }); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() { - let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); - - with_unset_scanner_timing_env(|| { - assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS)); - }); - } - - #[test] - fn test_single_disk_default_speed_uses_regular_scanner_default() { - assert_eq!(single_disk_default_speed(), ScannerSpeed::Default); - } - - #[test] - fn test_maintenance_feature_inspection_is_bounded_and_conservative() { - assert_eq!(maintenance_inspection_decision(1, 1, 1), MaintenanceInspectionDecision::Accept); - assert_eq!(maintenance_inspection_decision(1, 2, 1), MaintenanceInspectionDecision::Retry); - assert_eq!( - maintenance_inspection_decision(1, 2, MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS), - MaintenanceInspectionDecision::PreserveBaseCycle - ); - } - - #[test] - fn clean_idle_backoff_grows_to_cap() { - let base_interval = Duration::from_secs(60); - let max_interval = CLEAN_IDLE_MAX_INTERVAL; - let mut backoff = ScannerCleanIdleBackoff::default(); - - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(60)); - for expected_secs in [ - 120, 240, 480, 960, 1_920, 3_840, 7_680, 15_360, 30_720, 61_440, 86_400, 86_400, - ] { - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - assert_eq!( - backoff.effective_interval(base_interval, max_interval, true), - Duration::from_secs(expected_secs) - ); - } - } - - #[test] - fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() { - let mut backoff = ScannerRetryBackoff::default(); - assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None); - - for expected in [5, 10, 20, 40, 80, 160, 320] { - backoff.record_retryable_cycle(true); - assert_eq!( - backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), - Some(Duration::from_secs(expected)) - ); - } - for _ in 0..20 { - backoff.record_retryable_cycle(true); - } - assert_eq!( - backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), - Some(Duration::from_secs(24 * 60 * 60)) - ); - - backoff.record_retryable_cycle(false); - assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None); - } - - #[test] - fn superseded_retry_backoff_respects_a_faster_configured_cycle() { - let mut backoff = ScannerRetryBackoff::default(); - backoff.record_retryable_cycle(true); - - // A configured cycle shorter than the base still wins: retrying sooner - // than the operator's own cadence buys nothing. - assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(3))); - backoff.record_retryable_cycle(true); - assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(6))); - } - - #[test] - fn superseded_retry_backoff_grows_from_the_default_cycle() { - let mut backoff = ScannerRetryBackoff::default(); - // The first race after a write burst retries in seconds, not a whole - // cycle, while repeated supersedes still climb toward the cap. - for expected in [5, 10, 20, 40] { - backoff.record_retryable_cycle(true); - assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(expected))); - } - } - - #[test] - fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() { - let runtime_config = ScannerRuntimeConfig { - cycle_interval: Duration::from_secs(60), - bitrot_cycle: None, - ..Default::default() - }; - let mut clean_idle_backoff = ScannerCleanIdleBackoff::default(); - - let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); - assert_eq!(plan.delay, Duration::from_secs(60)); - - for expected in [120, 240] { - record_scanner_cycle_result( - &mut clean_idle_backoff, - &runtime_config, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); - assert_eq!(plan.delay, Duration::from_secs(expected)); - } - - for (wake_reason, outcome, dirty_work_observed) in [ - (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Completed, true), - (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Partial, false), - (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Failed, false), - (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::CompletedWithPendingMaintenance, false), - (ScannerCycleWakeReason::DirtyUsage, ScannerCycleOutcome::Completed, false), - ] { - record_scanner_cycle_result( - &mut clean_idle_backoff, - &runtime_config, - true, - wake_reason, - outcome, - dirty_work_observed, - ); - let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); - assert_eq!(plan.effective_interval, Duration::from_secs(60)); - assert_eq!(plan.delay, Duration::from_secs(60)); - - record_scanner_cycle_result( - &mut clean_idle_backoff, - &runtime_config, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - } - - clean_idle_backoff.reset(); - for _ in 0..32 { - record_scanner_cycle_result( - &mut clean_idle_backoff, - &runtime_config, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - } - let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, |interval| interval.mul_f64(1.1)); - assert_eq!(plan.effective_interval, CLEAN_IDLE_MAX_INTERVAL); - assert!(plan.delay < CLEAN_IDLE_MAX_INTERVAL); - assert_eq!( - plan.delay, - CLEAN_IDLE_MAX_INTERVAL.saturating_sub(CLEAN_IDLE_MAX_INTERVAL.mul_f64(1.1) - CLEAN_IDLE_MAX_INTERVAL) - ); - } - - #[test] - #[serial] - fn scanner_cycle_schedule_status_reports_effective_backoff() { - record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7); - - let status = scanner_cycle_schedule_status(); - - assert_eq!(status.effective_interval_seconds, 86_401); - assert!(status.clean_idle_backoff_enabled); - assert_eq!(status.clean_idle_backoff_multiplier, 2_048); - assert!(status.superseded_retry_backoff_enabled); - assert_eq!(status.superseded_cycles, 7); - - reset_scanner_cycle_schedule(); - let status = scanner_cycle_schedule_status(); - assert_eq!(status.effective_interval_seconds, 0); - assert!(!status.clean_idle_backoff_enabled); - assert_eq!(status.clean_idle_backoff_multiplier, 1); - assert!(!status.superseded_retry_backoff_enabled); - assert_eq!(status.superseded_cycles, 0); - } - - #[test] - fn clean_idle_backoff_resets_for_non_idle_work() { - let base_interval = Duration::from_secs(60); - let max_interval = CLEAN_IDLE_MAX_INTERVAL; - let mut backoff = ScannerCleanIdleBackoff::default(); - - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240)); - - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::DirtyUsage, - ScannerCycleOutcome::Completed, - false, - ); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); - - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Partial, - false, - ); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); - - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Failed, - false, - ); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); - - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - true, - ); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); - - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::CompletedWithPendingMaintenance, - false, - ); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); - } - - #[test] - fn test_dirty_work_is_observed_across_cycle_waits() { - assert!(scanner_cycle_observed_dirty_work(true, 7, 7)); - assert!(scanner_cycle_observed_dirty_work(false, 7, 8)); - assert!(!scanner_cycle_observed_dirty_work(false, 7, 7)); - } - - #[test] - fn clean_idle_backoff_never_shortens_base_interval() { - let base_interval = Duration::from_secs(48 * 60 * 60); - let mut backoff = ScannerCleanIdleBackoff::default(); - - backoff.record_cycle( - base_interval, - CLEAN_IDLE_MAX_INTERVAL, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - - assert_eq!(backoff.effective_interval(base_interval, CLEAN_IDLE_MAX_INTERVAL, true), base_interval); - } - - #[test] - fn clean_idle_backoff_resets_while_disabled() { - let base_interval = Duration::from_secs(60); - let max_interval = CLEAN_IDLE_MAX_INTERVAL; - let mut backoff = ScannerCleanIdleBackoff::default(); - - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - backoff.record_cycle( - base_interval, - max_interval, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240)); - - backoff.record_cycle( - base_interval, - max_interval, - false, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - false, - ); - - assert_eq!(backoff.effective_interval(base_interval, max_interval, false), base_interval); - assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); - } - - #[test] - fn clean_idle_backoff_policy_preserves_explicit_and_maintenance_cycles() { - let no_features = ScannerMaintenanceFeatures::default(); - let default_config = ScannerRuntimeConfig::default(); - assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &default_config)); - assert!(!scanner_clean_idle_backoff_enabled(false, true, no_features, &default_config)); - assert!(!scanner_clean_idle_backoff_enabled(true, false, no_features, &default_config)); - - for source in [ScannerRuntimeConfigSource::Env, ScannerRuntimeConfigSource::Config] { - let mut config = default_config.clone(); - config.cycle_interval_source = source; - assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &config)); - } - - for source in [ - ScannerRuntimeConfigSource::Env, - ScannerRuntimeConfigSource::Config, - ScannerRuntimeConfigSource::ScannerCompatConfig, - ] { - let mut explicit_bitrot_config = default_config.clone(); - explicit_bitrot_config.bitrot_cycle = Some(Duration::from_secs(60 * 60)); - explicit_bitrot_config.bitrot_cycle_source = source; - assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config)); - - explicit_bitrot_config.bitrot_cycle = None; - assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config)); - } - - for features in [ - ScannerMaintenanceFeatures { - lifecycle: true, - ..Default::default() - }, - ScannerMaintenanceFeatures { - replication: true, - ..Default::default() - }, - ScannerMaintenanceFeatures { - inspection_failed: true, - ..Default::default() - }, - ] { - assert!(!scanner_clean_idle_backoff_enabled(true, true, features, &default_config)); - } - } - - #[test] - fn clean_idle_backoff_requires_activity_probes() { - let default_config = ScannerRuntimeConfig::default(); - let no_features = ScannerMaintenanceFeatures::default(); - assert!(scanner_activity_probe_required(true, false, no_features, &default_config)); - assert!(!scanner_activity_probe_required(false, false, no_features, &default_config)); - assert!(!scanner_activity_probe_required(true, true, no_features, &default_config)); - - let mut explicit_cycle = default_config.clone(); - explicit_cycle.cycle_interval_source = ScannerRuntimeConfigSource::Env; - assert!(!scanner_activity_probe_required(true, false, no_features, &explicit_cycle)); - - let lifecycle = ScannerMaintenanceFeatures { - lifecycle: true, - ..Default::default() - }; - assert!(!scanner_activity_probe_required(true, false, lifecycle, &default_config)); - } - - #[test] - #[serial] - fn clean_idle_cap_preserves_default_bitrot_coverage_window() { - let config = ScannerRuntimeConfig { - bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)), - bitrot_cycle_source: ScannerRuntimeConfigSource::Default, - ..Default::default() - }; - - with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || { - let max_interval = scanner_clean_idle_max_interval(Duration::from_secs(60), &config); - assert_eq!(max_interval, Duration::from_millis(2_531_250)); - let positive_jitter = max_interval.mul_f64(1.1); - let actual_delay = cap_clean_idle_cycle_delay(positive_jitter, max_interval, true); - assert!(actual_delay < max_interval); - assert_eq!(actual_delay, max_interval.saturating_sub(positive_jitter - max_interval)); - assert!(actual_delay.saturating_mul(1024) <= config.bitrot_cycle.expect("bitrot cycle should be configured")); - }); - } - - #[test] - fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() { - let config = ScannerRuntimeConfig { - bitrot_cycle: None, - ..Default::default() - }; - - assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), CLEAN_IDLE_MAX_INTERVAL); - } - - #[test] - #[serial] - fn clean_idle_cap_never_shortens_the_base_cycle() { - let config = ScannerRuntimeConfig { - bitrot_cycle: Some(Duration::from_secs(60)), - bitrot_cycle_source: ScannerRuntimeConfigSource::Default, - ..Default::default() - }; - - with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || { - assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), Duration::from_secs(60)); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() { - let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); - - with_var_unset(ENV_SCANNER_CYCLE, || { - with_var_unset("MINIO_SCANNER_CYCLE", || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || { - with_var(ENV_SCANNER_SPEED, Some("slowest"), || { - assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS)); - }); - }); - }); - }); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() { - let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); - - with_var_unset(ENV_SCANNER_CYCLE, || { - with_var_unset("MINIO_SCANNER_CYCLE", || { - with_var(ENV_SCANNER_START_DELAY_SECS, Some("120"), || { - assert_eq!(cycle_interval(), Duration::from_secs(120)); - }); - }); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_supports_minio_speed_alias() { - with_var_unset(ENV_SCANNER_SPEED, || { - with_var_unset(ENV_SCANNER_CYCLE, || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { - with_var("MINIO_SCANNER_SPEED", Some("slowest"), || { - assert_eq!(cycle_interval(), Duration::from_secs(30 * 60)); - }); - }); - }); - }); - } - - #[test] - #[serial] - fn test_cycle_interval_supports_minio_cycle_alias() { - with_var_unset(ENV_SCANNER_CYCLE, || { - with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { - with_var("MINIO_SCANNER_CYCLE", Some("90"), || { - assert_eq!(cycle_interval(), Duration::from_secs(90)); - }); - }); - }); - } - - #[test] - fn test_randomized_cycle_delay_handles_small_start_delay() { - // 0 is treated as minimum 1 second before jitter, with lower bound preserved. - let delay = randomized_cycle_delay_for(Duration::from_secs(0)); - assert!(delay >= Duration::from_secs(1), "expected delay >= 1s"); - assert!(delay < Duration::from_secs(2), "expected delay < 2s"); - } - - #[tokio::test] - #[serial] - async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - - let ctx = CancellationToken::new(); - let dirty_generation = crate::scanner_io::dirty_usage_generation(); - let mut wait = Box::pin(wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - Some(dirty_generation), - crate::runtime_config::scanner_runtime_config_generation(), - crate::scanner_io::scanner_maintenance_generation(), - || false, - )); - assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); - - crate::scanner_io::record_dirty_usage_bucket("photos"); - let reason = tokio::time::timeout(Duration::from_secs(1), wait) - .await - .expect("dirty usage should wake scanner before timer"); - - assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - } - - #[tokio::test] - #[serial] - async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let dirty_generation = crate::scanner_io::dirty_usage_generation(); - crate::scanner_io::record_dirty_usage_bucket("photos"); - - let ctx = CancellationToken::new(); - let reason = wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - Some(dirty_generation), - crate::runtime_config::scanner_runtime_config_generation(), - crate::scanner_io::scanner_maintenance_generation(), - || false, - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - crate::scanner_io::record_dirty_usage_bucket("photos"); - let dirty_generation = crate::scanner_io::dirty_usage_generation(); - let ctx = CancellationToken::new(); - let wait = wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - Some(dirty_generation), - crate::runtime_config::scanner_runtime_config_generation(), - crate::scanner_io::scanner_maintenance_generation(), - || false, - ); - - let reason = wait.await; - - assert_eq!(reason, ScannerCycleWakeReason::Timer); - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let wait = wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - None, - crate::runtime_config::scanner_runtime_config_generation(), - crate::scanner_io::scanner_maintenance_generation(), - || false, - ); - - crate::scanner_io::record_dirty_usage_bucket("photos"); - assert_eq!(wait.await, ScannerCycleWakeReason::Timer); - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - } - - #[tokio::test] - #[serial] - async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - crate::scanner_io::record_dirty_usage_bucket("photos"); - let dirty_generation = crate::scanner_io::dirty_usage_generation(); - let ctx = CancellationToken::new(); - let mut wait = Box::pin(wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - Some(dirty_generation), - crate::runtime_config::scanner_runtime_config_generation(), - crate::scanner_io::scanner_maintenance_generation(), - || false, - )); - assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); - - crate::scanner_io::record_dirty_usage_bucket("photos"); - let reason = tokio::time::timeout(Duration::from_secs(1), wait) - .await - .expect("a newer mutation of an already-dirty bucket should wake scanner"); - - assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - } - - #[tokio::test] - #[serial] - async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let observed_generation = crate::runtime_config::scanner_runtime_config_generation(); - let ctx = CancellationToken::new(); - let mut wait = Box::pin(wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - Some(crate::scanner_io::dirty_usage_generation()), - observed_generation, - crate::scanner_io::scanner_maintenance_generation(), - || false, - )); - assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); - - let mut config = rustfs_config::server_config::Config::new(); - config.set_defaults(); - crate::runtime_config::apply_scanner_runtime_config(&config).expect("default scanner config should apply"); - let reason = tokio::time::timeout(Duration::from_secs(1), wait) - .await - .expect("runtime config should wake scanner before timer"); - - assert_eq!(reason, ScannerCycleWakeReason::RuntimeConfig); - crate::runtime_config::refresh_scanner_runtime_config_for_tests(); - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - } - - #[tokio::test] - #[serial] - async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let observed_generation = crate::scanner_io::scanner_maintenance_generation(); - let ctx = CancellationToken::new(); - let mut wait = Box::pin(wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - Some(crate::scanner_io::dirty_usage_generation()), - crate::runtime_config::scanner_runtime_config_generation(), - observed_generation, - || false, - )); - assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); - - crate::scanner_io::record_scanner_maintenance_change("photos"); - let reason = tokio::time::timeout(Duration::from_secs(1), wait) - .await - .expect("maintenance change should wake scanner before timer"); - - assert_eq!(reason, ScannerCycleWakeReason::MaintenanceConfig); - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - } - - #[tokio::test] - async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() { - let ctx = CancellationToken::new(); - let reason = wait_for_next_scanner_cycle( - &ctx, - Duration::from_secs(60), - Some(crate::scanner_io::dirty_usage_generation()), - crate::runtime_config::scanner_runtime_config_generation(), - crate::scanner_io::scanner_maintenance_generation(), - || true, - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost); - } - - fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity { - ScannerNodeActivity { - instance_id: epoch.to_string(), - namespace_generation, - maintenance_generation, - protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, - topology_digest: [3; 32], - data_movement_active: false, - dirty_usage_generation: 5, - dirty_usage_pending: false, - } - } - - #[test] - fn scanner_activity_snapshot_digest_fences_storage_topology() { - let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); - let mut changed = first.clone(); - changed.get_mut("node-2").expect("node should exist").topology_digest = [4; 32]; - - assert_ne!(scanner_activity_snapshot_digest(&first), scanner_activity_snapshot_digest(&changed)); - } - - #[test] - fn scanner_activity_snapshot_digest_fences_peer_protocol_upgrades() { - let legacy = BTreeMap::from([( - "node-2".to_string(), - ScannerNodeActivity { - protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, - ..scanner_node_activity("epoch-a", 7, 3) - }, - )]); - let previous = BTreeMap::from([( - "node-2".to_string(), - ScannerNodeActivity { - protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, - ..scanner_node_activity("epoch-a", 7, 3) - }, - )]); - let current = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); - - assert_ne!(scanner_activity_snapshot_digest(&legacy), scanner_activity_snapshot_digest(¤t)); - assert_ne!(scanner_activity_snapshot_digest(&previous), scanner_activity_snapshot_digest(¤t)); - } - - #[test] - fn scanner_activity_snapshot_fences_data_movement() { - let idle = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); - let mut moving = idle.clone(); - moving.get_mut("node-2").expect("node should exist").data_movement_active = true; - - assert!(scanner_activity_allows_usage_publication(&idle)); - assert!(!scanner_activity_allows_usage_publication(&moving)); - assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving)); - } - - #[test] - fn scanner_activity_snapshot_digest_fences_dirty_usage_state() { - let clean = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); - let pending = BTreeMap::from([( - "node-2".to_string(), - ScannerNodeActivity { - dirty_usage_generation: 6, - dirty_usage_pending: true, - ..scanner_node_activity("epoch-a", 7, 3) - }, - )]); - - assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending)); - } - - #[test] - fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() { - let snapshot = BTreeMap::from([ - ( - LOCAL_SCANNER_ACTIVITY_NODE.to_string(), - ScannerNodeActivity { - dirty_usage_generation: 7, - dirty_usage_pending: true, - ..scanner_node_activity("epoch-local", 7, 3) - }, - ), - ("node-2".to_string(), scanner_node_activity("epoch-clean", 7, 3)), - ( - "node-3".to_string(), - ScannerNodeActivity { - dirty_usage_generation: 11, - dirty_usage_pending: true, - ..scanner_node_activity("epoch-dirty", 7, 3) - }, - ), - ]); - - assert_eq!( - scanner_dirty_usage_acknowledgements(&snapshot), - vec![ScannerDirtyUsageAcknowledgement { - host: "node-3".to_string(), - instance_id: "epoch-dirty".to_string(), - generation: 11, - }] - ); - } - - #[test] - fn scanner_activity_rejects_one_process_claimed_by_multiple_hosts() { - let mut instances = BTreeMap::new(); - record_scanner_activity_instance(&mut instances, "node-1", "0123456789abcdef0123456789abcdef") - .expect("first host should establish the instance identity"); - let err = record_scanner_activity_instance(&mut instances, "node-2", "0123456789abcdef0123456789abcdef") - .expect_err("a process identity must not represent two cluster nodes"); - - assert!(err.contains("node-1 and node-2")); - } - - #[test] - fn scanner_activity_observation_requires_a_complete_baseline() { - let mut seen = None; - let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); - - let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first.clone())); - assert_eq!(observation, ScannerActivityObservation::Unverified); - assert!(error.is_none()); - - let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first)); - assert_eq!(observation, ScannerActivityObservation::Unchanged); - assert!(error.is_none()); - - let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); - let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(changed)); - assert_eq!(observation, ScannerActivityObservation::Changed); - assert!(error.is_none()); - - let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]); - let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted)); - assert_eq!(observation, ScannerActivityObservation::Changed); - assert!(error.is_none()); - - let (observation, error) = - apply_scanner_activity_probe_result(&mut seen, Err("peer does not support activity probes".to_string())); - assert_eq!(observation, ScannerActivityObservation::Unverified); - assert_eq!(error.as_deref(), Some("peer does not support activity probes")); - assert!(seen.is_none()); - } - - #[test] - fn remote_maintenance_change_is_distinct_from_namespace_activity() { - let previous = BTreeMap::from([ - (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)), - ("node-2".to_string(), scanner_node_activity("remote", 7, 3)), - ]); - let remote_maintenance_changed = BTreeMap::from([ - (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)), - ("node-2".to_string(), scanner_node_activity("remote", 7, 4)), - ]); - assert_eq!( - compare_scanner_activity(&previous, &remote_maintenance_changed), - ScannerActivityObservation::MaintenanceChanged - ); - - let local_maintenance_changed = BTreeMap::from([ - (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 3)), - ("node-2".to_string(), scanner_node_activity("remote", 7, 3)), - ]); - assert_eq!( - compare_scanner_activity(&previous, &local_maintenance_changed), - ScannerActivityObservation::Changed - ); - } - - #[test] - fn local_maintenance_wakeup_releases_a_remote_maintenance_block() { - let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance); - assert!(blocked); - - let unblocked = scanner_activity_backoff_blocked_after_wake(blocked, ScannerCycleWakeReason::MaintenanceConfig); - assert!(!unblocked); - assert!(scanner_activity_backoff_blocked_after_wake( - blocked, - ScannerCycleWakeReason::ClusterActivity - )); - } - - #[test] - fn scanner_activity_after_a_cycle_restores_the_base_interval() { - let runtime_config = ScannerRuntimeConfig { - cycle_interval: Duration::from_secs(60), - ..Default::default() - }; - let mut backoff = ScannerCleanIdleBackoff { interval_multiplier: 8 }; - - record_scanner_cycle_result( - &mut backoff, - &runtime_config, - true, - ScannerCycleWakeReason::Timer, - ScannerCycleOutcome::Completed, - scanner_activity_observed_work(ScannerActivityObservation::Changed), - ); - - let plan = scanner_cycle_wait_plan(&runtime_config, backoff, true, std::convert::identity); - assert_eq!(plan.effective_interval, Duration::from_secs(60)); - assert_eq!(plan.delay, Duration::from_secs(60)); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); - let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); - - let reason = wait_for_next_scanner_cycle_with_activity( - &ctx, - Duration::from_secs(120), - Some(Duration::from_secs(60)), - &mut seen, - ScannerCycleObservedGenerations { - dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), - runtime_config: crate::runtime_config::scanner_runtime_config_generation(), - maintenance: crate::scanner_io::scanner_maintenance_generation(), - defer_cluster_activity: false, - }, - || false, - || std::future::ready(Ok(changed.clone())), - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity); - assert_eq!(seen, Some(changed)); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); - let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); - - let reason = wait_for_next_scanner_cycle_with_activity( - &ctx, - Duration::from_secs(120), - Some(Duration::from_secs(60)), - &mut seen, - ScannerCycleObservedGenerations { - dirty_usage: None, - runtime_config: crate::runtime_config::scanner_runtime_config_generation(), - maintenance: crate::scanner_io::scanner_maintenance_generation(), - defer_cluster_activity: true, - }, - || false, - || std::future::ready(Ok(changed.clone())), - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::Timer); - assert_eq!(seen, Some(changed)); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); - let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 4))]); - - let reason = wait_for_next_scanner_cycle_with_activity( - &ctx, - Duration::from_secs(120), - Some(Duration::from_secs(60)), - &mut seen, - ScannerCycleObservedGenerations { - dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), - runtime_config: crate::runtime_config::scanner_runtime_config_generation(), - maintenance: crate::scanner_io::scanner_maintenance_generation(), - defer_cluster_activity: false, - }, - || false, - || std::future::ready(Ok(changed.clone())), - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::ClusterMaintenance); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); - - let reason = wait_for_next_scanner_cycle_with_activity( - &ctx, - Duration::from_secs(120), - Some(Duration::from_secs(60)), - &mut seen, - ScannerCycleObservedGenerations { - dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), - runtime_config: crate::runtime_config::scanner_runtime_config_generation(), - maintenance: crate::scanner_io::scanner_maintenance_generation(), - defer_cluster_activity: false, - }, - || false, - || std::future::ready(Err("node-2 is unreachable".to_string())), - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::ClusterActivityUnavailable); - assert!(seen.is_none()); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let expected = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); - let mut seen = Some(expected.clone()); - - let reason = wait_for_next_scanner_cycle_with_activity( - &ctx, - Duration::from_secs(120), - Some(Duration::from_secs(60)), - &mut seen, - ScannerCycleObservedGenerations { - dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), - runtime_config: crate::runtime_config::scanner_runtime_config_generation(), - maintenance: crate::scanner_io::scanner_maintenance_generation(), - defer_cluster_activity: false, - }, - || false, - || std::future::ready(Ok(expected.clone())), - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::Timer); - assert_eq!(seen, Some(expected)); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn scanner_activity_probe_wait_is_cancellation_aware() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let cancel = ctx.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(61)).await; - cancel.cancel(); - }); - let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); - - let reason = wait_for_next_scanner_cycle_with_activity( - &ctx, - Duration::from_secs(120), - Some(Duration::from_secs(60)), - &mut seen, - ScannerCycleObservedGenerations { - dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), - runtime_config: crate::runtime_config::scanner_runtime_config_generation(), - maintenance: crate::scanner_io::scanner_maintenance_generation(), - defer_cluster_activity: false, - }, - || false, - std::future::pending::>, - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::Cancelled); - } - - #[tokio::test(start_paused = true)] - #[serial] - async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() { - crate::scanner_io::clear_dirty_usage_buckets_for_tests(); - let ctx = CancellationToken::new(); - let lock_lost = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let lose_lock = Arc::clone(&lock_lost); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(61)).await; - lose_lock.store(true, std::sync::atomic::Ordering::Release); - }); - let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); - - let reason = wait_for_next_scanner_cycle_with_activity( - &ctx, - Duration::from_secs(120), - Some(Duration::from_secs(60)), - &mut seen, - ScannerCycleObservedGenerations { - dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), - runtime_config: crate::runtime_config::scanner_runtime_config_generation(), - maintenance: crate::scanner_io::scanner_maintenance_generation(), - defer_cluster_activity: false, - }, - || lock_lost.load(std::sync::atomic::Ordering::Acquire), - std::future::pending::>, - ) - .await; - - assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost); - } - - #[test] - #[serial] - fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() { - with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { - let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle()); - assert_eq!(mode, HealScanMode::Deep); - }); - } - - #[test] - #[serial] - fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() { - with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { - let recent = Utc::now() - chrono::Duration::minutes(30); - let old = Utc::now() - chrono::Duration::hours(2); - - assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent), bitrot_scan_cycle()), HealScanMode::Normal); - assert_eq!(get_cycle_scan_mode(2048, 0, Some(old), bitrot_scan_cycle()), HealScanMode::Deep); - }); - } - - #[test] - #[serial] - fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() { - with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || { - assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal); - }); - } - - #[test] - #[serial] - fn test_background_heal_info_for_scan_start_marks_deep_active() { - let now = Utc::now(); - let info = - background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now, bitrot_scan_cycle()) - .expect("deep scan should update background heal info"); - - assert_eq!(info.current_scan_mode, HealScanMode::Deep); - assert_eq!(info.bitrot_start_cycle, 7); - assert_eq!(info.bitrot_start_time, Some(now)); - } - - #[test] - #[serial] - fn test_background_heal_info_for_scan_start_keeps_deep_window_start() { - with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || { - let started_at = Utc::now(); - let info = BackgroundHealInfo { - bitrot_start_time: Some(started_at), - bitrot_start_cycle: 7, - current_scan_mode: HealScanMode::Normal, - }; - - let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now(), bitrot_scan_cycle()) - .expect("deep scan should mark active status"); - - assert_eq!(info.current_scan_mode, HealScanMode::Deep); - assert_eq!(info.bitrot_start_cycle, 7); - assert_eq!(info.bitrot_start_time, Some(started_at)); - }); - } - - #[test] - fn test_background_heal_info_for_scan_complete_marks_deep_idle() { - let started_at = Utc::now(); - let info = BackgroundHealInfo { - bitrot_start_time: Some(started_at), - bitrot_start_cycle: 7, - current_scan_mode: HealScanMode::Deep, - }; - - let info = background_heal_info_for_scan_complete(info, HealScanMode::Deep) - .expect("completed deep scan should update background heal info"); - - assert_eq!(info.current_scan_mode, HealScanMode::Normal); - assert_eq!(info.bitrot_start_cycle, 7); - assert_eq!(info.bitrot_start_time, Some(started_at)); - } - - #[test] - fn test_background_heal_info_for_scan_complete_leaves_normal_scan_unchanged() { - let info = BackgroundHealInfo { - bitrot_start_time: Some(Utc::now()), - bitrot_start_cycle: 7, - current_scan_mode: HealScanMode::Normal, - }; - - assert!(background_heal_info_for_scan_complete(info, HealScanMode::Normal).is_none()); - } - - #[test] - fn test_background_heal_info_for_failed_scan_preserves_deep_mode() { - let info = BackgroundHealInfo { - bitrot_start_time: Some(Utc::now()), - bitrot_start_cycle: 7, - current_scan_mode: HealScanMode::Deep, - }; - - assert!(background_heal_info_for_scan_result(info, HealScanMode::Deep, false).is_none()); - } - - #[test] - fn test_retain_recent_cycle_completions_keeps_last_entries() { - let base = Utc::now(); - let keep = data_usage_update_dir_cycles() as usize; - let mut completed: Vec<_> = (0..keep + 2).map(|i| base + chrono::Duration::seconds(i as i64)).collect(); - - retain_recent_cycle_completions(&mut completed); - - assert_eq!(completed.len(), keep); - assert_eq!(completed.first().copied(), Some(base + chrono::Duration::seconds(2))); - assert_eq!(completed.last().copied(), Some(base + chrono::Duration::seconds((keep + 1) as i64))); - } -} +mod tests; diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs new file mode 100644 index 000000000..61428afbf --- /dev/null +++ b/crates/scanner/src/scanner/activity.rs @@ -0,0 +1,773 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// Cycle wake/backoff policy and scanner activity observation (probing, generations, topology digest). +use super::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ScannerCycleWakeReason { + Timer, + DirtyUsage, + ClusterActivity, + ClusterMaintenance, + ClusterActivityUnavailable, + RuntimeConfig, + MaintenanceConfig, + LeaderLockLost, + Cancelled, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ScannerCycleOutcome { + Completed, + CompletedWithPendingMaintenance, + Partial, + Superseded, + Deferred(ScannerCycleDeferReason), + Failed, +} + +pub(crate) fn scanner_cycle_outcome_with_pending_maintenance( + outcome: ScannerCycleOutcome, + pending_maintenance_work: bool, +) -> ScannerCycleOutcome { + if outcome == ScannerCycleOutcome::Completed && pending_maintenance_work { + ScannerCycleOutcome::CompletedWithPendingMaintenance + } else { + outcome + } +} + +pub(super) async fn remote_dirty_usage_acknowledgement_pending( + cycle: u64, + acknowledgement_count: usize, + acknowledgement: F, +) -> bool +where + F: Future>, + E: std::fmt::Display, +{ + match acknowledgement.await { + Ok(dirty_usage_pending) => dirty_usage_pending, + Err(err) => { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + cycle, + acknowledgement_count, + error = %err, + state = "remote_dirty_usage_acknowledgement_pending", + "Scanner cycle left remote dirty usage acknowledgements pending" + ); + true + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ScannerCleanIdleBackoff { + pub(super) interval_multiplier: u32, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct ScannerRetryBackoff { + pub(super) consecutive_cycles: u32, +} + +impl ScannerRetryBackoff { + pub(super) fn record_retryable_cycle(&mut self, retryable: bool) { + if retryable { + self.consecutive_cycles = self.consecutive_cycles.saturating_add(1); + } else { + self.consecutive_cycles = 0; + } + } + + pub(super) fn retry_interval(self, configured_interval: Duration) -> Option { + let exponent = self.consecutive_cycles.checked_sub(1)?.min(31); + let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); + let base_interval = configured_interval + .max(Duration::from_secs(1)) + .min(SCANNER_RETRY_BASE_INTERVAL); + let cap = SCANNER_RETRY_MAX_INTERVAL.max(configured_interval.max(Duration::from_secs(1))); + Some(base_interval.saturating_mul(multiplier).min(cap)) + } +} + +impl Default for ScannerCleanIdleBackoff { + fn default() -> Self { + Self { interval_multiplier: 1 } + } +} + +impl ScannerCleanIdleBackoff { + pub(super) fn reset(&mut self) { + self.interval_multiplier = 1; + } + + pub(super) fn effective_interval(self, base_interval: Duration, max_interval: Duration, enabled: bool) -> Duration { + let base_interval = base_interval.max(Duration::from_secs(1)); + if !enabled { + return base_interval; + } + + let max_interval = max_interval.max(base_interval); + base_interval.saturating_mul(self.interval_multiplier).min(max_interval) + } + + pub(super) fn record_cycle( + &mut self, + base_interval: Duration, + max_interval: Duration, + enabled: bool, + wake_reason: ScannerCycleWakeReason, + outcome: ScannerCycleOutcome, + dirty_work_observed: bool, + ) { + if !enabled + || wake_reason != ScannerCycleWakeReason::Timer + || outcome != ScannerCycleOutcome::Completed + || dirty_work_observed + { + self.reset(); + return; + } + + let max_interval = max_interval.max(base_interval.max(Duration::from_secs(1))); + if self.effective_interval(base_interval, max_interval, true) < max_interval { + self.interval_multiplier = self.interval_multiplier.saturating_mul(CLEAN_IDLE_BACKOFF_FACTOR); + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct ScannerMaintenanceInspectionRetry { + pub(super) consecutive_failures: u32, + pub(super) retry_at: Option, +} + +impl ScannerMaintenanceInspectionRetry { + pub(super) fn from_features(features: ScannerMaintenanceFeatures, now: Instant) -> Self { + let mut retry = Self::default(); + retry.record_inspection(features, now); + retry + } + + pub(super) fn reset(&mut self) { + self.consecutive_failures = 0; + self.retry_at = None; + } + + pub(super) fn retry_interval(self) -> Option { + if self.consecutive_failures == 0 { + return None; + } + + let exponent = self.consecutive_failures.saturating_sub(1).min(31); + let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); + Some( + MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL + .saturating_mul(multiplier) + .min(MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL), + ) + } + + pub(super) fn record_inspection(&mut self, features: ScannerMaintenanceFeatures, now: Instant) { + if !features.inspection_failed { + self.reset(); + return; + } + + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + self.retry_at = self.retry_interval().map(|interval| now + interval); + } + + pub(super) fn retry_due( + self, + features: ScannerMaintenanceFeatures, + wake_reason: ScannerCycleWakeReason, + now: Instant, + ) -> bool { + features.inspection_failed + && wake_reason == ScannerCycleWakeReason::Timer + && self.retry_at.is_some_and(|retry_at| now >= retry_at) + } +} + +pub(super) fn scanner_cycle_observed_dirty_work( + pending_before_wait: bool, + generation_before_wait: u64, + generation_after_cycle: u64, +) -> bool { + pending_before_wait || generation_before_wait != generation_after_cycle +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ScannerCycleWaitPlan { + pub(super) effective_interval: Duration, + pub(super) clean_idle_max_interval: Duration, + pub(super) delay: Duration, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ScannerCycleObservedGenerations { + pub(super) dirty_usage: Option, + pub(super) runtime_config: u64, + pub(super) maintenance: u64, + pub(super) defer_cluster_activity: bool, +} + +pub(super) const LOCAL_SCANNER_ACTIVITY_NODE: &str = ""; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ScannerNodeActivity { + pub(super) instance_id: String, + pub(super) namespace_generation: u64, + pub(super) maintenance_generation: u64, + pub(super) protocol_version: u32, + pub(super) topology_digest: [u8; 32], + pub(super) data_movement_active: bool, + pub(super) dirty_usage_generation: u64, + pub(super) dirty_usage_pending: bool, +} + +pub(crate) type ScannerActivitySnapshot = BTreeMap; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ScannerDirtyUsageAcknowledgement { + pub(crate) host: String, + pub(crate) instance_id: String, + pub(crate) generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ScannerActivityObservation { + NotRequired, + Unchanged, + Changed, + MaintenanceChanged, + Unverified, +} + +pub(super) fn scanner_cycle_wait_plan( + runtime_config: &ScannerRuntimeConfig, + clean_idle_backoff: ScannerCleanIdleBackoff, + clean_idle_backoff_enabled: bool, + jitter: impl FnOnce(Duration) -> Duration, +) -> ScannerCycleWaitPlan { + let clean_idle_max_interval = scanner_clean_idle_max_interval(runtime_config.cycle_interval, runtime_config); + let effective_interval = + clean_idle_backoff.effective_interval(runtime_config.cycle_interval, clean_idle_max_interval, clean_idle_backoff_enabled); + let delay = cap_clean_idle_cycle_delay(jitter(effective_interval), clean_idle_max_interval, clean_idle_backoff_enabled); + + ScannerCycleWaitPlan { + effective_interval, + clean_idle_max_interval, + delay, + } +} + +pub(super) fn record_scanner_cycle_result( + clean_idle_backoff: &mut ScannerCleanIdleBackoff, + runtime_config: &ScannerRuntimeConfig, + clean_idle_backoff_enabled: bool, + wake_reason: ScannerCycleWakeReason, + outcome: ScannerCycleOutcome, + dirty_work_observed: bool, +) { + clean_idle_backoff.record_cycle( + runtime_config.cycle_interval, + scanner_clean_idle_max_interval(runtime_config.cycle_interval, runtime_config), + clean_idle_backoff_enabled, + wake_reason, + outcome, + dirty_work_observed, + ); +} + +pub(super) fn scanner_clean_idle_backoff_configured(runtime_config: &ScannerRuntimeConfig) -> bool { + let bitrot_cycle_allows_backoff = + runtime_config.bitrot_cycle.is_none() || runtime_config.bitrot_cycle_source == ScannerRuntimeConfigSource::Default; + runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default && bitrot_cycle_allows_backoff +} + +pub(super) fn scanner_clean_idle_max_interval(base_interval: Duration, runtime_config: &ScannerRuntimeConfig) -> Duration { + let policy_max = CLEAN_IDLE_MAX_INTERVAL.max(base_interval); + let Some(bitrot_cycle) = runtime_config.bitrot_cycle else { + return policy_max; + }; + if runtime_config.bitrot_cycle_source != ScannerRuntimeConfigSource::Default { + return policy_max; + } + + let selection_window = heal_object_select_prob(); + if selection_window == 0 { + return policy_max; + } + + bitrot_cycle + .checked_div(selection_window) + .unwrap_or(base_interval) + .max(base_interval) + .min(policy_max) +} + +pub(super) fn scanner_clean_idle_backoff_enabled( + topology_supported: bool, + cluster_activity_ready: bool, + features: ScannerMaintenanceFeatures, + runtime_config: &ScannerRuntimeConfig, +) -> bool { + topology_supported + && cluster_activity_ready + && !features.needs_regular_cycle() + && scanner_clean_idle_backoff_configured(runtime_config) +} + +pub(super) fn scanner_activity_probe_required( + topology_supported: bool, + backoff_blocked: bool, + features: ScannerMaintenanceFeatures, + runtime_config: &ScannerRuntimeConfig, +) -> bool { + topology_supported + && !backoff_blocked + && !features.needs_regular_cycle() + && scanner_clean_idle_backoff_configured(runtime_config) +} + +pub(super) fn scanner_activity_observed_work(observation: ScannerActivityObservation) -> bool { + matches!( + observation, + ScannerActivityObservation::Changed + | ScannerActivityObservation::MaintenanceChanged + | ScannerActivityObservation::Unverified + ) +} + +pub(super) fn scanner_activity_backoff_blocked_after_wake(currently_blocked: bool, wake_reason: ScannerCycleWakeReason) -> bool { + match wake_reason { + ScannerCycleWakeReason::ClusterMaintenance => true, + ScannerCycleWakeReason::MaintenanceConfig => false, + _ => currently_blocked, + } +} + +pub(super) async fn wait_for_next_scanner_cycle( + ctx: &CancellationToken, + delay: Duration, + dirty_usage_generation_seen: Option, + runtime_config_generation: u64, + maintenance_generation: u64, + is_lock_lost: F, +) -> ScannerCycleWakeReason +where + F: Fn() -> bool, +{ + let sleep = tokio::time::sleep(delay); + tokio::pin!(sleep); + let lock_poll = tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL); + tokio::pin!(lock_poll); + + loop { + if is_lock_lost() { + return ScannerCycleWakeReason::LeaderLockLost; + } + if scanner_runtime_config_generation() != runtime_config_generation { + return ScannerCycleWakeReason::RuntimeConfig; + } + if scanner_maintenance_generation() != maintenance_generation { + return ScannerCycleWakeReason::MaintenanceConfig; + } + if dirty_usage_generation_seen.is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) { + return ScannerCycleWakeReason::DirtyUsage; + } + + tokio::select! { + _ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled, + _ = &mut sleep => return ScannerCycleWakeReason::Timer, + _ = &mut lock_poll => { + if is_lock_lost() { + return ScannerCycleWakeReason::LeaderLockLost; + } + lock_poll.as_mut().reset(Instant::now() + SCANNER_LEADER_LOCK_POLL_INTERVAL); + } + _ = dirty_usage_bucket_notified() => { + if scanner_runtime_config_generation() != runtime_config_generation { + return ScannerCycleWakeReason::RuntimeConfig; + } + if scanner_maintenance_generation() != maintenance_generation { + return ScannerCycleWakeReason::MaintenanceConfig; + } + if dirty_usage_generation_seen + .is_some_and(|seen| dirty_usage_buckets_pending() && dirty_usage_generation() != seen) + { + return ScannerCycleWakeReason::DirtyUsage; + } + } + _ = scanner_runtime_config_changed() => { + if scanner_runtime_config_generation() != runtime_config_generation { + return ScannerCycleWakeReason::RuntimeConfig; + } + } + _ = scanner_maintenance_changed() => { + if scanner_maintenance_generation() != maintenance_generation { + return ScannerCycleWakeReason::MaintenanceConfig; + } + } + } + } +} + +pub(super) async fn wait_for_next_scanner_cycle_with_activity( + ctx: &CancellationToken, + delay: Duration, + activity_poll_interval: Option, + activity_seen: &mut Option, + generations: ScannerCycleObservedGenerations, + is_lock_lost: F, + mut probe_activity: Probe, +) -> ScannerCycleWakeReason +where + F: Fn() -> bool, + Probe: FnMut() -> ProbeFuture, + ProbeFuture: Future>, +{ + let deadline = Instant::now() + delay; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return ScannerCycleWakeReason::Timer; + } + let wait_slice = activity_poll_interval + .map(|interval| interval.max(Duration::from_secs(1)).min(remaining)) + .unwrap_or(remaining); + let wake_reason = wait_for_next_scanner_cycle( + ctx, + wait_slice, + generations.dirty_usage, + generations.runtime_config, + generations.maintenance, + &is_lock_lost, + ) + .await; + if wake_reason != ScannerCycleWakeReason::Timer || Instant::now() >= deadline { + return wake_reason; + } + + let Some(_) = activity_poll_interval else { + return ScannerCycleWakeReason::Timer; + }; + if is_lock_lost() { + return ScannerCycleWakeReason::LeaderLockLost; + } + + let probe = probe_activity(); + tokio::pin!(probe); + let lock_lost = async { + loop { + tokio::time::sleep(SCANNER_LEADER_LOCK_POLL_INTERVAL).await; + if is_lock_lost() { + break; + } + } + }; + tokio::pin!(lock_lost); + let probe_result = tokio::select! { + result = &mut probe => result, + _ = ctx.cancelled() => return ScannerCycleWakeReason::Cancelled, + _ = &mut lock_lost => return ScannerCycleWakeReason::LeaderLockLost, + }; + + let had_baseline = activity_seen.is_some(); + let (observation, probe_error) = apply_scanner_activity_probe_result(activity_seen, probe_result); + if let Some(err) = probe_error { + log_scanner_activity_probe_error(had_baseline, &err); + } + match observation { + ScannerActivityObservation::Unchanged | ScannerActivityObservation::NotRequired => {} + ScannerActivityObservation::Changed if !generations.defer_cluster_activity => { + return ScannerCycleWakeReason::ClusterActivity; + } + ScannerActivityObservation::Changed => {} + ScannerActivityObservation::MaintenanceChanged => return ScannerCycleWakeReason::ClusterMaintenance, + ScannerActivityObservation::Unverified if !generations.defer_cluster_activity => { + return ScannerCycleWakeReason::ClusterActivityUnavailable; + } + ScannerActivityObservation::Unverified => {} + } + } +} + +pub(super) fn log_scanner_activity_probe_error(had_baseline: bool, err: &str) { + if had_baseline { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cluster_activity_probe_failed", + error = %err, + "Scanner cluster activity probe failed; preserving the base cycle" + ); + } else { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cluster_activity_probe_unavailable", + error = %err, + "Scanner cluster activity probe remains unavailable" + ); + } +} + +pub(super) fn compare_scanner_activity( + previous: &ScannerActivitySnapshot, + current: &ScannerActivitySnapshot, +) -> ScannerActivityObservation { + if previous == current { + return ScannerActivityObservation::Unchanged; + } + + for (host, current_activity) in current { + let Some(previous_activity) = previous.get(host) else { + continue; + }; + if host != LOCAL_SCANNER_ACTIVITY_NODE + && previous_activity.instance_id == current_activity.instance_id + && previous_activity.maintenance_generation != current_activity.maintenance_generation + { + return ScannerActivityObservation::MaintenanceChanged; + } + } + + ScannerActivityObservation::Changed +} + +pub(super) fn apply_scanner_activity_probe_result( + activity_seen: &mut Option, + result: Result, +) -> (ScannerActivityObservation, Option) { + match result { + Ok(current) => { + let observation = match activity_seen.as_ref() { + Some(previous) => compare_scanner_activity(previous, ¤t), + None => ScannerActivityObservation::Unverified, + }; + *activity_seen = Some(current); + (observation, None) + } + Err(err) => { + *activity_seen = None; + (ScannerActivityObservation::Unverified, Some(err)) + } + } +} + +pub(super) async fn observe_scanner_activity( + storeapi: &Arc, + distributed: bool, + activity_seen: &mut Option, +) -> ScannerActivityObservation { + let had_baseline = activity_seen.is_some(); + let (observation, probe_error) = + apply_scanner_activity_probe_result(activity_seen, probe_scanner_activity(storeapi, distributed).await); + if let Some(err) = probe_error { + log_scanner_activity_probe_error(had_baseline, &err); + } + observation +} + +pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { + let mut hasher = Sha256::new(); + hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); + for (host, activity) in snapshot { + let host = host.as_bytes(); + let instance_id = activity.instance_id.as_bytes(); + hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(host); + hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(instance_id); + hasher.update(activity.namespace_generation.to_be_bytes()); + hasher.update(activity.maintenance_generation.to_be_bytes()); + hasher.update(activity.protocol_version.to_be_bytes()); + hasher.update(activity.topology_digest); + hasher.update([u8::from(activity.data_movement_active)]); + hasher.update(activity.dirty_usage_generation.to_be_bytes()); + hasher.update([u8::from(activity.dirty_usage_pending)]); + } + hasher.finalize().into() +} + +pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool { + snapshot.values().all(|activity| !activity.data_movement_active) +} + +pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySnapshot) -> Vec { + snapshot + .iter() + .filter(|(host, activity)| host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE && activity.dirty_usage_pending) + .map(|(host, activity)| ScannerDirtyUsageAcknowledgement { + host: host.clone(), + instance_id: activity.instance_id.clone(), + generation: activity.dirty_usage_generation, + }) + .collect() +} + +pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] { + let endpoint_pools = storeapi.endpoints(); + let mut hasher = Sha256::new(); + hasher.update(u64::try_from(endpoint_pools.0.len()).unwrap_or(u64::MAX).to_be_bytes()); + for (pool_index, pool) in endpoint_pools.0.iter().enumerate() { + hasher.update(u64::try_from(pool_index).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(u64::try_from(pool.set_count).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(u64::try_from(pool.drives_per_set).unwrap_or(u64::MAX).to_be_bytes()); + let mut endpoints = pool.endpoints.as_ref().iter().collect::>(); + endpoints.sort_unstable_by(|left, right| { + (left.pool_idx, left.set_idx, left.disk_idx, left.url.as_str()).cmp(&( + right.pool_idx, + right.set_idx, + right.disk_idx, + right.url.as_str(), + )) + }); + hasher.update(u64::try_from(endpoints.len()).unwrap_or(u64::MAX).to_be_bytes()); + for endpoint in endpoints { + hasher.update(endpoint.pool_idx.to_be_bytes()); + hasher.update(endpoint.set_idx.to_be_bytes()); + hasher.update(endpoint.disk_idx.to_be_bytes()); + let url = endpoint.url.as_str().as_bytes(); + hasher.update(u64::try_from(url.len()).unwrap_or(u64::MAX).to_be_bytes()); + hasher.update(url); + } + } + hasher.finalize().into() +} + +pub(super) fn record_scanner_activity_instance( + instance_hosts: &mut BTreeMap, + host: &str, + instance_id: &str, +) -> Result<(), String> { + if let Some(existing_host) = instance_hosts.insert(instance_id.to_string(), host.to_string()) { + return Err(format!( + "scanner activity peers {existing_host} and {host} report the same process instance" + )); + } + Ok(()) +} + +pub(crate) async fn probe_scanner_activity(storeapi: &ECStore, distributed: bool) -> Result { + let topology_digest = scanner_topology_digest(storeapi); + let data_movement_active = storeapi.scanner_data_movement_active().await; + let namespace_generation = storeapi.scanner_namespace_mutation_generation(); + let maintenance_generation = scanner_maintenance_generation(); + let dirty_usage = scanner_dirty_usage_state(); + if namespace_generation == u64::MAX || maintenance_generation == u64::MAX || dirty_usage.generation == u64::MAX { + return Err("local scanner activity generation is exhausted".to_string()); + } + let local_instance_id = crate::scanner_io::scanner_activity_epoch().to_string(); + let mut instance_hosts = BTreeMap::from([(local_instance_id.clone(), LOCAL_SCANNER_ACTIVITY_NODE.to_string())]); + let mut snapshot = ScannerActivitySnapshot::from([( + LOCAL_SCANNER_ACTIVITY_NODE.to_string(), + ScannerNodeActivity { + instance_id: local_instance_id, + namespace_generation, + maintenance_generation, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest, + data_movement_active, + dirty_usage_generation: dirty_usage.generation, + dirty_usage_pending: dirty_usage.pending, + }, + )]); + if !distributed { + return Ok(snapshot); + } + + let notification_system = storeapi + .notification_system() + .ok_or_else(|| "notification system is not initialized".to_string())?; + let peers = notification_system + .scanner_activity_snapshots() + .await + .map_err(|err| err.to_string())?; + for (host, activity) in peers { + if activity.namespace_generation == u64::MAX || activity.maintenance_generation == u64::MAX { + return Err(format!("scanner activity peer {host} exhausted its activity generation")); + } + let (peer_topology_digest, peer_data_movement_active, peer_dirty_usage_generation, peer_dirty_usage_pending) = + match activity.protocol_version { + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION => { + return Err(format!("scanner activity peer {host} cannot verify data movement publication fencing")); + } + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION => { + return Err(format!( + "scanner activity peer {host} cannot safely share scanner cache locks with protocol {}", + SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION + )); + } + SCANNER_ACTIVITY_PROTOCOL_VERSION => ( + activity + .topology_digest + .ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?, + activity + .data_movement_active + .ok_or_else(|| format!("scanner activity peer {host} omitted its data movement state"))?, + activity + .dirty_usage_generation + .ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage generation"))?, + activity + .dirty_usage_pending + .ok_or_else(|| format!("scanner activity peer {host} omitted its dirty usage state"))?, + ), + version => { + return Err(format!( + "scanner activity peer {host} uses protocol {version}, expected {}", + SCANNER_ACTIVITY_PROTOCOL_VERSION + )); + } + }; + if peer_dirty_usage_generation == u64::MAX { + return Err(format!("scanner activity peer {host} exhausted its dirty usage generation")); + } + if peer_topology_digest != topology_digest { + return Err(format!("scanner activity peer {host} has a different storage topology")); + } + record_scanner_activity_instance(&mut instance_hosts, &host, &activity.instance_id)?; + if snapshot + .insert( + host.clone(), + ScannerNodeActivity { + instance_id: activity.instance_id, + namespace_generation: activity.namespace_generation, + maintenance_generation: activity.maintenance_generation, + protocol_version: activity.protocol_version, + topology_digest: peer_topology_digest, + data_movement_active: peer_data_movement_active, + dirty_usage_generation: peer_dirty_usage_generation, + dirty_usage_pending: peer_dirty_usage_pending, + }, + ) + .is_some() + { + return Err(format!("duplicate scanner activity peer: {host}")); + } + } + Ok(snapshot) +} diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs new file mode 100644 index 000000000..6f3af4b81 --- /dev/null +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -0,0 +1,498 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// Scanner cycle-state codec, persisted usage floors, and cycle-state persistence. +use super::*; + +#[derive(Debug, thiserror::Error)] +pub(super) enum ScannerCycleStateError { + #[error("failed to encode scanner cycle state: {0}")] + Encode(#[from] rmp_serde::encode::Error), + #[error("failed to decode scanner cycle state: {0}")] + Decode(#[from] rmp_serde::decode::Error), + #[error("{0}")] + InvalidData(&'static str), +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct PersistedUsageFloor { + pub(super) next_cycle: u64, + pub(super) leader_epoch: u64, +} + +pub(super) fn encode_scanner_cycle_state( + cycle_info: &CurrentCycle, + leader_epoch: u64, +) -> Result, ScannerCycleStateError> { + if cycle_info.next == u64::MAX { + return Err(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted")); + } + let cycle_info_buf = rmp_serde::to_vec(cycle_info)?; + let mut buf = Vec::with_capacity(cycle_info_buf.len() + SCANNER_CYCLE_STATE_HEADER_LEN); + buf.extend_from_slice(&cycle_info.next.to_le_bytes()); + buf.extend_from_slice(SCANNER_CYCLE_STATE_MAGIC); + buf.extend_from_slice(&leader_epoch.to_le_bytes()); + buf.extend_from_slice(&cycle_info_buf); + Ok(buf) +} + +pub(super) fn decode_scanner_cycle_state(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> { + if buf.len() < 8 { + return Err(ScannerCycleStateError::InvalidData("scanner cycle state is truncated")); + } + + let persisted_next = u64::from_le_bytes( + buf[0..8] + .try_into() + .map_err(|_| ScannerCycleStateError::InvalidData("scanner cycle counter is truncated"))?, + ); + if persisted_next == u64::MAX { + return Err(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted")); + } + if buf.len() == 8 { + return Ok(( + CurrentCycle { + next: persisted_next, + ..Default::default() + }, + 0, + )); + } + + let (leader_epoch, payload) = if buf.len() >= 16 && &buf[8..16] == SCANNER_CYCLE_STATE_MAGIC { + if buf.len() < SCANNER_CYCLE_STATE_HEADER_LEN { + return Err(ScannerCycleStateError::InvalidData("scanner cycle fencing header is truncated")); + } + let epoch = u64::from_le_bytes( + buf[16..24] + .try_into() + .map_err(|_| ScannerCycleStateError::InvalidData("scanner leader epoch is truncated"))?, + ); + if epoch == 0 { + return Err(ScannerCycleStateError::InvalidData("scanner leader epoch is zero")); + } + (epoch, &buf[SCANNER_CYCLE_STATE_HEADER_LEN..]) + } else { + (0, &buf[8..]) + }; + + let cycle_info = rmp_serde::from_slice::(payload)?; + if cycle_info.next != persisted_next { + return Err(ScannerCycleStateError::InvalidData("scanner cycle counter disagrees with encoded state")); + } + Ok((cycle_info, leader_epoch)) +} + +pub(crate) fn decode_persisted_scanner_cycle_fence(buf: &[u8]) -> Result<(u64, u64), ScannerError> { + decode_scanner_cycle_state(buf) + .map(|(cycle, leader_epoch)| (cycle.next, leader_epoch)) + .map_err(|err| ScannerError::Other(format!("persisted scanner cycle state is invalid: {err}"))) +} + +#[cfg(test)] +pub(crate) fn encode_scanner_cycle_fence_for_test(next_cycle: u64, leader_epoch: u64) -> Vec { + encode_scanner_cycle_state( + &CurrentCycle { + next: next_cycle, + ..Default::default() + }, + leader_epoch, + ) + .expect("test scanner cycle fence should encode") +} + +pub(crate) async fn current_scanner_leader_epoch() -> Result { + let store = crate::resolve_scanner_object_store_handle() + .ok_or_else(|| ScannerError::Other("scanner object layer is unavailable".to_string()))?; + match read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await { + Ok(buf) => { + let (_, leader_epoch) = decode_persisted_scanner_cycle_fence(&buf)?; + if leader_epoch == 0 { + return Err(ScannerError::Other("persisted scanner cycle state has no leader epoch".to_string())); + } + Ok(leader_epoch) + } + Err(err) => Err(ScannerError::Other(format!("failed to read persisted scanner leader epoch: {err}"))), + } +} + +pub(super) fn decode_scanner_cycle_state_for_startup(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> { + if buf.is_empty() { + Ok((CurrentCycle::default(), 0)) + } else { + decode_scanner_cycle_state(buf) + } +} + +pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(), ScannerCycleStateError> { + let next = cycle_info + .next + .checked_add(1) + .filter(|next| *next < u64::MAX) + .ok_or(ScannerCycleStateError::InvalidData("scanner cycle counter is exhausted"))?; + cycle_info.next = next; + Ok(()) +} + +pub(super) async fn persisted_usage_floor(storeapi: Arc) -> Result { + let mut floor = PersistedUsageFloor::default(); + let update_floor = |floor: &mut PersistedUsageFloor, usage: DataUsageInfo, path: &str| -> Result<(), ScannerError> { + floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); + if let Some(completed_cycle) = usage.scanner_cycle { + let next_cycle = completed_cycle + .checked_add(1) + .filter(|next| *next < u64::MAX) + .ok_or_else(|| ScannerError::Other(format!("persisted scanner usage cycle is exhausted in {path}")))?; + floor.next_cycle = floor.next_cycle.max(next_cycle); + } + Ok(()) + }; + for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] { + let backup_path = format!("{primary_path}.bkp"); + let mut pair_found = false; + for path in [primary_path, backup_path.as_str()] { + let data = match read_config(storeapi.clone(), path).await { + Ok(data) => { + pair_found = true; + data + } + Err(EcstoreError::ConfigNotFound) => continue, + Err(err) => { + return Err(ScannerError::Other(format!( + "failed to read scanner usage epoch floor from {path}: {err}" + ))); + } + }; + let usage = serde_json::from_slice::(&data) + .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage floor from {path}: {err}")))?; + update_floor(&mut floor, usage, path)?; + } + if pair_found { + break; + } + } + Ok(floor) +} + +pub(super) fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_epoch: &mut u64, floor: PersistedUsageFloor) { + cycle_info.next = cycle_info.next.max(floor.next_cycle); + *leader_epoch = (*leader_epoch).max(floor.leader_epoch); +} + +pub(super) async fn persist_scanner_cycle_state( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, +) -> bool { + let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) { + Ok(buf) => buf, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "encode_failed", + error = %e, + "Scanner state encoding failed" + ); + return false; + } + }; + + for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { + if ctx.is_cancelled() { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "cancelled_before_save", + retry, + "Scanner state persistence cancelled by the leader fence" + ); + return false; + } + + #[cfg(test)] + notify_scanner_cycle_state_persist_test_hook(leader_epoch); + match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) + .await + { + Ok(object_info) => { + let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "missing_revision", + "Scanner state save returned no ETag" + ); + return false; + }; + *revision = DataUsageCacheRevision::Etag(etag); + if ctx.is_cancelled() { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "cancelled_after_save", + retry, + "Scanner state save completed after the leader fence was cancelled" + ); + return false; + } + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "saved", + "Scanner state saved" + ); + return true; + } + Err(EcstoreError::PreconditionFailed) => { + let (persisted, persisted_revision) = + match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await { + Ok(result) => result, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_reload_failed", + error = %e, + "Scanner state conflict reconciliation failed" + ); + return false; + } + }; + *revision = persisted_revision; + if ctx.is_cancelled() { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "cancelled_after_conflict", + retry, + "Scanner state conflict reconciliation cancelled by the leader fence" + ); + return false; + } + + if let Some(persisted) = persisted { + if persisted.len() < 8 { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_state_invalid", + length = persisted.len(), + "Scanner state conflict winner is truncated" + ); + return false; + } + + let (persisted_cycle, persisted_epoch) = match decode_scanner_cycle_state(&persisted) { + Ok(state) => state, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_state_decode_failed", + error = %e, + "Scanner state conflict winner could not be decoded" + ); + return false; + } + }; + if persisted_epoch != leader_epoch { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_epoch_fenced", + expected_epoch = leader_epoch, + persisted_epoch, + "Scanner state save rejected by a newer leadership epoch" + ); + return false; + } + + if persisted_cycle.next >= cycle_info.next { + *cycle_info = persisted_cycle; + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_reconciled", + retry, + "Scanner state adopted the current persisted cycle" + ); + return true; + } + } + + if retry < SCANNER_PERSIST_CAS_RETRIES { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_retry", + retry = retry + 1, + "Scanner state CAS conflict will be retried" + ); + continue; + } + + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "conflict_retries_exhausted", + retries = SCANNER_PERSIST_CAS_RETRIES, + "Scanner state CAS conflict retries exhausted" + ); + return false; + } + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "failed", + error = %e, + "Scanner state persistence failed" + ); + return false; + } + } + } + + false +} + +pub(super) async fn finalize_partial_scan_cycle( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + cycle_metrics_guard: &mut ScannerCycleMetricsGuard, +) -> bool { + // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter + // must still advance (and persist) because per-bucket next_cycle is stamped from + // it and compacted folders are only rescanned when their hash matches + // next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle + // expiry and usage refresh on every folder outside the stuck window. + if let Err(err) = advance_scanner_cycle(cycle_info) { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cycle_counter_exhausted", + error = %err, + "Scanner partial cycle could not advance" + ); + mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await; + return false; + } + cycle_info.current = 0; + global_metrics().clear_current_scan_mode(); + let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; + cycle_metrics_guard.finish(cycle_info.clone()).await; + persisted +} + +pub(super) async fn persist_required_scanner_cycle_floor( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + required_cycle: u64, + cycle_metrics_guard: &mut ScannerCycleMetricsGuard, +) -> bool { + if required_cycle <= cycle_info.current || required_cycle == u64::MAX { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + current_cycle = cycle_info.current, + required_cycle, + state = "invalid_cache_cycle_floor", + "Scanner cache cycle floor is invalid" + ); + mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await; + return false; + } + + cycle_info.next = cycle_info.next.max(required_cycle); + cycle_info.current = 0; + global_metrics().clear_current_scan_mode(); + let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; + cycle_metrics_guard.finish(cycle_info.clone()).await; + persisted +} + +pub(super) async fn await_scanner_cycle_with_lock_fence( + cycle_ctx: &CancellationToken, + cycle: Cycle, + lock_lost: LockLost, +) -> Option +where + Cycle: Future, + LockLost: Future, +{ + tokio::pin!(cycle); + tokio::pin!(lock_lost); + tokio::select! { + biased; + _ = &mut lock_lost => { + cycle_ctx.cancel(); + tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await.ok() + } + output = &mut cycle => Some(output), + } +} diff --git a/crates/scanner/src/scanner/heal_info.rs b/crates/scanner/src/scanner/heal_info.rs new file mode 100644 index 000000000..18d0511f4 --- /dev/null +++ b/crates/scanner/src/scanner/heal_info.rs @@ -0,0 +1,109 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// The background-heal info object persisted between scanner cycles. +use super::*; + +/// Background healing information +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackgroundHealInfo { + /// Bitrot scan start time + pub bitrot_start_time: Option>, + /// Bitrot scan start cycle + pub bitrot_start_cycle: u64, + /// Current scan mode + pub current_scan_mode: HealScanMode, +} + +/// Read background healing information from storage +pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHealInfo { + // Skip for ErasureSD setup + if scanner_is_erasure_sd().await { + return BackgroundHealInfo::default(); + } + + // Get last healing information + match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await { + Ok(buf) => serde_json::from_slice::(&buf).unwrap_or_else(|e| { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "decode_failed", + error = %e, + "Scanner background heal decode failed" + ); + BackgroundHealInfo::default() + }), + Err(e) => { + // Only log if it's not a ConfigNotFound error + if e != EcstoreError::ConfigNotFound { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "read_failed", + error = %e, + "Scanner background heal read failed" + ); + } + BackgroundHealInfo::default() + } + } +} + +/// Save background healing information to storage +#[instrument(skip(storeapi))] +pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundHealInfo) { + // Skip for ErasureSD setup + if scanner_is_erasure_sd().await { + return; + } + + // Serialize to JSON + let data = match serde_json::to_vec(&info) { + Ok(data) => data, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "encode_failed", + error = %e, + "Scanner background heal encode failed" + ); + return; + } + }; + + // Save configuration + if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "save_failed", + error = %e, + "Scanner background heal save failed" + ); + } +} diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs new file mode 100644 index 000000000..0ac948549 --- /dev/null +++ b/crates/scanner/src/scanner/leadership.rs @@ -0,0 +1,363 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// Leader-lock claiming, usage-epoch fencing, and lock-loss handling. +use super::*; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ScannerLeadershipClaimReconcile { + Durable, + Changed, + Unchanged, +} + +pub(super) async fn reconcile_scanner_leadership_claim( + storeapi: Arc, + attempted: &[u8], + previous_revision: &DataUsageCacheRevision, + claimed_epoch: u64, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + persisted_epoch: &mut u64, +) -> Result { + let (persisted, persisted_revision) = read_config_with_revision(storeapi, DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner leadership claim: {err}")))?; + let revision_changed = &persisted_revision != previous_revision; + *revision = persisted_revision; + + let Some(persisted) = persisted else { + *cycle_info = CurrentCycle::default(); + return Ok(if revision_changed { + ScannerLeadershipClaimReconcile::Changed + } else { + ScannerLeadershipClaimReconcile::Unchanged + }); + }; + if persisted == attempted { + *persisted_epoch = claimed_epoch; + return Ok(ScannerLeadershipClaimReconcile::Durable); + } + + let (current, epoch) = decode_scanner_cycle_state(&persisted) + .map_err(|err| ScannerError::Other(format!("scanner leadership conflict winner is invalid: {err}")))?; + *cycle_info = current; + *persisted_epoch = (*persisted_epoch).max(epoch); + Ok(if revision_changed { + ScannerLeadershipClaimReconcile::Changed + } else { + ScannerLeadershipClaimReconcile::Unchanged + }) +} + +pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result { + serde_json::from_slice(data) + .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}"))) +} + +pub(super) async fn usage_snapshot_for_epoch_fence( + storeapi: Arc, + primary: Option<&[u8]>, +) -> Result { + if let Some(primary) = primary { + return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()); + } + + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let (backup, _) = read_config_with_revision(storeapi.clone(), &backup_path) + .await + .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?; + if let Some(backup) = backup.as_deref() { + return decode_usage_snapshot_for_epoch_fence(backup, &backup_path); + } + + for path in [ + LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), + format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + ] { + let (legacy, _) = read_config_with_revision(storeapi.clone(), &path) + .await + .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; + if let Some(legacy) = legacy.as_deref() { + return decode_usage_snapshot_for_epoch_fence(legacy, &path); + } + } + Ok(DataUsageInfo::default()) +} + +pub(super) async fn fence_scanner_usage_epoch( + ctx: &CancellationToken, + storeapi: Arc, + claimed_epoch: u64, +) -> Result<(), ScannerError> { + for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { + if ctx.is_cancelled() { + return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); + } + + let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?; + let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?; + match usage.scanner_epoch { + Some(epoch) if epoch > claimed_epoch => { + return Err(ScannerError::Other(format!( + "scanner usage epoch fence lost to newer leader: claimed={claimed_epoch}, persisted={epoch}" + ))); + } + Some(epoch) if epoch == claimed_epoch => return Ok(()), + Some(_) | None => {} + } + usage.scanner_epoch = Some(claimed_epoch); + let data = serde_json::to_vec(&usage) + .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; + + let save_result = + save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) + .await; + if save_result + .as_ref() + .ok() + .and_then(|object_info| object_info.etag.as_deref()) + .is_some_and(|etag| !etag.is_empty()) + { + return Ok(()); + } + + let (persisted, persisted_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to reconcile scanner usage epoch fence: {err}")))?; + if let Some(persisted) = persisted { + let persisted = decode_usage_snapshot_for_epoch_fence(&persisted, DATA_USAGE_OBJ_NAME_PATH.as_str())?; + match persisted.scanner_epoch { + Some(epoch) if epoch == claimed_epoch => return Ok(()), + Some(epoch) if epoch > claimed_epoch => { + return Err(ScannerError::Other(format!( + "scanner usage epoch fence lost to newer leader: claimed={claimed_epoch}, persisted={epoch}" + ))); + } + Some(_) | None => {} + } + } + + let precondition_failed = matches!(save_result, Err(EcstoreError::PreconditionFailed)); + if retry < SCANNER_PERSIST_CAS_RETRIES && (precondition_failed || persisted_revision != revision) { + continue; + } + return Err(ScannerError::Other(match save_result { + Ok(_) => "scanner usage epoch fence returned no ETag and could not be confirmed".to_string(), + Err(err) => format!("scanner usage epoch fence save failed: {err}"), + })); + } + + Err(ScannerError::Other("scanner usage epoch fence retries exhausted".to_string())) +} + +pub(super) async fn complete_scanner_leadership_claim( + ctx: &CancellationToken, + storeapi: Arc, + claimed_epoch: u64, +) -> bool { + if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "usage_epoch_fence_failed", + claimed_epoch, + error = %err, + "Scanner leadership usage epoch fencing failed" + ); + return false; + } + !ctx.is_cancelled() +} + +pub(super) async fn claim_scanner_leadership( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + persisted_epoch: &mut u64, +) -> bool { + for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { + if ctx.is_cancelled() { + return false; + } + let Some(claimed_epoch) = persisted_epoch.checked_add(1) else { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_epoch_exhausted", + "Scanner leadership epoch is exhausted" + ); + return false; + }; + let data = match encode_scanner_cycle_state(cycle_info, claimed_epoch) { + Ok(data) => data, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_encode_failed", + error = %err, + "Scanner leadership claim encoding failed" + ); + return false; + } + }; + let previous_revision = revision.clone(); + + let save_result = + save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions()) + .await; + match save_result { + Ok(object_info) => { + if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) { + *revision = DataUsageCacheRevision::Etag(etag); + *persisted_epoch = claimed_epoch; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + } + + match reconcile_scanner_leadership_claim( + storeapi.clone(), + &data, + &previous_revision, + claimed_epoch, + cycle_info, + revision, + persisted_epoch, + ) + .await + { + Ok(ScannerLeadershipClaimReconcile::Durable) => { + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + } + Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue, + Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_missing_revision", + "Scanner leadership claim returned no ETag and could not be confirmed" + ); + return false; + } + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_reconcile_failed", + error = %err, + "Scanner leadership claim read-back failed" + ); + return false; + } + } + } + Err(err) => { + let precondition_failed = matches!(err, EcstoreError::PreconditionFailed); + match reconcile_scanner_leadership_claim( + storeapi.clone(), + &data, + &previous_revision, + claimed_epoch, + cycle_info, + revision, + persisted_epoch, + ) + .await + { + Ok(ScannerLeadershipClaimReconcile::Durable) => { + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + } + Ok(ScannerLeadershipClaimReconcile::Changed) + if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => + { + continue; + } + Ok(ScannerLeadershipClaimReconcile::Unchanged) + if precondition_failed && retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => + { + continue; + } + Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = if precondition_failed { + "leader_claim_conflicts_exhausted" + } else { + "leader_claim_failed" + }, + error = %err, + "Scanner leadership claim failed" + ); + return false; + } + Err(reconcile_err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "leader_claim_reload_failed", + error = %reconcile_err, + save_error = %err, + "Scanner leadership claim reconciliation failed" + ); + return false; + } + } + } + } + } + + false +} + +pub(super) async fn record_scanner_leader_lock_lost(message: &'static str) { + reset_scanner_cycle_schedule(); + record_scanner_leader_lock_state("lost"); + global_metrics() + .record_scanner_leader_liveness("lost", false, "leader lock refresh quorum lost") + .await; + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_LOCK_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + lock_name = "leader.lock", + state = "lost", + reason = message, + "Scanner leader lock lost" + ); +} diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs new file mode 100644 index 000000000..e2c9c1599 --- /dev/null +++ b/crates/scanner/src/scanner/tests.rs @@ -0,0 +1,3920 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::EcstoreResult; +use crate::{ + Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader, + ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader, + init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, +}; +use serial_test::serial; +use std::collections::HashMap; +use std::io::Cursor; +use std::task::Poll; +use temp_env::{with_var, with_var_unset}; +use tokio::io::AsyncReadExt; +use tokio::sync::Mutex; + +const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60; + +async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc) { + init_ecstore_config_for_scanner_tests(); + let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created"); + let mut endpoints = Vec::new(); + for disk_index in 0..4 { + let disk_path = temp_dir.path().join(format!("disk{disk_index}")); + tokio::fs::create_dir_all(&disk_path) + .await + .expect("scanner cycle test disk should be created"); + let mut endpoint = + Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: "scanner-cycle-metrics".to_string(), + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }]); + let instance_ctx = Arc::new(InstanceContext::new()); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("scanner cycle test disks should initialize"); + let store = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address should parse"), + endpoint_pools, + CancellationToken::new(), + instance_ctx, + ) + .await + .expect("scanner cycle test ECStore should initialize"); + init_bucket_metadata_sys_for_scanner_tests(store.clone()).await; + + (temp_dir, store) +} + +fn assert_run_data_scanner_signature(_run: F) +where + F: Fn(CancellationToken, Arc) -> Fut, + Fut: Future>, +{ +} + +#[test] +fn run_data_scanner_keeps_its_two_argument_api() { + assert_run_data_scanner_signature(run_data_scanner); +} + +#[tokio::test] +async fn scanner_cycle_lock_fence_cancels_cycle_context() { + let cycle_ctx = CancellationToken::new(); + let observed_ctx = cycle_ctx.clone(); + let output = await_scanner_cycle_with_lock_fence( + &cycle_ctx, + async move { + observed_ctx.cancelled().await; + observed_ctx.is_cancelled() + }, + std::future::ready(()), + ) + .await; + + assert_eq!(output, Some(true)); + assert!(cycle_ctx.is_cancelled()); +} + +#[tokio::test] +async fn scanner_cycle_lock_fence_preserves_completed_cycle() { + let cycle_ctx = CancellationToken::new(); + let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::ready(7_u8), std::future::pending()).await; + + assert_eq!(output, Some(7)); + assert!(!cycle_ctx.is_cancelled()); +} + +#[tokio::test] +async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() { + let cycle_ctx = CancellationToken::new(); + let output = await_scanner_cycle_with_lock_fence(&cycle_ctx, std::future::pending::<()>(), std::future::ready(())).await; + + assert_eq!(output, None); + assert!(cycle_ctx.is_cancelled()); +} + +struct ScannerDefaultSpeedGuard; + +impl ScannerDefaultSpeedGuard { + fn set(speed: ScannerSpeed) -> Self { + set_scanner_default_speed(speed); + Self + } +} + +impl Drop for ScannerDefaultSpeedGuard { + fn drop(&mut self) { + set_scanner_default_speed(ScannerSpeed::Default); + } +} + +struct ScannerDefaultCycleGuard; + +impl ScannerDefaultCycleGuard { + fn set(secs: u64) -> Self { + set_scanner_default_cycle_secs(Some(secs)); + Self + } +} + +impl Drop for ScannerDefaultCycleGuard { + fn drop(&mut self) { + set_scanner_default_cycle_secs(None); + } +} + +#[derive(Debug, Default)] +struct MemoryConfigStore { + objects: Mutex>>, + revisions: Mutex>, + fail_put_number: Mutex>, + error_after_commit_put_number: Mutex>, + interleaving_puts: Mutex)>>, + cancel_after_interleaving_puts: Mutex>, + cancel_after_successful_puts: Mutex>, + replace_after_successful_puts: Mutex)>>, + put_counts: Mutex>, +} + +fn memory_config_key(bucket: &str, object: &str) -> String { + format!("{bucket}/{object}") +} + +#[async_trait::async_trait] +impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore { + type Error = EcstoreError; + type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + _range: Option, + _h: http::HeaderMap, + _opts: &ObjectOptions, + ) -> EcstoreResult { + let key = memory_config_key(bucket, object); + let data = self + .objects + .lock() + .await + .get(&key) + .cloned() + .ok_or(EcstoreError::FileNotFound)?; + let revision = *self.revisions.lock().await.entry(key).or_insert(1); + + Ok(GetObjectReader { + stream: Box::new(Cursor::new(data)), + object_info: ObjectInfo { + etag: Some(format!("memory-{revision}")), + ..Default::default() + }, + buffered_body: None, + body_source: Default::default(), + }) + } + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut PutObjReader, + opts: &ObjectOptions, + ) -> EcstoreResult { + let mut buf = Vec::new(); + data.stream.read_to_end(&mut buf).await?; + let key = memory_config_key(bucket, object); + let put_count = { + let mut put_counts = self.put_counts.lock().await; + let put_count = put_counts.entry(key.clone()).or_insert(0); + *put_count += 1; + *put_count + }; + + if self.fail_put_number.lock().await.get(&key) == Some(&put_count) { + return Err(EcstoreError::other("injected put failure")); + } + + let interleaving_data = { + let mut interleaving_puts = self.interleaving_puts.lock().await; + if interleaving_puts + .get(&key) + .is_some_and(|(expected_put, _)| *expected_put == put_count) + { + interleaving_puts.remove(&key).map(|(_, data)| data) + } else { + None + } + }; + let cancel_after_interleaving = if interleaving_data.is_some() { + self.cancel_after_interleaving_puts.lock().await.remove(&key) + } else { + None + }; + let replacement = { + let mut replacements = self.replace_after_successful_puts.lock().await; + if replacements + .get(&key) + .is_some_and(|(expected_put, _)| *expected_put == put_count) + { + replacements.remove(&key).map(|(_, replacement)| replacement) + } else { + None + } + }; + let mut objects = self.objects.lock().await; + let mut revisions = self.revisions.lock().await; + if let Some(interleaving_data) = interleaving_data { + let revision = revisions.get(&key).copied().unwrap_or(0) + 1; + objects.insert(key.clone(), interleaving_data); + revisions.insert(key.clone(), revision); + if let Some(cancel) = cancel_after_interleaving { + cancel.cancel(); + } + } + let current_revision = objects.contains_key(&key).then(|| revisions.get(&key).copied().unwrap_or(1)); + if let Some(preconditions) = &opts.http_preconditions { + if preconditions + .if_none_match + .as_deref() + .is_some_and(|condition| !condition.trim().is_empty()) + && current_revision.is_some() + { + return Err(EcstoreError::PreconditionFailed); + } + if let Some(expected) = preconditions + .if_match + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let actual = current_revision.map(|revision| format!("memory-{revision}")); + if actual.as_deref() != Some(expected.trim_matches('"')) { + return Err(EcstoreError::PreconditionFailed); + } + } + } + + let revision = current_revision.unwrap_or(0) + 1; + objects.insert(key.clone(), buf); + revisions.insert(key.clone(), revision); + if let Some(replacement) = replacement { + objects.insert(key.clone(), replacement); + revisions.insert(key.clone(), revision + 1); + } + drop(revisions); + drop(objects); + let cancel_after_success = { + let mut cancellations = self.cancel_after_successful_puts.lock().await; + if cancellations + .get(&key) + .is_some_and(|(expected_put, _)| *expected_put == put_count) + { + cancellations.remove(&key).map(|(_, cancel)| cancel) + } else { + None + } + }; + if let Some(cancel) = cancel_after_success { + cancel.cancel(); + } + if self.error_after_commit_put_number.lock().await.get(&key) == Some(&put_count) { + return Err(EcstoreError::other("injected post-commit put failure")); + } + Ok(ObjectInfo { + etag: Some(format!("memory-{revision}")), + ..Default::default() + }) + } +} + +fn with_unset_scanner_timing_env(f: impl FnOnce()) { + with_var_unset(ENV_SCANNER_SPEED, || { + with_var_unset("MINIO_SCANNER_SPEED", || { + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset("MINIO_SCANNER_CYCLE", || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, f); + }); + }); + }); + }); + }); +} + +#[test] +fn test_randomized_cycle_delay_keeps_configured_start_delay() { + // 120s with ±10% jitter should stay clearly above the historic 30s cap. + let delay = randomized_cycle_delay_for(Duration::from_secs(120)); + assert!(delay > Duration::from_secs(30), "expected delay > 30s, got {delay:?}"); + // Jitter window should stay within configured bounds. + assert!(delay >= Duration::from_secs(108)); + assert!(delay <= Duration::from_secs(132)); +} + +#[test] +fn test_randomized_cycle_delay_bounds_extreme_interval() { + let delay = randomized_cycle_delay_for(Duration::MAX); + + assert!(delay >= MAX_SCANNER_SCHEDULE_DELAY.mul_f64(0.9)); + assert!(delay <= MAX_SCANNER_SCHEDULE_DELAY); +} + +#[test] +fn test_initial_scanner_delay_uses_configured_start_delay() { + let delay = initial_scanner_delay_for(Some(120)); + assert!(delay >= Duration::from_secs(108)); + assert!(delay <= Duration::from_secs(132)); +} + +#[test] +#[serial] +fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() { + with_var(ENV_SCANNER_CYCLE, Some("120"), || { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + let delay = initial_scanner_delay_for(None); + assert!(delay >= Duration::from_secs(108)); + assert!(delay <= Duration::from_secs(132)); + }); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); +} + +#[test] +fn test_initial_scanner_delay_skips_for_cold_usage_cache_with_buckets() { + let delay = initial_scanner_delay_for_startup(Some(120), true, true, false); + assert_eq!(delay, Duration::ZERO); +} + +#[test] +fn test_initial_scanner_delay_keeps_configured_delay_for_warm_usage_cache_no_replication() { + let delay = initial_scanner_delay_for_startup(Some(120), false, true, false); + assert!(delay >= Duration::from_secs(108)); + assert!(delay <= Duration::from_secs(132)); +} + +#[test] +fn test_initial_scanner_delay_skips_for_cold_usage_cache_without_buckets() { + let delay = initial_scanner_delay_for_startup(Some(120), true, false, false); + assert_eq!(delay, Duration::ZERO); +} + +#[test] +fn test_initial_scanner_delay_skips_for_active_replication_warm_cache() { + // Warm cache + active replication rules → skip startup delay so that FAILED-status objects + // from a crash are healed on the first cycle, not after a 27-33 min sleep. + let delay = initial_scanner_delay_for_startup(Some(120), false, true, true); + assert_eq!(delay, Duration::ZERO); +} + +#[test] +fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() { + // Active replication but no buckets → no objects to scan, keep normal delay. + let delay = initial_scanner_delay_for_startup(Some(120), false, false, true); + assert!(delay >= Duration::from_secs(108)); + assert!(delay <= Duration::from_secs(132)); +} + +#[test] +#[serial] +fn test_scanner_cycle_max_duration_uses_env() { + with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("42"), || { + assert_eq!(scanner_cycle_max_duration(), Some(Duration::from_secs(42))); + }); +} + +#[test] +#[serial] +fn test_scanner_cycle_max_duration_default_is_disabled() { + with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { + assert_eq!(scanner_cycle_max_duration(), None); + }); +} + +#[tokio::test] +async fn test_scanner_cycle_budget_cancels_after_duration() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_millis(1)), + ..Default::default() + }, + ); + + tokio::time::timeout(Duration::from_secs(5), budget.token().cancelled()) + .await + .expect("scanner cycle budget should cancel after max duration"); + + assert!(budget.budget_elapsed()); + assert!(budget.token().is_cancelled()); +} + +#[tokio::test] +async fn test_scanner_cycle_budget_drop_cancels_child_without_elapsed() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(60)), + ..Default::default() + }, + ); + let token = budget.token(); + + drop(budget); + + assert!(token.is_cancelled()); +} + +#[test] +#[serial] +fn test_scanner_cycle_budget_config_uses_work_budget_env() { + with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("100"), || { + with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("25"), || { + let config = scanner_cycle_budget_config(); + assert_eq!(config.max_objects, Some(100)); + assert_eq!(config.max_directories, Some(25)); + }); + }); +} + +#[test] +#[serial] +fn test_scanner_cycle_budget_config_disables_zero_work_budgets() { + with_var(ENV_SCANNER_CYCLE_MAX_OBJECTS, Some("0"), || { + with_var(ENV_SCANNER_CYCLE_MAX_DIRECTORIES, Some("0"), || { + let config = scanner_cycle_budget_config(); + assert_eq!(config.max_objects, None); + assert_eq!(config.max_directories, None); + }); + }); +} + +#[test] +fn test_scan_cycle_partial_reason_maps_budget_reason() { + assert_eq!( + scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Runtime)), + ScanCyclePartialReason::Runtime + ); + assert_eq!( + scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Objects)), + ScanCyclePartialReason::Objects + ); + assert_eq!( + scan_cycle_partial_reason(Some(ScannerCycleBudgetReason::Directories)), + ScanCyclePartialReason::Directories + ); + assert_eq!(scan_cycle_partial_reason(None), ScanCyclePartialReason::Unknown); +} + +#[test] +fn test_scan_cycle_partial_source_maps_budget_reason() { + assert_eq!(scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Runtime)), None); + assert_eq!( + scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Objects)), + Some(ScannerWorkSource::Usage) + ); + assert_eq!( + scan_cycle_partial_source(Some(ScannerCycleBudgetReason::Directories)), + Some(ScannerWorkSource::Usage) + ); + assert_eq!(scan_cycle_partial_source(None), None); +} + +#[tokio::test] +#[serial] +async fn test_mark_scan_cycle_idle_clears_published_cycle_state() { + let mut cycle_info = CurrentCycle { + current: 12, + next: 13, + cycle_completed: vec![Utc::now()], + started: Utc::now(), + }; + + global_metrics().set_current_scan_mode(HealScanMode::Deep); + let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + + mark_scan_cycle_idle(&mut cycle_info, &mut cycle_metrics_guard).await; + + let published = global_metrics() + .get_cycle() + .await + .expect("scanner cycle state should remain published"); + + assert_eq!(cycle_info.current, 0); + assert_eq!(cycle_info.next, 13); + assert_eq!(published.current, 0); + assert_eq!(published.next, 13); + assert_eq!(global_metrics().current_scan_mode(), HealScanMode::Unknown); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() { + let cycle_started = Utc::now() - chrono::Duration::seconds(5); + let mut cycle_info = CurrentCycle { + current: 0, + next: 1, + started: cycle_started, + ..Default::default() + }; + let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + let setup_report = global_metrics().report().await; + assert!(setup_report.current_cycle_active); + assert_eq!(setup_report.current_cycle, 0); + assert_eq!(setup_report.current_started.as_second(), cycle_started.timestamp()); + assert_eq!( + setup_report.current_started.subsec_nanosecond(), + i32::try_from(cycle_started.timestamp_subsec_nanos()).expect("chrono nanoseconds fit in i32") + ); + + mark_scan_cycle_idle(&mut cycle_info, &mut guard).await; + let idle_report = global_metrics().report().await; + assert!(!idle_report.current_cycle_active); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() { + let mut cycle_info = CurrentCycle { + current: 12, + next: 13, + started: Utc::now(), + ..Default::default() + }; + let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + + cycle_info.current = 0; + tokio::task::yield_now().await; + let finalizing_report = global_metrics().report().await; + assert!(finalizing_report.current_cycle_active); + assert_eq!(finalizing_report.current_cycle, 12); + + guard.finish(cycle_info).await; + let idle_report = global_metrics().report().await; + assert!(!idle_report.current_cycle_active); + assert_eq!(idle_report.current_cycle, 0); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn scanner_cycle_metrics_guard_drop_clears_activity() { + let guard = ScannerCycleMetricsGuard::new(CurrentCycle { + current: 12, + next: 13, + started: Utc::now(), + ..Default::default() + }) + .await; + assert!(global_metrics().report().await.current_cycle_active); + + drop(guard); + + assert!(!global_metrics().report().await.current_cycle_active); + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let ctx = CancellationToken::new(); + let mut cycle_info = CurrentCycle::default(); + let mut revision = DataUsageCacheRevision::Missing; + let leader_epoch = u64::MAX - 1; + let state_persist_reached = Arc::new(Notify::new()); + let _state_persist_hook = set_scanner_cycle_state_persist_test_hook(leader_epoch, state_persist_reached.clone()); + let state_lock = store + .new_ns_lock(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("scanner cycle state lock should be created"); + let state_guard = state_lock + .get_write_lock(Duration::from_secs(1)) + .await + .expect("scanner cycle state lock should be acquired"); + let mut cycle = Box::pin(run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch)); + let waker = std::task::Waker::noop(); + let mut context = std::task::Context::from_waker(waker); + + assert!(cycle.as_mut().poll(&mut context).is_pending()); + let active = global_metrics().report().await; + assert!(active.current_cycle_active); + assert_eq!(active.current_cycle, 0); + + tokio::time::timeout(Duration::from_secs(30), async { + tokio::select! { + outcome = &mut cycle => panic!("scanner cycle finished before state persistence was released: {outcome:?}"), + _ = state_persist_reached.notified() => {} + } + }) + .await + .expect("scanner cycle should reach state persistence"); + let finalizing = global_metrics().report().await; + assert!(finalizing.current_cycle_active); + + drop(state_guard); + let outcome = tokio::time::timeout(Duration::from_secs(30), cycle) + .await + .expect("scanner cycle should finish"); + assert!(matches!( + outcome, + ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance + )); + assert!(!global_metrics().report().await.current_cycle_active); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + + assert!(finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await); + + assert_eq!(cycle_info.next, 13); + assert_eq!(cycle_info.current, 0); + assert!(cycle_info.cycle_completed.is_empty()); + assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1")); + + let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("cycle state should be persisted after a partial cycle"); + assert_eq!( + u64::from_le_bytes(buf[0..8].try_into().expect("persisted state should start with the counter")), + 13 + ); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle info should decode"); + assert_eq!(decoded.next, 13); + assert_eq!(decoded.current, 0); + assert_eq!(epoch, 1); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn scanner_cycle_recovers_to_newer_durable_cache_floor() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + + assert!( + persist_required_scanner_cycle_floor( + &ctx, + store.clone(), + &mut cycle_info, + &mut revision, + 7, + 19, + &mut cycle_metrics_guard, + ) + .await + ); + assert_eq!(cycle_info.current, 0); + assert_eq!(cycle_info.next, 19); + + let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("recovered cycle floor should be persisted"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("recovered cycle state should decode"); + assert_eq!(decoded.current, 0); + assert_eq!(decoded.next, 19); + assert_eq!(epoch, 7); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn scanner_cycle_rejects_invalid_cache_floor() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + + assert!( + !persist_required_scanner_cycle_floor( + &ctx, + store.clone(), + &mut cycle_info, + &mut revision, + 7, + 12, + &mut cycle_metrics_guard, + ) + .await + ); + assert_eq!(cycle_info.next, 12); + assert_eq!(revision, DataUsageCacheRevision::Missing); + let mut max_cycle_info = CurrentCycle { + current: 12, + next: 12, + ..Default::default() + }; + let mut max_cycle_metrics_guard = ScannerCycleMetricsGuard::new(max_cycle_info.clone()).await; + assert!( + !persist_required_scanner_cycle_floor( + &ctx, + store.clone(), + &mut max_cycle_info, + &mut revision, + 7, + u64::MAX, + &mut max_cycle_metrics_guard, + ) + .await + ); + assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); + + global_metrics().set_cycle(None).await; +} + +#[test] +fn scanner_cycle_state_decodes_legacy_and_fenced_formats() { + let cycle = CurrentCycle { + current: 12, + next: 13, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut legacy = cycle.next.to_le_bytes().to_vec(); + legacy.extend(cycle.marshal().expect("legacy cycle state should encode")); + + let (legacy_cycle, legacy_epoch) = decode_scanner_cycle_state(&legacy).expect("legacy cycle state should remain readable"); + assert_eq!(legacy_cycle.next, 13); + assert_eq!(legacy_epoch, 0); + + let fenced = encode_scanner_cycle_state(&cycle, 7).expect("fenced cycle state should encode"); + let (fenced_cycle, fenced_epoch) = decode_scanner_cycle_state(&fenced).expect("fenced cycle state should decode"); + assert_eq!(fenced_cycle.next, 13); + assert_eq!(fenced_epoch, 7); +} + +#[test] +fn scanner_startup_fails_closed_on_nonempty_corrupt_cycle_state() { + assert_eq!( + decode_scanner_cycle_state_for_startup(&[]) + .expect("missing cycle state should use defaults") + .1, + 0 + ); + assert!(decode_scanner_cycle_state_for_startup(&[1]).is_err()); + + let mut corrupt_fenced = 13_u64.to_le_bytes().to_vec(); + corrupt_fenced.extend_from_slice(SCANNER_CYCLE_STATE_MAGIC); + corrupt_fenced.extend_from_slice(&7_u64.to_le_bytes()); + corrupt_fenced.extend_from_slice(b"not-msgpack"); + assert!(decode_scanner_cycle_state_for_startup(&corrupt_fenced).is_err()); + assert!(decode_scanner_cycle_state_for_startup(&u64::MAX.to_le_bytes()).is_err()); + + let exhausted = CurrentCycle { + next: u64::MAX, + ..Default::default() + }; + assert!(encode_scanner_cycle_state(&exhausted, 7).is_err()); +} + +#[tokio::test] +async fn scanner_startup_uses_primary_and_backup_usage_floor() { + let store = Arc::new(MemoryConfigStore::default()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] { + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, path), + serde_json::to_vec(&DataUsageInfo { + scanner_epoch: Some(epoch), + scanner_cycle: Some(cycle), + ..Default::default() + }) + .expect("usage snapshot should encode"), + ); + } + + let floor = persisted_usage_floor(store).await.expect("usage floor should load"); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 104, + leader_epoch: 11, + } + ); + + let mut cycle = CurrentCycle::default(); + let mut epoch = 0; + apply_persisted_usage_floor(&mut cycle, &mut epoch, floor); + assert_eq!(cycle.next, 104); + assert_eq!(epoch, 11); +} + +#[test] +fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() { + let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::now()), 1); + legacy.usage_snapshot_complete = false; + + assert!(data_usage_info_is_cold(&legacy)); + assert!(!data_usage_info_is_cold(&complete_usage_with_bucket_count( + Some(std::time::SystemTime::now()), + 1, + ))); + assert!(!data_usage_info_is_cold(&DataUsageInfo { + last_update: Some(std::time::SystemTime::now()), + usage_snapshot_complete: true, + ..Default::default() + })); +} + +#[test] +fn scanner_startup_prompts_only_for_a_newer_valid_observation() { + let authoritative = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH), + scanner_epoch: Some(4), + scanner_cycle: Some(10), + ..complete_usage_with_bucket_count(None, 0) + }; + let observed = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)), + scanner_epoch: Some(4), + scanner_cycle: Some(11), + usage_snapshot_converged: Some(false), + usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()), + ..complete_usage_with_bucket_count(None, 0) + }; + + assert!(usage_cache_needs_prompt_scan(&authoritative, Some(&observed))); + assert!(!usage_cache_needs_prompt_scan(&authoritative, None)); + + let mut converged = observed.clone(); + converged.usage_snapshot_converged = Some(true); + assert!(!usage_cache_needs_prompt_scan(&authoritative, Some(&converged))); + + let mut legacy_observation = observed; + legacy_observation.usage_snapshot_converged = None; + assert!(!usage_cache_needs_prompt_scan(&authoritative, Some(&legacy_observation))); +} + +#[tokio::test] +async fn scanner_startup_prefers_v2_over_legacy_usage() { + let store = Arc::new(MemoryConfigStore::default()); + let legacy = DataUsageInfo { + scanner_epoch: Some(19), + scanner_cycle: Some(41), + last_update: Some(std::time::SystemTime::now()), + ..Default::default() + }; + let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + legacy_data.clone(), + ); + + assert_eq!( + read_data_usage_config_for_startup(&store) + .await + .expect("legacy startup usage should load"), + Some(legacy_data) + ); + assert_eq!( + persisted_usage_floor(store.clone()) + .await + .expect("legacy usage floor should seed the upgrade"), + PersistedUsageFloor { + next_cycle: 42, + leader_epoch: 19, + } + ); + + let authoritative = DataUsageInfo { + scanner_epoch: Some(23), + scanner_cycle: Some(51), + last_update: Some(std::time::SystemTime::now()), + usage_snapshot_complete: true, + ..Default::default() + }; + let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode"); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + authoritative_data.clone(), + ); + + assert_eq!( + read_data_usage_config_for_startup(&store) + .await + .expect("v2 startup usage should load"), + Some(authoritative_data) + ); + assert_eq!( + persisted_usage_floor(store.clone()) + .await + .expect("v2 usage floor should be authoritative"), + PersistedUsageFloor { + next_cycle: 52, + leader_epoch: 23, + } + ); + + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + b"corrupt-v2".to_vec(), + ); + assert_eq!( + read_data_usage_config_for_startup(&store) + .await + .expect("startup inspection should preserve authoritative bytes"), + Some(b"corrupt-v2".to_vec()) + ); + assert!( + persisted_usage_floor(store).await.is_err(), + "corrupt v2 state must not fall back to a legacy writer" + ); +} + +#[tokio::test] +async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() { + let store = Arc::new(MemoryConfigStore::default()); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + b"not-json".to_vec(), + ); + + assert!(persisted_usage_floor(store.clone()).await.is_err()); + + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&DataUsageInfo { + scanner_cycle: Some(u64::MAX - 1), + ..Default::default() + }) + .expect("usage snapshot should encode"), + ); + assert!(persisted_usage_floor(store).await.is_err()); +} + +#[tokio::test] +#[serial] +async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + + for cycle in [9, 10] { + let (sender, receiver) = mpsc::channel(1); + sender + .send(DataUsageInfo { + scanner_epoch: Some(1), + scanner_cycle: Some(cycle), + last_update: Some(std::time::SystemTime::now()), + ..complete_usage_with_bucket_count(None, 0) + }) + .await + .expect("usage update should queue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(ctx.clone(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup = read_config(store.clone(), &backup_path).await; + if cycle == 9 { + assert!(matches!(backup, Err(EcstoreError::ConfigNotFound))); + } else { + let saved = + serde_json::from_slice::(&backup.expect("the tenth durable scanner cycle should create a backup")) + .expect("backup usage snapshot should decode"); + assert_eq!(saved.scanner_cycle, Some(10)); + assert_eq!(saved.scanner_epoch, Some(1)); + } + } +} + +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for MemoryConfigStore { + async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult { + let key = memory_config_key(bucket, object); + let mut objects = self.objects.lock().await; + if !objects.contains_key(&key) { + return Err(EcstoreError::FileNotFound); + } + let mut revisions = self.revisions.lock().await; + if let Some(expected) = opts + .http_preconditions + .as_ref() + .and_then(|preconditions| preconditions.if_match.as_deref()) + { + let actual = revisions.get(&key).map(|revision| format!("memory-{revision}")); + if actual.as_deref() != Some(expected.trim_matches('"')) { + return Err(EcstoreError::PreconditionFailed); + } + } + objects.remove(&key); + revisions.remove(&key); + Ok(ObjectInfo::default()) + } +} + +#[test] +fn scanner_cycle_advance_fails_before_reserved_exhausted_value() { + let mut cycle = CurrentCycle { + next: u64::MAX - 2, + ..Default::default() + }; + advance_scanner_cycle(&mut cycle).expect("last persistable scanner cycle should remain valid"); + assert_eq!(cycle.next, u64::MAX - 1); + assert!(advance_scanner_cycle(&mut cycle).is_err()); + assert_eq!(cycle.next, u64::MAX - 1); +} + +#[tokio::test] +#[serial] +async fn test_finalize_partial_scan_cycle_reports_persist_failure() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.fail_put_number.lock().await.insert(key, 1); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; + + assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await); + assert_eq!(cycle_info.next, 13); + assert_eq!(cycle_info.current, 0); + assert_eq!(revision, DataUsageCacheRevision::Missing); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +#[serial] +async fn test_persist_scanner_cycle_state_reconciles_newer_winner() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut initial_revision = DataUsageCacheRevision::Missing; + let mut initial = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await); + + let mut current_revision = initial_revision.clone(); + let mut stale_revision = initial_revision; + let mut current = CurrentCycle { + next: 14, + ..initial.clone() + }; + let mut stale = CurrentCycle { next: 13, ..initial }; + + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut current_revision, 1).await); + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut stale, &mut stale_revision, 1).await); + + let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("new leader cycle state should remain persisted"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); + assert_eq!(decoded.next, 14); + assert_eq!(epoch, 1); + assert_eq!(stale.next, 14); + assert!(matches!(current_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + assert!(matches!(stale_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + + global_metrics().set_cycle(None).await; +} + +#[tokio::test] +async fn test_persist_scanner_cycle_state_retries_after_stale_winner() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut initial_revision = DataUsageCacheRevision::Missing; + let mut initial = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut initial_revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let stale = CurrentCycle { + next: 13, + ..initial.clone() + }; + let stale_buf = encode_scanner_cycle_state(&stale, 1).expect("stale cycle state should encode"); + store.interleaving_puts.lock().await.insert(key, (2, stale_buf)); + + let mut current = CurrentCycle { next: 14, ..initial }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut current, &mut initial_revision, 1).await); + + let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("newer cycle state should replace the stale conflict winner"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); + assert_eq!(decoded.next, 14); + assert_eq!(epoch, 1); + assert_eq!(current.next, 14); + assert!(matches!(initial_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-3")); +} + +#[tokio::test] +async fn test_persist_scanner_cycle_state_stops_retry_after_leader_fence() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut initial = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut initial, &mut revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let replacement = CurrentCycle { + next: 13, + ..initial.clone() + }; + let replacement_buf = encode_scanner_cycle_state(&replacement, 2).expect("replacement cycle state should encode"); + store.interleaving_puts.lock().await.insert(key.clone(), (2, replacement_buf)); + store + .cancel_after_interleaving_puts + .lock() + .await + .insert(key.clone(), ctx.clone()); + + let mut stale_leader = CurrentCycle { next: 14, ..initial }; + assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut stale_leader, &mut revision, 1).await); + + let buf = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("replacement leader cycle state should remain persisted"); + let (decoded, epoch) = decode_scanner_cycle_state(&buf).expect("persisted cycle state should decode"); + assert_eq!(decoded.next, 13); + assert_eq!(epoch, 2); + assert_eq!(stale_leader.next, 14); + assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&2)); +} + +#[tokio::test] +async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conflict() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let old_epoch_commit = CurrentCycle { + next: 14, + ..cycle.clone() + }; + store.interleaving_puts.lock().await.insert( + key.clone(), + ( + 2, + encode_scanner_cycle_state(&old_epoch_commit, 1).expect("old-epoch cycle state should encode"), + ), + ); + + let mut persisted_epoch = 8; + assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + + let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("new leadership claim should remain persisted"); + let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode"); + assert_eq!(claimed_cycle.next, 14); + assert_eq!(claimed_epoch, 9); + assert_eq!(persisted_epoch, 9); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&3)); +} + +#[tokio::test] +async fn test_leadership_claim_confirms_commit_after_returned_error() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store.error_after_commit_put_number.lock().await.insert(key.clone(), 1); + store.error_after_commit_put_number.lock().await.insert(usage_key.clone(), 1); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + let mut persisted_epoch = 0; + + assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); + + let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("ambiguous leadership claim should be durable"); + let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("claimed cycle state should decode"); + assert_eq!(claimed_cycle.next, 12); + assert_eq!(claimed_epoch, 1); + assert_eq!(persisted_epoch, 1); + assert!(matches!(revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-1")); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&1)); + let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("ambiguous usage epoch fence should be durable"); + assert_eq!( + serde_json::from_slice::(&usage) + .expect("usage epoch fence should decode") + .scanner_epoch, + Some(1) + ); + assert_eq!(store.put_counts.lock().await.get(&usage_key), Some(&1)); +} + +#[tokio::test] +async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { + let store = Arc::new(MemoryConfigStore::default()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let mut old_usage = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(4), + scanner_cycle: Some(11), + ..Default::default() + }; + old_usage.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + old_usage.buckets_count = 1; + old_usage.calculate_totals(); + let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode"); + store.objects.lock().await.insert(usage_key.clone(), old_data.clone()); + store.revisions.lock().await.insert(usage_key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + started: Utc::now(), + ..Default::default() + }; + let mut persisted_epoch = 4; + assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); + + let (fenced_data, fenced_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("fenced usage snapshot should load"); + let fenced = serde_json::from_slice::(fenced_data.as_deref().expect("fenced usage snapshot should exist")) + .expect("fenced usage snapshot should decode"); + assert_eq!(fenced.scanner_epoch, Some(5)); + assert_eq!(fenced.objects_total_count, 2); + assert_eq!(fenced.buckets_usage.get("bucket-a").map(|usage| usage.size), Some(84)); + assert!(matches!(fenced_revision, DataUsageCacheRevision::Etag(ref etag) if etag == "memory-2")); + + let stale_save = save_config_with_preconditions( + store, + DATA_USAGE_OBJ_NAME_PATH.as_str(), + old_data, + DataUsageCacheRevision::Etag("memory-1".to_string()).preconditions(), + ) + .await; + assert!(matches!(stale_save, Err(EcstoreError::PreconditionFailed))); +} + +#[tokio::test] +async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + cycle_completed: vec![], + started: Utc::now(), + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store + .cancel_after_successful_puts + .lock() + .await + .insert(key.clone(), (2, ctx.clone())); + cycle.next = 14; + assert!(!persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + + let (persisted, persisted_revision) = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("committed old-epoch state should load"); + let mut replacement_cycle = decode_scanner_cycle_state( + persisted + .as_deref() + .expect("old-epoch state should have committed before cancellation"), + ) + .expect("old-epoch state should decode") + .0; + let mut replacement_revision = persisted_revision; + let mut replacement_epoch = 1; + let replacement_ctx = CancellationToken::new(); + assert!( + claim_scanner_leadership( + &replacement_ctx, + store.clone(), + &mut replacement_cycle, + &mut replacement_revision, + &mut replacement_epoch, + ) + .await + ); + + let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("replacement leadership claim should persist"); + let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode"); + assert_eq!(claimed_cycle.next, 14); + assert_eq!(claimed_epoch, 2); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_preserves_newer_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(2); + let ctx = CancellationToken::new(); + + let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2); + let older = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1); + + sender.send(newer).await.expect("newer usage snapshot should enqueue"); + sender.send(older).await.expect("older usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + let objects = store.objects.lock().await; + let saved = objects + .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str())) + .expect("data usage config should be saved"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + + assert_eq!(saved.buckets_count, 2); + assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20))); + assert_eq!(outcome, DataUsagePersistOutcome::Current); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_fences_interleaving_newer_writer() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2); + let stale = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 1); + store + .interleaving_puts + .lock() + .await + .insert(key.clone(), (1, serde_json::to_vec(&newer).expect("newer usage snapshot should encode"))); + + sender.send(stale).await.expect("stale usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + let objects = store.objects.lock().await; + let saved = objects + .get(&key) + .expect("interleaving newer usage snapshot should remain saved"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + assert_eq!(saved.buckets_count, 2); + assert_eq!(saved.last_update, newer.last_update); + assert_eq!(outcome, DataUsagePersistOutcome::Current); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_does_not_resurrect_deleted_bucket_after_conflict() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let mut initial = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + ..Default::default() + }; + initial.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + initial.bucket_sizes.insert("bucket-a".to_string(), 84); + initial.buckets_count = 1; + initial.calculate_totals(); + mark_usage_snapshot_complete(&mut initial); + let initial_data = serde_json::to_vec(&initial).expect("initial usage snapshot should encode"); + store.objects.lock().await.insert(key.clone(), initial_data.clone()); + store.revisions.lock().await.insert(key.clone(), 1); + + let mut deleted = initial.clone(); + deleted.buckets_usage.clear(); + deleted.bucket_sizes.clear(); + deleted.buckets_count = 0; + deleted.calculate_totals(); + mark_usage_snapshot_complete(&mut deleted); + store + .interleaving_puts + .lock() + .await + .insert(key.clone(), (1, serde_json::to_vec(&deleted).expect("deleted snapshot should encode"))); + + let mut incoming = initial; + incoming.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)); + incoming.scanner_cycle = Some(13); + let (sender, receiver) = mpsc::channel(1); + sender.send(incoming).await.expect("stale scanner snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline( + CancellationToken::new(), + store.clone(), + receiver, + Some(8), + Some(DataUsagePersistBaseline { + data: Some(Bytes::from(initial_data)), + revision: DataUsageCacheRevision::Etag("memory-1".to_string()), + }), + ) + .await; + + assert_eq!(outcome, DataUsagePersistOutcome::Current); + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("deleted usage snapshot should remain"); + let saved = serde_json::from_slice::(&saved).expect("deleted usage snapshot should decode"); + assert!(!saved.buckets_usage.contains_key("bucket-a")); + assert!(!saved.bucket_sizes.contains_key("bucket-a")); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&1)); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_updates_backup_with_new_bucket() { + let store = Arc::new(MemoryConfigStore::default()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + let deleted = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(8), + scanner_cycle: Some(1), + ..complete_usage_with_bucket_count(None, 0) + }; + store.objects.lock().await.insert( + backup_key.clone(), + serde_json::to_vec(&deleted).expect("deleted backup snapshot should encode"), + ); + store.revisions.lock().await.insert(backup_key.clone(), 1); + + let (sender, receiver) = mpsc::channel(11); + for cycle in 2_u64..=12 { + let mut incoming = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20 + cycle)), + scanner_epoch: Some(8), + scanner_cycle: Some(cycle), + ..Default::default() + }; + incoming.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + incoming.bucket_sizes.insert("bucket-a".to_string(), 84); + incoming.buckets_count = 1; + incoming.calculate_totals(); + mark_usage_snapshot_complete(&mut incoming); + sender.send(incoming).await.expect("usage snapshot should enqueue"); + } + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + + let saved = store + .objects + .lock() + .await + .get(&backup_key) + .cloned() + .expect("deleted backup snapshot should remain"); + let saved = serde_json::from_slice::(&saved).expect("backup snapshot should decode"); + assert!(saved.buckets_usage.contains_key("bucket-a")); + assert!(saved.bucket_sizes.contains_key("bucket-a")); + assert_eq!(saved.scanner_cycle, Some(10)); + assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1)); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_repairs_backup_after_primary_only_commit() { + let store = Arc::new(MemoryConfigStore::default()); + let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + let durable = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), + scanner_epoch: Some(8), + scanner_cycle: Some(10), + ..complete_usage_with_bucket_count(None, 0) + }; + let encoded = serde_json::to_vec(&durable).expect("usage snapshot should encode"); + store.objects.lock().await.insert(main_key.clone(), encoded.clone()); + store.revisions.lock().await.insert(main_key.clone(), 1); + + let (sender, receiver) = mpsc::channel(1); + sender.send(durable).await.expect("usage snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::AlreadyDurable + ); + assert_eq!(store.objects.lock().await.get(&backup_key), Some(&encoded)); + assert_eq!(store.put_counts.lock().await.get(&main_key), None); + assert_eq!(store.put_counts.lock().await.get(&backup_key), Some(&1)); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_copies_concurrent_bucket_removal_to_backup() { + let store = Arc::new(MemoryConfigStore::default()); + let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + + let mut incoming = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), + scanner_epoch: Some(8), + scanner_cycle: Some(10), + ..Default::default() + }; + incoming.buckets_usage.insert( + "bucket-a".to_string(), + rustfs_data_usage::BucketUsageInfo { + objects_count: 2, + size: 84, + ..Default::default() + }, + ); + incoming.bucket_sizes.insert("bucket-a".to_string(), 84); + incoming.buckets_count = 1; + incoming.calculate_totals(); + mark_usage_snapshot_complete(&mut incoming); + + let mut deleted = incoming.clone(); + deleted.last_update = Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(31)); + deleted.buckets_usage.clear(); + deleted.bucket_sizes.clear(); + deleted.buckets_count = 0; + deleted.calculate_totals(); + mark_usage_snapshot_complete(&mut deleted); + store.replace_after_successful_puts.lock().await.insert( + main_key.clone(), + (1, serde_json::to_vec(&deleted).expect("deleted primary snapshot should encode")), + ); + store.objects.lock().await.insert( + backup_key.clone(), + serde_json::to_vec(&incoming).expect("existing backup snapshot should encode"), + ); + store.revisions.lock().await.insert(backup_key.clone(), 1); + + let (sender, receiver) = mpsc::channel(1); + sender.send(incoming).await.expect("usage snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + + for key in [main_key, backup_key] { + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("usage snapshot should remain"); + let saved = serde_json::from_slice::(&saved).expect("usage snapshot should decode"); + assert!(!saved.buckets_usage.contains_key("bucket-a")); + assert!(!saved.bucket_sizes.contains_key("bucket-a")); + } +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_retries_after_stale_interleaving_writer() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let initial = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10)), 3); + let stale_winner = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 3); + let current = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), 3); + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&initial).expect("initial usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + store.interleaving_puts.lock().await.insert( + key.clone(), + (1, serde_json::to_vec(&stale_winner).expect("stale usage snapshot should encode")), + ); + + sender + .send(current.clone()) + .await + .expect("current usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + let objects = store.objects.lock().await; + let saved = objects + .get(&key) + .expect("current usage snapshot should replace the stale conflict winner"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + assert_eq!(saved.buckets_count, 3); + assert_eq!(saved.last_update, current.last_update); + assert_eq!(outcome, DataUsagePersistOutcome::Saved); + drop(objects); + assert_eq!(store.put_counts.lock().await.get(&key), Some(&2)); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_rejects_untimestamped_complete_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(2); + let ctx = CancellationToken::new(); + + let timestamped = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 2); + let untimestamped = complete_usage_with_bucket_count(None, 1); + + sender + .send(timestamped) + .await + .expect("timestamped usage snapshot should enqueue"); + sender + .send(untimestamped) + .await + .expect("untimestamped usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + let objects = store.objects.lock().await; + let saved = objects + .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str())) + .expect("data usage config should be saved"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + + assert_eq!(saved.buckets_count, 2); + assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20))); + assert_eq!(outcome, DataUsagePersistOutcome::Failed); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_recognizes_already_durable_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let snapshot = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_cycle: Some(12), + ..complete_usage_with_bucket_count(None, 2) + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&snapshot).expect("durable usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + sender + .send(snapshot) + .await + .expect("ambiguous committed snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + assert_eq!(outcome, DataUsagePersistOutcome::AlreadyDurable); + assert_eq!(store.put_counts.lock().await.get(&key), None); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_advances_past_changed_same_epoch_cycle() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let durable = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + ..complete_usage_with_bucket_count(None, 2) + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&durable).expect("durable usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(30)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + ..complete_usage_with_bucket_count(None, 3) + }) + .await + .expect("changed retry snapshot should enqueue"); + drop(sender); + + let outcome = + store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), receiver, Some(8)).await; + + assert_eq!(outcome, DataUsagePersistOutcome::PriorCycleDurable); + assert_eq!(store.put_counts.lock().await.get(&key), None); + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("first snapshot should remain durable"); + assert_eq!( + serde_json::from_slice::(&saved) + .expect("durable usage snapshot should decode") + .buckets_count, + 2 + ); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_orders_scanner_cycles_before_wall_clock() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let existing = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)), + scanner_cycle: Some(12), + ..complete_usage_with_bucket_count(None, 2) + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + let (older_sender, older_receiver) = mpsc::channel(1); + let older = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), + scanner_cycle: Some(11), + ..complete_usage_with_bucket_count(None, 1) + }; + older_sender.send(older).await.expect("older-cycle snapshot should enqueue"); + drop(older_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await, + DataUsagePersistOutcome::Current + ); + + let (newer_sender, newer_receiver) = mpsc::channel(1); + let newer = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), + scanner_cycle: Some(13), + ..complete_usage_with_bucket_count(None, 3) + }; + newer_sender + .send(newer.clone()) + .await + .expect("newer-cycle snapshot should enqueue"); + drop(newer_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), newer_receiver).await, + DataUsagePersistOutcome::Saved + ); + + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("newer scanner cycle should be persisted"); + assert_eq!( + serde_json::from_slice::(&saved) + .expect("persisted usage snapshot should decode") + .scanner_cycle, + Some(13) + ); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_orders_leader_epochs_before_cycles() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let existing = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(200)), + scanner_epoch: Some(8), + scanner_cycle: Some(12), + ..complete_usage_with_bucket_count(None, 2) + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + let (older_sender, older_receiver) = mpsc::channel(1); + older_sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), + scanner_epoch: Some(7), + scanner_cycle: Some(99), + ..complete_usage_with_bucket_count(None, 1) + }) + .await + .expect("old-epoch snapshot should enqueue"); + drop(older_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), older_receiver).await, + DataUsagePersistOutcome::Current + ); + + let (newer_sender, newer_receiver) = mpsc::channel(1); + newer_sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), + scanner_epoch: None, + scanner_cycle: Some(1), + ..complete_usage_with_bucket_count(None, 3) + }) + .await + .expect("replacement-epoch snapshot should enqueue"); + drop(newer_sender); + assert_eq!( + store_data_usage_in_backend_with_outcome_for_epoch(CancellationToken::new(), store.clone(), newer_receiver, Some(9),) + .await, + DataUsagePersistOutcome::Saved + ); + + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("replacement leader snapshot should persist"); + let saved = serde_json::from_slice::(&saved).expect("persisted usage snapshot should decode"); + assert_eq!(saved.scanner_epoch, Some(9)); + assert_eq!(saved.scanner_cycle, Some(1)); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_keeps_first_same_cycle_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let existing = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(100)), + scanner_cycle: Some(12), + ..complete_usage_with_bucket_count(None, 2) + }; + store + .objects + .lock() + .await + .insert(key.clone(), serde_json::to_vec(&existing).expect("existing usage snapshot should encode")); + store.revisions.lock().await.insert(key.clone(), 1); + + let (sender, receiver) = mpsc::channel(1); + sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(300)), + scanner_cycle: Some(12), + ..complete_usage_with_bucket_count(None, 3) + }) + .await + .expect("conflicting same-cycle snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Current + ); + let saved = store + .objects + .lock() + .await + .get(&key) + .cloned() + .expect("first same-cycle snapshot should remain persisted"); + assert_eq!( + serde_json::from_slice::(&saved) + .expect("persisted usage snapshot should decode") + .buckets_count, + 2 + ); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_rejects_incomplete_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(2); + let complete_update = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10); + + sender + .send(complete_usage_with_bucket_count(Some(complete_update), 1)) + .await + .expect("complete usage snapshot should enqueue"); + sender + .send(DataUsageInfo { + last_update: Some(complete_update + Duration::from_secs(1)), + buckets_count: 1, + ..Default::default() + }) + .await + .expect("incomplete usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await; + + let objects = store.objects.lock().await; + let saved = objects + .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str())) + .expect("complete data usage snapshot should remain saved"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + assert_eq!(saved.last_update, Some(complete_update)); + assert!(saved.is_complete_bucket_usage_snapshot()); + assert_eq!(outcome, DataUsagePersistOutcome::Failed); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_preserves_superseded_status() { + let store = Arc::new(MemoryConfigStore::default()); + let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let authoritative = DataUsageInfo { + scanner_epoch: Some(7), + scanner_cycle: Some(10), + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 1) + }; + let authoritative_bytes = serde_json::to_vec(&authoritative).expect("authoritative snapshot should encode"); + store + .objects + .lock() + .await + .insert(authoritative_key.clone(), authoritative_bytes.clone()); + store.revisions.lock().await.insert(authoritative_key.clone(), 1); + + let (sender, receiver) = mpsc::channel(1); + sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)), + scanner_epoch: Some(7), + scanner_cycle: Some(11), + usage_snapshot_converged: Some(false), + ..complete_usage_with_bucket_count(None, 1) + }) + .await + .expect("superseded usage snapshot should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await; + let saved = store + .objects + .lock() + .await + .get(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())) + .cloned() + .expect("superseded usage snapshot should persist"); + let saved = serde_json::from_slice::(&saved).expect("persisted usage snapshot should decode"); + + assert_eq!(outcome, DataUsagePersistOutcome::Saved); + assert!(saved.is_complete_bucket_usage_snapshot()); + assert_eq!(saved.usage_snapshot_converged, Some(false)); + assert_eq!(saved.usage_snapshot_authoritative_baseline, Some(authoritative.snapshot_identity())); + assert_eq!( + store.objects.lock().await.get(&authoritative_key), + Some(&authoritative_bytes), + "an observation must never lower the quota-authoritative snapshot" + ); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_removes_observed_after_authoritative_save() { + let store = Arc::new(MemoryConfigStore::default()); + let authoritative_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let authoritative = DataUsageInfo { + scanner_epoch: Some(7), + scanner_cycle: Some(10), + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 1) + }; + store.objects.lock().await.insert( + authoritative_key.clone(), + serde_json::to_vec(&authoritative).expect("authoritative snapshot should encode"), + ); + store.revisions.lock().await.insert(authoritative_key, 1); + + let (sender, receiver) = mpsc::channel(1); + sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1)), + scanner_epoch: Some(7), + scanner_cycle: Some(11), + usage_snapshot_converged: Some(false), + ..complete_usage_with_bucket_count(None, 1) + }) + .await + .expect("superseded usage snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + let observed_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()); + assert!(store.objects.lock().await.contains_key(&observed_key)); + + let (sender, receiver) = mpsc::channel(1); + sender + .send(DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(2)), + scanner_epoch: Some(7), + scanner_cycle: Some(12), + usage_snapshot_converged: Some(true), + ..complete_usage_with_bucket_count(None, 1) + }) + .await + .expect("authoritative usage snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + assert!( + !store.objects.lock().await.contains_key(&observed_key), + "an authoritative snapshot should retire stale observations" + ); + + let next_authoritative = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(3)), + scanner_epoch: Some(7), + scanner_cycle: Some(13), + usage_snapshot_converged: Some(true), + ..complete_usage_with_bucket_count(None, 1) + }; + let newer_observed = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(4)), + scanner_epoch: Some(7), + scanner_cycle: Some(14), + usage_snapshot_converged: Some(false), + usage_snapshot_authoritative_baseline: Some(next_authoritative.snapshot_identity()), + ..complete_usage_with_bucket_count(None, 1) + }; + store.objects.lock().await.insert( + observed_key.clone(), + serde_json::to_vec(&newer_observed).expect("newer observed snapshot should encode"), + ); + store.revisions.lock().await.insert(observed_key.clone(), 3); + + let (sender, receiver) = mpsc::channel(1); + sender + .send(next_authoritative) + .await + .expect("next authoritative usage snapshot should enqueue"); + drop(sender); + + assert_eq!( + store_data_usage_in_backend_with_outcome(CancellationToken::new(), store.clone(), receiver).await, + DataUsagePersistOutcome::Saved + ); + assert!( + store.objects.lock().await.contains_key(&observed_key), + "a newer observation must survive stale authoritative cleanup" + ); +} + +fn mark_usage_snapshot_complete(info: &mut DataUsageInfo) { + info.usage_snapshot_complete = true; +} + +fn complete_usage_with_bucket_count(last_update: Option, buckets_count: u64) -> DataUsageInfo { + let mut info = DataUsageInfo { + last_update, + buckets_count, + usage_snapshot_complete: true, + ..Default::default() + }; + for index in 0..buckets_count { + let bucket = format!("bucket-{index}"); + info.buckets_usage.insert(bucket.clone(), Default::default()); + info.bucket_sizes.insert(bucket, 0); + } + info +} + +fn usage_with_last_update(last_update: Option) -> DataUsageInfo { + complete_usage_with_bucket_count(last_update, 0) +} + +#[test] +fn test_stale_data_usage_update_reason_allows_newer_incoming() { + let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + let incoming = usage_with_last_update(Some(now)); + let existing = usage_with_last_update(Some(now - Duration::from_secs(60))); + assert_eq!(stale_data_usage_update_reason(&incoming, &existing, now), None); +} + +#[test] +fn test_stale_data_usage_update_reason_skips_older_or_equal_incoming() { + let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + let existing = usage_with_last_update(Some(now - Duration::from_secs(60))); + + let older = usage_with_last_update(Some(now - Duration::from_secs(120))); + assert_eq!(stale_data_usage_update_reason(&older, &existing, now), Some("older_or_equal_last_update")); + + let equal = usage_with_last_update(existing.last_update); + assert_eq!(stale_data_usage_update_reason(&equal, &existing, now), Some("older_or_equal_last_update")); +} + +#[test] +fn test_stale_data_usage_update_reason_allows_save_when_existing_is_future_dated() { + // Existing snapshot timestamp beyond the clock tolerance is untrustworthy + // (clock step-back / slower-clock leader): the save must be allowed even + // though incoming <= existing, otherwise usage stats freeze forever. + let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + let existing = + usage_with_last_update(Some(now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE + Duration::from_secs(1))); + let incoming = usage_with_last_update(Some(now)); + assert_eq!(stale_data_usage_update_reason(&incoming, &existing, now), None); +} + +#[test] +fn test_stale_data_usage_update_reason_skips_at_exact_tolerance_boundary() { + // Exactly at now + tolerance is still within the trusted window. + let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + let existing = usage_with_last_update(Some(now + rustfs_data_usage::USAGE_LAST_UPDATE_FUTURE_TOLERANCE)); + let incoming = usage_with_last_update(Some(now)); + assert_eq!( + stale_data_usage_update_reason(&incoming, &existing, now), + Some("older_or_equal_last_update") + ); +} + +#[test] +fn test_stale_data_usage_update_reason_preserves_none_handling() { + let now = std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + + let incoming_none = usage_with_last_update(None); + let existing_some = usage_with_last_update(Some(now - Duration::from_secs(60))); + assert_eq!( + stale_data_usage_update_reason(&incoming_none, &existing_some, now), + Some("missing_incoming_last_update") + ); + + let incoming_some = usage_with_last_update(Some(now)); + let existing_none = usage_with_last_update(None); + assert_eq!(stale_data_usage_update_reason(&incoming_some, &existing_none, now), None); + + let both_none = usage_with_last_update(None); + assert_eq!(stale_data_usage_update_reason(&both_none, &usage_with_last_update(None), now), None); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_keeps_backup_when_primary_save_fails() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(11); + let ctx = CancellationToken::new(); + + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let main_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + let old_backup = b"old-backup".to_vec(); + + store.objects.lock().await.insert(backup_key.clone(), old_backup.clone()); + store.fail_put_number.lock().await.insert(main_key.clone(), 11); + + for idx in 1_u64..=11 { + sender + .send(complete_usage_with_bucket_count( + Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(idx)), + idx, + )) + .await + .expect("usage snapshot should enqueue"); + } + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store.clone(), receiver).await; + + let objects = store.objects.lock().await; + assert_eq!( + objects.get(&backup_key), + Some(&old_backup), + "primary save failure must not overwrite the previous backup" + ); + let saved = objects + .get(&main_key) + .expect("last successful primary usage snapshot should remain saved"); + let saved = serde_json::from_slice::(saved).expect("saved usage snapshot should decode"); + assert_eq!(saved.buckets_count, 10); + assert_eq!(saved.last_update, Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(10))); + assert_eq!(outcome, DataUsagePersistOutcome::Failed); +} + +#[tokio::test] +async fn test_store_data_usage_in_backend_reports_missing_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let ctx = CancellationToken::new(); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome(ctx, store, receiver).await; + + assert_eq!(outcome, DataUsagePersistOutcome::NoUpdate); +} + +#[test] +fn test_scanner_cycle_completion_prioritizes_persist_failure() { + assert_eq!( + scanner_cycle_completion_outcome( + ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable), + DataUsagePersistOutcome::NoUpdate, + false, + false, + ), + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable) + ); + assert_eq!( + scanner_cycle_completion_outcome( + ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), + DataUsagePersistOutcome::Saved, + false, + false, + ), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome( + ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), + DataUsagePersistOutcome::NoUpdate, + true, + false, + ), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome( + ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), + DataUsagePersistOutcome::Failed, + false, + false, + ), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, false, false), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Failed, true, true), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, true, true), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, false), + ScannerCycleOutcome::Partial + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Saved, true, true), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Saved, true, false), + ScannerCycleOutcome::Completed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::AlreadyDurable, true, false,), + ScannerCycleOutcome::Completed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::PriorCycleDurable, true, false,), + ScannerCycleOutcome::Completed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, false, false), + ScannerCycleOutcome::Completed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::Current, true, false), + ScannerCycleOutcome::Failed + ); + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Complete, DataUsagePersistOutcome::NoUpdate, false, false), + ScannerCycleOutcome::Failed + ); + for persist_outcome in [ + DataUsagePersistOutcome::NoUpdate, + DataUsagePersistOutcome::Current, + DataUsagePersistOutcome::Saved, + ] { + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, persist_outcome, true, false), + ScannerCycleOutcome::Superseded + ); + } + assert_eq!( + scanner_cycle_completion_outcome(ScannerCycleStatus::Superseded, DataUsagePersistOutcome::Saved, true, true), + ScannerCycleOutcome::Failed + ); +} + +#[test] +#[serial] +fn finalizing_a_saved_cycle_acknowledges_its_exact_dirty_snapshot() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let remote_acknowledgement = ScannerDirtyUsageAcknowledgement { + host: "node-2".to_string(), + instance_id: "0123456789abcdef0123456789abcdef".to_string(), + generation: 11, + }; + let unsaved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot.clone())) + .with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(unsaved, DataUsagePersistOutcome::NoUpdate); + assert_eq!(outcome, ScannerCycleOutcome::Failed); + assert!(acknowledgements.is_empty()); + assert!(crate::scanner_io::dirty_usage_buckets_pending()); + + let saved = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)) + .with_remote_dirty_usage_acknowledgements(vec![remote_acknowledgement.clone()]); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(saved, DataUsagePersistOutcome::Saved); + assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert_eq!(acknowledgements, vec![remote_acknowledgement]); + assert!(!crate::scanner_io::dirty_usage_buckets_pending()); +} + +#[tokio::test] +async fn scanner_cycle_keeps_remote_pending_acknowledgement() { + let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::(true))).await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending), + ScannerCycleOutcome::CompletedWithPendingMaintenance + ); + + let cleared = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::(false))).await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, cleared), + ScannerCycleOutcome::Completed + ); + + let failed = remote_dirty_usage_acknowledgement_pending( + 7, + 1, + std::future::ready(Err::(std::io::Error::other("injected acknowledgement failure"))), + ) + .await; + assert_eq!( + scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, failed), + ScannerCycleOutcome::CompletedWithPendingMaintenance + ); +} + +#[test] +#[serial] +fn finalizing_an_already_durable_cycle_acknowledges_its_exact_dirty_snapshot() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::AlreadyDurable); + + assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert!(acknowledgements.is_empty()); + assert!(!crate::scanner_io::dirty_usage_buckets_pending()); +} + +#[test] +#[serial] +fn finalizing_a_prior_same_cycle_snapshot_keeps_new_dirty_work_pending() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let durable = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Complete, Some(dirty_snapshot)); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(durable, DataUsagePersistOutcome::PriorCycleDurable); + + assert_eq!(outcome, ScannerCycleOutcome::Completed); + assert!(acknowledgements.is_empty()); + assert!(crate::scanner_io::dirty_usage_buckets_pending()); + crate::scanner_io::clear_dirty_usage_bucket("photos"); +} + +#[test] +#[serial] +fn finalizing_a_durable_superseded_snapshot_keeps_dirty_work_pending() { + crate::scanner_io::clear_dirty_usage_bucket("photos"); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_snapshot = crate::scanner_io::dirty_usage_buckets_for_tests(); + + let superseded = crate::scanner_io::ScannerCycleResult::new(ScannerCycleStatus::Superseded, Some(dirty_snapshot)); + let (outcome, _, acknowledgements) = finalize_scanner_cycle_result(superseded, DataUsagePersistOutcome::Saved); + + assert_eq!(outcome, ScannerCycleOutcome::Superseded); + assert!(acknowledgements.is_empty()); + assert!(crate::scanner_io::dirty_usage_buckets_pending()); + crate::scanner_io::clear_dirty_usage_bucket("photos"); +} + +#[test] +#[serial] +fn data_usage_persist_wait_covers_cache_retries_and_backup() { + with_var(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, Some("7"), || { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + assert_eq!(data_usage_persist_timeout(), Duration::from_millis(31_350)); + }); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); +} + +#[tokio::test] +async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() { + let ctx = CancellationToken::new(); + let mut task = AbortOnDropHandle::new(tokio::spawn(async { + std::future::pending::<()>().await; + DataUsagePersistOutcome::Saved + })); + ctx.cancel(); + + let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(60)).await; + + assert!(matches!(result, DataUsagePersistTaskResult::Cancelled)); + assert!(task.is_finished()); +} + +#[tokio::test(start_paused = true)] +async fn data_usage_persist_wait_aborts_after_timeout() { + let ctx = CancellationToken::new(); + let mut task = AbortOnDropHandle::new(tokio::spawn(async { + std::future::pending::<()>().await; + DataUsagePersistOutcome::Saved + })); + + let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(30)).await; + + assert!(matches!(result, DataUsagePersistTaskResult::TimedOut)); + assert!(task.is_finished()); +} + +#[tokio::test(start_paused = true)] +async fn maintenance_feature_inspection_preserves_base_cycle_after_timeout() { + let ctx = CancellationToken::new(); + + let result = wait_for_maintenance_feature_inspection( + &ctx, + std::future::pending::(), + Duration::from_secs(30), + ) + .await; + + assert_eq!(result, MaintenanceInspectionAttempt::TimedOut); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn stable_maintenance_detection_preserves_base_cycle_after_timeout() { + let ctx = CancellationToken::new(); + + let (features, generation) = detect_stable_scanner_maintenance_features_with( + &ctx, + std::future::pending::, + Duration::from_secs(30), + ) + .await + .expect("timeout should preserve the scanner rather than stop it"); + + assert!(features.inspection_failed); + assert_eq!(generation, scanner_maintenance_generation()); + assert!(!scanner_clean_idle_backoff_enabled( + true, + true, + features, + &ScannerRuntimeConfig::default() + )); +} + +#[tokio::test(start_paused = true)] +async fn failed_maintenance_inspection_uses_bounded_retry_backoff() { + let failed = ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }; + let mut retry = ScannerMaintenanceInspectionRetry::from_features(failed, Instant::now()); + + assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL)); + assert!(!retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now())); + tokio::time::advance(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL).await; + assert!(retry.retry_due(failed, ScannerCycleWakeReason::Timer, Instant::now())); + assert!(!retry.retry_due(failed, ScannerCycleWakeReason::DirtyUsage, Instant::now())); + + retry.record_inspection(failed, Instant::now()); + assert_eq!( + retry.retry_interval(), + Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_BASE_INTERVAL.saturating_mul(2)) + ); + for _ in 0..8 { + retry.record_inspection(failed, Instant::now()); + } + assert_eq!(retry.retry_interval(), Some(MAINTENANCE_FEATURE_INSPECTION_RETRY_MAX_INTERVAL)); + + retry.record_inspection(ScannerMaintenanceFeatures::default(), Instant::now()); + assert_eq!(retry, ScannerMaintenanceInspectionRetry::default()); +} + +#[tokio::test] +async fn maintenance_feature_inspection_stops_on_cancellation() { + let ctx = CancellationToken::new(); + ctx.cancel(); + + let result = wait_for_maintenance_feature_inspection( + &ctx, + std::future::pending::(), + Duration::from_secs(30), + ) + .await; + + assert_eq!(result, MaintenanceInspectionAttempt::Cancelled); +} + +#[test] +#[serial] +fn test_cycle_interval_prefers_explicit_cycle_override() { + with_var(ENV_SCANNER_SPEED, Some("slowest"), || { + with_var(ENV_SCANNER_CYCLE, Some("42"), || { + assert_eq!(cycle_interval(), Duration::from_secs(42)); + }); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() { + let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); + + with_var(ENV_SCANNER_CYCLE, Some("42"), || { + assert_eq!(cycle_interval(), Duration::from_secs(42)); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_uses_scanner_default_speed_override_when_unconfigured() { + let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest); + + with_unset_scanner_timing_env(|| { + assert_eq!(cycle_interval(), Duration::from_secs(30 * 60)); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_prefers_explicit_speed_over_default_speed_override() { + let _guard = ScannerDefaultSpeedGuard::set(ScannerSpeed::Slowest); + + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset("MINIO_SCANNER_CYCLE", || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || { + with_var(ENV_SCANNER_SPEED, Some("fastest"), || { + assert_eq!(cycle_interval(), Duration::from_secs(1)); + }); + }); + }); + }); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() { + let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); + + with_unset_scanner_timing_env(|| { + assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS)); + }); +} + +#[test] +fn test_single_disk_default_speed_uses_regular_scanner_default() { + assert_eq!(single_disk_default_speed(), ScannerSpeed::Default); +} + +#[test] +fn test_maintenance_feature_inspection_is_bounded_and_conservative() { + assert_eq!(maintenance_inspection_decision(1, 1, 1), MaintenanceInspectionDecision::Accept); + assert_eq!(maintenance_inspection_decision(1, 2, 1), MaintenanceInspectionDecision::Retry); + assert_eq!( + maintenance_inspection_decision(1, 2, MAX_MAINTENANCE_FEATURE_INSPECTION_ATTEMPTS), + MaintenanceInspectionDecision::PreserveBaseCycle + ); +} + +#[test] +fn clean_idle_backoff_grows_to_cap() { + let base_interval = Duration::from_secs(60); + let max_interval = CLEAN_IDLE_MAX_INTERVAL; + let mut backoff = ScannerCleanIdleBackoff::default(); + + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(60)); + for expected_secs in [ + 120, 240, 480, 960, 1_920, 3_840, 7_680, 15_360, 30_720, 61_440, 86_400, 86_400, + ] { + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!( + backoff.effective_interval(base_interval, max_interval, true), + Duration::from_secs(expected_secs) + ); + } +} + +#[test] +fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() { + let mut backoff = ScannerRetryBackoff::default(); + assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None); + + for expected in [5, 10, 20, 40, 80, 160, 320] { + backoff.record_retryable_cycle(true); + assert_eq!( + backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), + Some(Duration::from_secs(expected)) + ); + } + for _ in 0..20 { + backoff.record_retryable_cycle(true); + } + assert_eq!( + backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), + Some(Duration::from_secs(24 * 60 * 60)) + ); + + backoff.record_retryable_cycle(false); + assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None); +} + +#[test] +fn superseded_retry_backoff_respects_a_faster_configured_cycle() { + let mut backoff = ScannerRetryBackoff::default(); + backoff.record_retryable_cycle(true); + + // A configured cycle shorter than the base still wins: retrying sooner + // than the operator's own cadence buys nothing. + assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(3))); + backoff.record_retryable_cycle(true); + assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(6))); +} + +#[test] +fn superseded_retry_backoff_grows_from_the_default_cycle() { + let mut backoff = ScannerRetryBackoff::default(); + // The first race after a write burst retries in seconds, not a whole + // cycle, while repeated supersedes still climb toward the cap. + for expected in [5, 10, 20, 40] { + backoff.record_retryable_cycle(true); + assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(expected))); + } +} + +#[test] +fn scanner_cycle_wait_plan_drives_growth_resets_and_bitrot_cap() { + let runtime_config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + bitrot_cycle: None, + ..Default::default() + }; + let mut clean_idle_backoff = ScannerCleanIdleBackoff::default(); + + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); + assert_eq!(plan.delay, Duration::from_secs(60)); + + for expected in [120, 240] { + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); + assert_eq!(plan.delay, Duration::from_secs(expected)); + } + + for (wake_reason, outcome, dirty_work_observed) in [ + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Completed, true), + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Partial, false), + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::Failed, false), + (ScannerCycleWakeReason::Timer, ScannerCycleOutcome::CompletedWithPendingMaintenance, false), + (ScannerCycleWakeReason::DirtyUsage, ScannerCycleOutcome::Completed, false), + ] { + record_scanner_cycle_result(&mut clean_idle_backoff, &runtime_config, true, wake_reason, outcome, dirty_work_observed); + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, std::convert::identity); + assert_eq!(plan.effective_interval, Duration::from_secs(60)); + assert_eq!(plan.delay, Duration::from_secs(60)); + + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + } + + clean_idle_backoff.reset(); + for _ in 0..32 { + record_scanner_cycle_result( + &mut clean_idle_backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + } + let plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, true, |interval| interval.mul_f64(1.1)); + assert_eq!(plan.effective_interval, CLEAN_IDLE_MAX_INTERVAL); + assert!(plan.delay < CLEAN_IDLE_MAX_INTERVAL); + assert_eq!( + plan.delay, + CLEAN_IDLE_MAX_INTERVAL.saturating_sub(CLEAN_IDLE_MAX_INTERVAL.mul_f64(1.1) - CLEAN_IDLE_MAX_INTERVAL) + ); +} + +#[test] +#[serial] +fn scanner_cycle_schedule_status_reports_effective_backoff() { + record_scanner_cycle_schedule(Duration::from_millis(86_400_001), true, 2_048, true, 7); + + let status = scanner_cycle_schedule_status(); + + assert_eq!(status.effective_interval_seconds, 86_401); + assert!(status.clean_idle_backoff_enabled); + assert_eq!(status.clean_idle_backoff_multiplier, 2_048); + assert!(status.superseded_retry_backoff_enabled); + assert_eq!(status.superseded_cycles, 7); + + reset_scanner_cycle_schedule(); + let status = scanner_cycle_schedule_status(); + assert_eq!(status.effective_interval_seconds, 0); + assert!(!status.clean_idle_backoff_enabled); + assert_eq!(status.clean_idle_backoff_multiplier, 1); + assert!(!status.superseded_retry_backoff_enabled); + assert_eq!(status.superseded_cycles, 0); +} + +#[test] +fn clean_idle_backoff_resets_for_non_idle_work() { + let base_interval = Duration::from_secs(60); + let max_interval = CLEAN_IDLE_MAX_INTERVAL; + let mut backoff = ScannerCleanIdleBackoff::default(); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240)); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::DirtyUsage, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Partial, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Failed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + true, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::CompletedWithPendingMaintenance, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); +} + +#[test] +fn test_dirty_work_is_observed_across_cycle_waits() { + assert!(scanner_cycle_observed_dirty_work(true, 7, 7)); + assert!(scanner_cycle_observed_dirty_work(false, 7, 8)); + assert!(!scanner_cycle_observed_dirty_work(false, 7, 7)); +} + +#[test] +fn clean_idle_backoff_never_shortens_base_interval() { + let base_interval = Duration::from_secs(48 * 60 * 60); + let mut backoff = ScannerCleanIdleBackoff::default(); + + backoff.record_cycle( + base_interval, + CLEAN_IDLE_MAX_INTERVAL, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + + assert_eq!(backoff.effective_interval(base_interval, CLEAN_IDLE_MAX_INTERVAL, true), base_interval); +} + +#[test] +fn clean_idle_backoff_resets_while_disabled() { + let base_interval = Duration::from_secs(60); + let max_interval = CLEAN_IDLE_MAX_INTERVAL; + let mut backoff = ScannerCleanIdleBackoff::default(); + + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + backoff.record_cycle( + base_interval, + max_interval, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), Duration::from_secs(240)); + + backoff.record_cycle( + base_interval, + max_interval, + false, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + false, + ); + + assert_eq!(backoff.effective_interval(base_interval, max_interval, false), base_interval); + assert_eq!(backoff.effective_interval(base_interval, max_interval, true), base_interval); +} + +#[test] +fn clean_idle_backoff_policy_preserves_explicit_and_maintenance_cycles() { + let no_features = ScannerMaintenanceFeatures::default(); + let default_config = ScannerRuntimeConfig::default(); + assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &default_config)); + assert!(!scanner_clean_idle_backoff_enabled(false, true, no_features, &default_config)); + assert!(!scanner_clean_idle_backoff_enabled(true, false, no_features, &default_config)); + + for source in [ScannerRuntimeConfigSource::Env, ScannerRuntimeConfigSource::Config] { + let mut config = default_config.clone(); + config.cycle_interval_source = source; + assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &config)); + } + + for source in [ + ScannerRuntimeConfigSource::Env, + ScannerRuntimeConfigSource::Config, + ScannerRuntimeConfigSource::ScannerCompatConfig, + ] { + let mut explicit_bitrot_config = default_config.clone(); + explicit_bitrot_config.bitrot_cycle = Some(Duration::from_secs(60 * 60)); + explicit_bitrot_config.bitrot_cycle_source = source; + assert!(!scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config)); + + explicit_bitrot_config.bitrot_cycle = None; + assert!(scanner_clean_idle_backoff_enabled(true, true, no_features, &explicit_bitrot_config)); + } + + for features in [ + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + replication: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }, + ] { + assert!(!scanner_clean_idle_backoff_enabled(true, true, features, &default_config)); + } +} + +#[test] +fn clean_idle_backoff_requires_activity_probes() { + let default_config = ScannerRuntimeConfig::default(); + let no_features = ScannerMaintenanceFeatures::default(); + assert!(scanner_activity_probe_required(true, false, no_features, &default_config)); + assert!(!scanner_activity_probe_required(false, false, no_features, &default_config)); + assert!(!scanner_activity_probe_required(true, true, no_features, &default_config)); + + let mut explicit_cycle = default_config.clone(); + explicit_cycle.cycle_interval_source = ScannerRuntimeConfigSource::Env; + assert!(!scanner_activity_probe_required(true, false, no_features, &explicit_cycle)); + + let lifecycle = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + assert!(!scanner_activity_probe_required(true, false, lifecycle, &default_config)); +} + +#[test] +#[serial] +fn clean_idle_cap_preserves_default_bitrot_coverage_window() { + let config = ScannerRuntimeConfig { + bitrot_cycle: Some(Duration::from_secs(30 * 24 * 60 * 60)), + bitrot_cycle_source: ScannerRuntimeConfigSource::Default, + ..Default::default() + }; + + with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || { + let max_interval = scanner_clean_idle_max_interval(Duration::from_secs(60), &config); + assert_eq!(max_interval, Duration::from_millis(2_531_250)); + let positive_jitter = max_interval.mul_f64(1.1); + let actual_delay = cap_clean_idle_cycle_delay(positive_jitter, max_interval, true); + assert!(actual_delay < max_interval); + assert_eq!(actual_delay, max_interval.saturating_sub(positive_jitter - max_interval)); + assert!(actual_delay.saturating_mul(1024) <= config.bitrot_cycle.expect("bitrot cycle should be configured")); + }); +} + +#[test] +fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() { + let config = ScannerRuntimeConfig { + bitrot_cycle: None, + ..Default::default() + }; + + assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), CLEAN_IDLE_MAX_INTERVAL); +} + +#[test] +#[serial] +fn clean_idle_cap_never_shortens_the_base_cycle() { + let config = ScannerRuntimeConfig { + bitrot_cycle: Some(Duration::from_secs(60)), + bitrot_cycle_source: ScannerRuntimeConfigSource::Default, + ..Default::default() + }; + + with_var("RUSTFS_HEAL_OBJECT_SELECT_PROB", Some("1024"), || { + assert_eq!(scanner_clean_idle_max_interval(Duration::from_secs(60), &config), Duration::from_secs(60)); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() { + let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); + + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset("MINIO_SCANNER_CYCLE", || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || { + with_var(ENV_SCANNER_SPEED, Some("slowest"), || { + assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS)); + }); + }); + }); + }); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() { + let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS); + + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset("MINIO_SCANNER_CYCLE", || { + with_var(ENV_SCANNER_START_DELAY_SECS, Some("120"), || { + assert_eq!(cycle_interval(), Duration::from_secs(120)); + }); + }); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_supports_minio_speed_alias() { + with_var_unset(ENV_SCANNER_SPEED, || { + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { + with_var("MINIO_SCANNER_SPEED", Some("slowest"), || { + assert_eq!(cycle_interval(), Duration::from_secs(30 * 60)); + }); + }); + }); + }); +} + +#[test] +#[serial] +fn test_cycle_interval_supports_minio_cycle_alias() { + with_var_unset(ENV_SCANNER_CYCLE, || { + with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { + with_var("MINIO_SCANNER_CYCLE", Some("90"), || { + assert_eq!(cycle_interval(), Duration::from_secs(90)); + }); + }); + }); +} + +#[test] +fn test_randomized_cycle_delay_handles_small_start_delay() { + // 0 is treated as minimum 1 second before jitter, with lower bound preserved. + let delay = randomized_cycle_delay_for(Duration::from_secs(0)); + assert!(delay >= Duration::from_secs(1), "expected delay >= 1s"); + assert!(delay < Duration::from_secs(2), "expected delay < 2s"); +} + +#[tokio::test] +#[serial] +async fn test_wait_for_next_scanner_cycle_wakes_for_dirty_usage() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + + let ctx = CancellationToken::new(); + let dirty_generation = crate::scanner_io::dirty_usage_generation(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + Some(dirty_generation), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + crate::scanner_io::record_dirty_usage_bucket("photos"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("dirty usage should wake scanner before timer"); + + assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial] +async fn test_wait_for_next_scanner_cycle_sees_unattempted_dirty_usage() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let dirty_generation = crate::scanner_io::dirty_usage_generation(); + crate::scanner_io::record_dirty_usage_bucket("photos"); + + let ctx = CancellationToken::new(); + let reason = wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + Some(dirty_generation), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn test_wait_for_next_scanner_cycle_retries_stable_dirty_usage_on_timer() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_generation = crate::scanner_io::dirty_usage_generation(); + let ctx = CancellationToken::new(); + let wait = wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + Some(dirty_generation), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + ); + + let reason = wait.await; + + assert_eq!(reason, ScannerCycleWakeReason::Timer); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn test_wait_for_next_scanner_cycle_can_defer_dirty_wakes_until_timer() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let wait = wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + None, + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + ); + + crate::scanner_io::record_dirty_usage_bucket("photos"); + assert_eq!(wait.await, ScannerCycleWakeReason::Timer); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial] +async fn test_wait_for_next_scanner_cycle_wakes_for_repeated_dirty_bucket() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + crate::scanner_io::record_dirty_usage_bucket("photos"); + let dirty_generation = crate::scanner_io::dirty_usage_generation(); + let ctx = CancellationToken::new(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + Some(dirty_generation), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + crate::scanner_io::record_dirty_usage_bucket("photos"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("a newer mutation of an already-dirty bucket should wake scanner"); + + assert_eq!(reason, ScannerCycleWakeReason::DirtyUsage); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial] +async fn test_wait_for_next_scanner_cycle_reschedules_for_runtime_config() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let observed_generation = crate::runtime_config::scanner_runtime_config_generation(); + let ctx = CancellationToken::new(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + Some(crate::scanner_io::dirty_usage_generation()), + observed_generation, + crate::scanner_io::scanner_maintenance_generation(), + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + let mut config = rustfs_config::server_config::Config::new(); + config.set_defaults(); + crate::runtime_config::apply_scanner_runtime_config(&config).expect("default scanner config should apply"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("runtime config should wake scanner before timer"); + + assert_eq!(reason, ScannerCycleWakeReason::RuntimeConfig); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial] +async fn test_wait_for_next_scanner_cycle_reschedules_for_maintenance_change() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let observed_generation = crate::scanner_io::scanner_maintenance_generation(); + let ctx = CancellationToken::new(); + let mut wait = Box::pin(wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + Some(crate::scanner_io::dirty_usage_generation()), + crate::runtime_config::scanner_runtime_config_generation(), + observed_generation, + || false, + )); + assert!(matches!(futures::poll!(&mut wait), Poll::Pending)); + + crate::scanner_io::record_scanner_maintenance_change("photos"); + let reason = tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("maintenance change should wake scanner before timer"); + + assert_eq!(reason, ScannerCycleWakeReason::MaintenanceConfig); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +async fn test_wait_for_next_scanner_cycle_stops_after_leader_lock_loss() { + let ctx = CancellationToken::new(); + let reason = wait_for_next_scanner_cycle( + &ctx, + Duration::from_secs(60), + Some(crate::scanner_io::dirty_usage_generation()), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + || true, + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost); +} + +fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity { + ScannerNodeActivity { + instance_id: epoch.to_string(), + namespace_generation, + maintenance_generation, + protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION, + topology_digest: [3; 32], + data_movement_active: false, + dirty_usage_generation: 5, + dirty_usage_pending: false, + } +} + +#[test] +fn scanner_activity_snapshot_digest_fences_storage_topology() { + let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut changed = first.clone(); + changed.get_mut("node-2").expect("node should exist").topology_digest = [4; 32]; + + assert_ne!(scanner_activity_snapshot_digest(&first), scanner_activity_snapshot_digest(&changed)); +} + +#[test] +fn scanner_activity_snapshot_digest_fences_peer_protocol_upgrades() { + let legacy = BTreeMap::from([( + "node-2".to_string(), + ScannerNodeActivity { + protocol_version: SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, + ..scanner_node_activity("epoch-a", 7, 3) + }, + )]); + let previous = BTreeMap::from([( + "node-2".to_string(), + ScannerNodeActivity { + protocol_version: SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, + ..scanner_node_activity("epoch-a", 7, 3) + }, + )]); + let current = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + + assert_ne!(scanner_activity_snapshot_digest(&legacy), scanner_activity_snapshot_digest(¤t)); + assert_ne!(scanner_activity_snapshot_digest(&previous), scanner_activity_snapshot_digest(¤t)); +} + +#[test] +fn scanner_activity_snapshot_fences_data_movement() { + let idle = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut moving = idle.clone(); + moving.get_mut("node-2").expect("node should exist").data_movement_active = true; + + assert!(scanner_activity_allows_usage_publication(&idle)); + assert!(!scanner_activity_allows_usage_publication(&moving)); + assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving)); +} + +#[test] +fn scanner_activity_snapshot_digest_fences_dirty_usage_state() { + let clean = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let pending = BTreeMap::from([( + "node-2".to_string(), + ScannerNodeActivity { + dirty_usage_generation: 6, + dirty_usage_pending: true, + ..scanner_node_activity("epoch-a", 7, 3) + }, + )]); + + assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending)); +} + +#[test] +fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() { + let snapshot = BTreeMap::from([ + ( + LOCAL_SCANNER_ACTIVITY_NODE.to_string(), + ScannerNodeActivity { + dirty_usage_generation: 7, + dirty_usage_pending: true, + ..scanner_node_activity("epoch-local", 7, 3) + }, + ), + ("node-2".to_string(), scanner_node_activity("epoch-clean", 7, 3)), + ( + "node-3".to_string(), + ScannerNodeActivity { + dirty_usage_generation: 11, + dirty_usage_pending: true, + ..scanner_node_activity("epoch-dirty", 7, 3) + }, + ), + ]); + + assert_eq!( + scanner_dirty_usage_acknowledgements(&snapshot), + vec![ScannerDirtyUsageAcknowledgement { + host: "node-3".to_string(), + instance_id: "epoch-dirty".to_string(), + generation: 11, + }] + ); +} + +#[test] +fn scanner_activity_rejects_one_process_claimed_by_multiple_hosts() { + let mut instances = BTreeMap::new(); + record_scanner_activity_instance(&mut instances, "node-1", "0123456789abcdef0123456789abcdef") + .expect("first host should establish the instance identity"); + let err = record_scanner_activity_instance(&mut instances, "node-2", "0123456789abcdef0123456789abcdef") + .expect_err("a process identity must not represent two cluster nodes"); + + assert!(err.contains("node-1 and node-2")); +} + +#[test] +fn scanner_activity_observation_requires_a_complete_baseline() { + let mut seen = None; + let first = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first.clone())); + assert_eq!(observation, ScannerActivityObservation::Unverified); + assert!(error.is_none()); + + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(first)); + assert_eq!(observation, ScannerActivityObservation::Unchanged); + assert!(error.is_none()); + + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(changed)); + assert_eq!(observation, ScannerActivityObservation::Changed); + assert!(error.is_none()); + + let restarted = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 8, 0))]); + let (observation, error) = apply_scanner_activity_probe_result(&mut seen, Ok(restarted)); + assert_eq!(observation, ScannerActivityObservation::Changed); + assert!(error.is_none()); + + let (observation, error) = + apply_scanner_activity_probe_result(&mut seen, Err("peer does not support activity probes".to_string())); + assert_eq!(observation, ScannerActivityObservation::Unverified); + assert_eq!(error.as_deref(), Some("peer does not support activity probes")); + assert!(seen.is_none()); +} + +#[test] +fn remote_maintenance_change_is_distinct_from_namespace_activity() { + let previous = BTreeMap::from([ + (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)), + ("node-2".to_string(), scanner_node_activity("remote", 7, 3)), + ]); + let remote_maintenance_changed = BTreeMap::from([ + (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 2)), + ("node-2".to_string(), scanner_node_activity("remote", 7, 4)), + ]); + assert_eq!( + compare_scanner_activity(&previous, &remote_maintenance_changed), + ScannerActivityObservation::MaintenanceChanged + ); + + let local_maintenance_changed = BTreeMap::from([ + (LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("local", 5, 3)), + ("node-2".to_string(), scanner_node_activity("remote", 7, 3)), + ]); + assert_eq!( + compare_scanner_activity(&previous, &local_maintenance_changed), + ScannerActivityObservation::Changed + ); +} + +#[test] +fn local_maintenance_wakeup_releases_a_remote_maintenance_block() { + let blocked = scanner_activity_backoff_blocked_after_wake(false, ScannerCycleWakeReason::ClusterMaintenance); + assert!(blocked); + + let unblocked = scanner_activity_backoff_blocked_after_wake(blocked, ScannerCycleWakeReason::MaintenanceConfig); + assert!(!unblocked); + assert!(scanner_activity_backoff_blocked_after_wake( + blocked, + ScannerCycleWakeReason::ClusterActivity + )); +} + +#[test] +fn scanner_activity_after_a_cycle_restores_the_base_interval() { + let runtime_config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let mut backoff = ScannerCleanIdleBackoff { interval_multiplier: 8 }; + + record_scanner_cycle_result( + &mut backoff, + &runtime_config, + true, + ScannerCycleWakeReason::Timer, + ScannerCycleOutcome::Completed, + scanner_activity_observed_work(ScannerActivityObservation::Changed), + ); + + let plan = scanner_cycle_wait_plan(&runtime_config, backoff, true, std::convert::identity); + assert_eq!(plan.effective_interval, Duration::from_secs(60)); + assert_eq!(plan.delay, Duration::from_secs(60)); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn distributed_clean_idle_wait_wakes_at_base_interval_for_remote_activity() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + defer_cluster_activity: false, + }, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::ClusterActivity); + assert_eq!(seen, Some(changed)); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn superseded_retry_wait_defers_dirty_cluster_activity_until_timer() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: None, + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + defer_cluster_activity: true, + }, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::Timer); + assert_eq!(seen, Some(changed)); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn distributed_clean_idle_wait_blocks_backoff_for_unpropagated_maintenance() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 4))]); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + defer_cluster_activity: false, + }, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::ClusterMaintenance); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn distributed_clean_idle_wait_fails_closed_when_a_peer_is_unverifiable() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + defer_cluster_activity: false, + }, + || false, + || std::future::ready(Err("node-2 is unreachable".to_string())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::ClusterActivityUnavailable); + assert!(seen.is_none()); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn distributed_clean_idle_wait_keeps_the_extended_deadline_when_peers_are_clean() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let expected = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]); + let mut seen = Some(expected.clone()); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + defer_cluster_activity: false, + }, + || false, + || std::future::ready(Ok(expected.clone())), + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::Timer); + assert_eq!(seen, Some(expected)); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn scanner_activity_probe_wait_is_cancellation_aware() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let cancel = ctx.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(61)).await; + cancel.cancel(); + }); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + defer_cluster_activity: false, + }, + || false, + std::future::pending::>, + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::Cancelled); +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn scanner_activity_probe_wait_stops_after_leader_lock_loss() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let ctx = CancellationToken::new(); + let lock_lost = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let lose_lock = Arc::clone(&lock_lost); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(61)).await; + lose_lock.store(true, std::sync::atomic::Ordering::Release); + }); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + Duration::from_secs(120), + Some(Duration::from_secs(60)), + &mut seen, + ScannerCycleObservedGenerations { + dirty_usage: Some(crate::scanner_io::dirty_usage_generation()), + runtime_config: crate::runtime_config::scanner_runtime_config_generation(), + maintenance: crate::scanner_io::scanner_maintenance_generation(), + defer_cluster_activity: false, + }, + || lock_lost.load(std::sync::atomic::Ordering::Acquire), + std::future::pending::>, + ) + .await; + + assert_eq!(reason, ScannerCycleWakeReason::LeaderLockLost); +} + +#[test] +#[serial] +fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() { + with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { + let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle()); + assert_eq!(mode, HealScanMode::Deep); + }); +} + +#[test] +#[serial] +fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() { + with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { + let recent = Utc::now() - chrono::Duration::minutes(30); + let old = Utc::now() - chrono::Duration::hours(2); + + assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent), bitrot_scan_cycle()), HealScanMode::Normal); + assert_eq!(get_cycle_scan_mode(2048, 0, Some(old), bitrot_scan_cycle()), HealScanMode::Deep); + }); +} + +#[test] +#[serial] +fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() { + with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || { + assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal); + }); +} + +#[test] +#[serial] +fn test_background_heal_info_for_scan_start_marks_deep_active() { + let now = Utc::now(); + let info = + background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now, bitrot_scan_cycle()) + .expect("deep scan should update background heal info"); + + assert_eq!(info.current_scan_mode, HealScanMode::Deep); + assert_eq!(info.bitrot_start_cycle, 7); + assert_eq!(info.bitrot_start_time, Some(now)); +} + +#[test] +#[serial] +fn test_background_heal_info_for_scan_start_keeps_deep_window_start() { + with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || { + let started_at = Utc::now(); + let info = BackgroundHealInfo { + bitrot_start_time: Some(started_at), + bitrot_start_cycle: 7, + current_scan_mode: HealScanMode::Normal, + }; + + let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now(), bitrot_scan_cycle()) + .expect("deep scan should mark active status"); + + assert_eq!(info.current_scan_mode, HealScanMode::Deep); + assert_eq!(info.bitrot_start_cycle, 7); + assert_eq!(info.bitrot_start_time, Some(started_at)); + }); +} + +#[test] +fn test_background_heal_info_for_scan_complete_marks_deep_idle() { + let started_at = Utc::now(); + let info = BackgroundHealInfo { + bitrot_start_time: Some(started_at), + bitrot_start_cycle: 7, + current_scan_mode: HealScanMode::Deep, + }; + + let info = background_heal_info_for_scan_complete(info, HealScanMode::Deep) + .expect("completed deep scan should update background heal info"); + + assert_eq!(info.current_scan_mode, HealScanMode::Normal); + assert_eq!(info.bitrot_start_cycle, 7); + assert_eq!(info.bitrot_start_time, Some(started_at)); +} + +#[test] +fn test_background_heal_info_for_scan_complete_leaves_normal_scan_unchanged() { + let info = BackgroundHealInfo { + bitrot_start_time: Some(Utc::now()), + bitrot_start_cycle: 7, + current_scan_mode: HealScanMode::Normal, + }; + + assert!(background_heal_info_for_scan_complete(info, HealScanMode::Normal).is_none()); +} + +#[test] +fn test_background_heal_info_for_failed_scan_preserves_deep_mode() { + let info = BackgroundHealInfo { + bitrot_start_time: Some(Utc::now()), + bitrot_start_cycle: 7, + current_scan_mode: HealScanMode::Deep, + }; + + assert!(background_heal_info_for_scan_result(info, HealScanMode::Deep, false).is_none()); +} + +#[test] +fn test_retain_recent_cycle_completions_keeps_last_entries() { + let base = Utc::now(); + let keep = data_usage_update_dir_cycles() as usize; + let mut completed: Vec<_> = (0..keep + 2).map(|i| base + chrono::Duration::seconds(i as i64)).collect(); + + retain_recent_cycle_completions(&mut completed); + + assert_eq!(completed.len(), keep); + assert_eq!(completed.first().copied(), Some(base + chrono::Duration::seconds(2))); + assert_eq!(completed.last().copied(), Some(base + chrono::Duration::seconds((keep + 1) as i64))); +} diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs new file mode 100644 index 000000000..44ff2e8ec --- /dev/null +++ b/crates/scanner/src/scanner/usage_store.rs @@ -0,0 +1,483 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// Data-usage snapshot persistence: CAS store pipeline, epoch baselines, and observed-snapshot cleanup. +use super::*; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) enum DataUsagePersistOutcome { + #[default] + NoUpdate, + Current, + AlreadyDurable, + PriorCycleDurable, + Saved, + Failed, +} + +#[derive(Clone, Debug)] +pub(super) struct DataUsagePersistBaseline { + pub(super) data: Option, + pub(super) revision: DataUsageCacheRevision, +} + +#[derive(Debug)] +pub(super) enum DataUsagePersistTaskResult { + Completed(DataUsagePersistOutcome), + Cancelled, + TimedOut, + JoinFailed(tokio::task::JoinError), +} + +pub(super) async fn wait_for_data_usage_persist_task( + ctx: &CancellationToken, + task: &mut AbortOnDropHandle, + timeout: Duration, +) -> DataUsagePersistTaskResult { + tokio::select! { + biased; + result = &mut *task => match result { + Ok(outcome) => DataUsagePersistTaskResult::Completed(outcome), + Err(err) => DataUsagePersistTaskResult::JoinFailed(err), + }, + _ = ctx.cancelled() => { + task.abort(); + let _ = (&mut *task).await; + DataUsagePersistTaskResult::Cancelled + }, + _ = tokio::time::sleep(timeout) => { + task.abort(); + let _ = (&mut *task).await; + DataUsagePersistTaskResult::TimedOut + } + } +} + +#[instrument(skip(ctx, storeapi))] +pub async fn store_data_usage_in_backend( + ctx: CancellationToken, + storeapi: Arc, + receiver: mpsc::Receiver, +) { + let _ = store_data_usage_in_backend_with_outcome(ctx, storeapi, receiver).await; +} + +pub(super) async fn store_data_usage_in_backend_with_outcome( + ctx: CancellationToken, + storeapi: Arc, + receiver: mpsc::Receiver, +) -> DataUsagePersistOutcome { + store_data_usage_in_backend_with_outcome_for_epoch(ctx, storeapi, receiver, None).await +} + +pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch( + ctx: CancellationToken, + storeapi: Arc, + receiver: mpsc::Receiver, + leader_epoch: Option, +) -> DataUsagePersistOutcome { + store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(ctx, storeapi, receiver, leader_epoch, None).await +} + +pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline( + ctx: CancellationToken, + storeapi: Arc, + mut receiver: mpsc::Receiver, + leader_epoch: Option, + initial_baseline: Option, +) -> DataUsagePersistOutcome { + let mut outcome = DataUsagePersistOutcome::NoUpdate; + let mut next_baseline = initial_baseline; + + 'updates: while let Some(mut data_usage_info) = receiver.recv().await { + let _activity_guard = ScannerActivityGuard::new(); + if ctx.is_cancelled() { + break; + } + if let Some(leader_epoch) = leader_epoch { + data_usage_info.scanner_epoch = Some(leader_epoch); + } + let observational = data_usage_info.usage_snapshot_converged == Some(false); + let target_path = if observational { + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() + } else { + DATA_USAGE_OBJ_NAME_PATH.as_str() + }; + + if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() { + let authoritative_data = match next_baseline.as_ref() { + Some(baseline) => baseline.data.clone(), + None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { + Ok((data, _)) => data.map(Bytes::from), + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_load_failed", + error = %err, + "Scanner could not identify the authoritative baseline for an observation" + ); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + }, + }; + let authoritative = match authoritative_data.as_deref() { + Some(data) => match serde_json::from_slice::(data) { + Ok(info) => info, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_decode_failed", + error = %err, + "Scanner refused to publish an observation from an invalid authoritative baseline" + ); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + }, + None => DataUsageInfo::default(), + }; + data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity()); + } + + if !data_usage_info.is_complete_bucket_usage_snapshot() { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %target_path, + state = "reject_incomplete_snapshot", + "Scanner refused to persist an incomplete data usage snapshot" + ); + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + + let data = match serde_json::to_vec(&data_usage_info) { + Ok(data) => data, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %target_path, + state = "encode_failed", + error = %e, + "Scanner data usage encode failed" + ); + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::EncodeFailed); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + }; + let sha256hex = (!data.is_empty()).then(|| hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower)); + let data = Bytes::from(data); + let backup_due = !observational && data_usage_backup_due(&data_usage_info); + let mut cas_retry = 0usize; + let save_outcome = loop { + if ctx.is_cancelled() { + break 'updates; + } + + let baseline = if !observational && cas_retry == 0 { + next_baseline.take() + } else { + None + }; + let (existing_data, revision) = match baseline { + Some(baseline) => (baseline.data, baseline.revision), + None => match read_config_with_revision(storeapi.clone(), target_path).await { + Ok((data, revision)) => (data.map(Bytes::from), revision), + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %target_path, + state = "revision_load_failed", + error = %e, + "Scanner data usage revision load failed" + ); + break DataUsagePersistOutcome::Failed; + } + }, + }; + let existing = existing_data + .as_deref() + .and_then(|buf| serde_json::from_slice::(buf).ok()); + if cas_retry > 0 && data_usage_reintroduces_missing_bucket(&data_usage_info, existing.as_ref()) { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %target_path, + incoming_scanner_epoch = ?data_usage_info.scanner_epoch, + incoming_scanner_cycle = ?data_usage_info.scanner_cycle, + state = "skip_deleted_bucket_reintroduction", + "Scanner usage update skipped after a concurrent bucket removal" + ); + break DataUsagePersistOutcome::Current; + } + if let Some(existing) = existing.as_ref() { + if existing == &data_usage_info { + break DataUsagePersistOutcome::AlreadyDurable; + } + if existing.scanner_epoch.is_some() + && existing.scanner_epoch == data_usage_info.scanner_epoch + && existing.scanner_cycle.is_some() + && existing.scanner_cycle == data_usage_info.scanner_cycle + { + break DataUsagePersistOutcome::PriorCycleDurable; + } + if let Some(reason) = stale_data_usage_update_reason(&data_usage_info, existing, std::time::SystemTime::now()) { + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %target_path, + incoming_scanner_epoch = ?data_usage_info.scanner_epoch, + existing_scanner_epoch = ?existing.scanner_epoch, + incoming_scanner_cycle = ?data_usage_info.scanner_cycle, + existing_scanner_cycle = ?existing.scanner_cycle, + incoming_last_update = ?data_usage_info.last_update, + existing_last_update = ?existing.last_update, + reason = reason, + state = "skip_stale_update", + "Scanner stale data usage update skipped" + ); + break DataUsagePersistOutcome::Current; + } + } + if ctx.is_cancelled() { + break 'updates; + } + + let done_save = Metrics::time(Metric::SaveUsage); + let save_result = save_config_shared_with_preconditions( + storeapi.clone(), + target_path, + data.clone(), + sha256hex.clone(), + revision.preconditions(), + ) + .await; + done_save(); + + match save_result { + Ok(object_info) => { + if !observational { + next_baseline = object_info + .etag + .filter(|etag| !etag.is_empty()) + .map(|etag| DataUsagePersistBaseline { + data: Some(data.clone()), + revision: DataUsageCacheRevision::Etag(etag), + }); + } + break DataUsagePersistOutcome::Saved; + } + Err(EcstoreError::PreconditionFailed) if cas_retry < SCANNER_PERSIST_CAS_RETRIES => { + cas_retry += 1; + debug!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %target_path, + state = "conflict_retry", + retry = cas_retry, + "Scanner data usage CAS conflict will be reconciled" + ); + } + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %target_path, + state = if matches!(e, EcstoreError::PreconditionFailed) { + "conflict_retries_exhausted" + } else { + "save_failed" + }, + error = %e, + "Scanner data usage save failed" + ); + break DataUsagePersistOutcome::Failed; + } + } + }; + + match save_outcome { + DataUsagePersistOutcome::Current => { + if observational { + invalidate_admin_data_usage_snapshot_cache().await; + } else { + invalidate_data_usage_snapshot_cache().await; + } + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale); + outcome = DataUsagePersistOutcome::Current; + continue; + } + DataUsagePersistOutcome::AlreadyDurable => { + if observational { + invalidate_admin_data_usage_snapshot_cache().await; + } else { + cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + invalidate_data_usage_snapshot_cache().await; + replace_bucket_usage_memory_from_info(&data_usage_info).await; + } + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + outcome = DataUsagePersistOutcome::AlreadyDurable; + } + DataUsagePersistOutcome::PriorCycleDurable => { + if observational { + invalidate_admin_data_usage_snapshot_cache().await; + } else { + cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + invalidate_data_usage_snapshot_cache().await; + } + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + outcome = DataUsagePersistOutcome::PriorCycleDurable; + } + DataUsagePersistOutcome::Failed | DataUsagePersistOutcome::NoUpdate => { + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + DataUsagePersistOutcome::Saved => { + if observational { + invalidate_admin_data_usage_snapshot_cache().await; + } else { + cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + invalidate_data_usage_snapshot_cache().await; + replace_bucket_usage_memory_from_info(&data_usage_info).await; + } + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + outcome = DataUsagePersistOutcome::Saved; + } + } + + if backup_due { + let done_save = Metrics::time(Metric::SaveUsage); + if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()), + state = "backup_save_failed", + error = %e, + "Scanner data usage backup save failed" + ); + } + done_save(); + } + } + + outcome +} + +pub(super) async fn cleanup_observed_data_usage_snapshot( + storeapi: Arc, + authoritative: &DataUsageInfo, +) { + let (observed_data, revision) = + match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { + Ok((Some(data), revision)) => (data, revision), + Ok((None, _)) => return, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + state = "observed_cleanup_read_failed", + error = %err, + "Scanner could not inspect observational data usage snapshot before authoritative cleanup" + ); + return; + } + }; + let observed = match serde_json::from_slice::(&observed_data) { + Ok(observed) => observed, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + state = "observed_cleanup_decode_failed", + error = %err, + "Scanner refused to remove an invalid observational data usage snapshot after authoritative save" + ); + return; + } + }; + if observed_data_usage_is_newer(&observed, authoritative) { + return; + } + + let result = storeapi + .delete_config_object( + RUSTFS_META_BUCKET, + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + ScannerObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + http_preconditions: Some(revision.preconditions()), + ..Default::default() + }, + ) + .await; + + match result { + Ok(_) + | Err( + EcstoreError::FileNotFound + | EcstoreError::ConfigNotFound + | EcstoreError::ObjectNotFound(_, _) + | EcstoreError::PreconditionFailed, + ) => {} + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + state = "observed_cleanup_failed", + error = %err, + "Scanner could not remove stale observational data usage snapshot after authoritative save" + ); + } + } +}