From 2b9dcc646f4b6801519d03418af1a7c5ed5f5026 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 20 Aug 2026 19:12:07 +0800 Subject: [PATCH] refactor(heal): split manager.rs queue/scheduler/scan children (#6303) Split the 6723-line manager.rs (44% inline tests) into a canonical manager.rs + manager/ module tree with zero behavior change: - manager.rs (~1830): HealManager and HealState, HealConfig, task report/snapshot types, overlap policy, admission classification and queue admission, submit paths, task-state queries, and the statistics surface - manager/queue.rs (~450): the priority heal queue, its per-key dedup index, and the queue bookkeeping structs - manager/scheduler.rs (~620): start_scheduler and the process_heal_queue consumption loop with its skip/metric helpers - manager/auto_scan.rs (~550): the automatic disk scanner - manager/unclean_shutdown.rs (~390): unclean-shutdown recovery and its durable replacement-intent helpers - manager/tests.rs (~2970): the inline test module as a child module All module paths are unchanged. The queue structs' fields and the cross-module helpers gain pub(super), whose scope equals the old single-module privacy domain; HealManager's private fields stay in the root and remain reachable from child impl blocks. Code is moved verbatim apart from those markers, heal-level super:: path fixes for the unclean-shutdown move, per-module import headers, and rustfmt re-wraps. The logging-guardrail rule for the manager demote_to_debug_when! count now sums manager.rs with its manager/*.rs children, since one scheduler site moved with process_heal_queue; the >= 6 threshold is unchanged and the forbidden admission info!/warn! pattern check keeps targeting the root admission code. Co-authored-by: heihutu --- crates/heal/src/heal/manager.rs | 4908 +---------------- crates/heal/src/heal/manager/auto_scan.rs | 555 ++ crates/heal/src/heal/manager/queue.rs | 452 ++ crates/heal/src/heal/manager/scheduler.rs | 618 +++ crates/heal/src/heal/manager/tests.rs | 2971 ++++++++++ .../heal/src/heal/manager/unclean_shutdown.rs | 390 ++ scripts/check_logging_guardrails.sh | 6 +- 7 files changed, 4998 insertions(+), 4902 deletions(-) create mode 100644 crates/heal/src/heal/manager/auto_scan.rs create mode 100644 crates/heal/src/heal/manager/queue.rs create mode 100644 crates/heal/src/heal/manager/scheduler.rs create mode 100644 crates/heal/src/heal/manager/tests.rs create mode 100644 crates/heal/src/heal/manager/unclean_shutdown.rs diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 5afa3ebc8..7c5c535b3 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -56,36 +56,6 @@ const LEGACY_ROOT_HEAL_PATH: &str = "."; const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3; const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30); -fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bool { - state.replacement_generation.as_deref() == Some(task_id) - && !state.replacement_targets.is_empty() - && ((!state.completed - && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) - && state.retry_count >= state.max_retries) - || (state.completed - && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending))) -} - -fn replacement_discovery_error_is_expected_for_deferred_endpoint( - error: &Error, - endpoint: &str, - deferred_replacement_endpoints: &HashSet, -) -> bool { - matches!(error, Error::Disk(DiskError::UnformattedDisk)) && deferred_replacement_endpoints.contains(endpoint) -} - -fn unblock_replacement_recovery_sets_after_validation( - blocked_sets: &mut HashSet, - retry_succeeded: HashSet, - retry_failed: &HashSet, -) { - for set_disk_id in retry_succeeded { - if !retry_failed.contains(&set_disk_id) { - blocked_sets.remove(&set_disk_id); - } - } -} - // Admission/scheduler outcomes for per-object requests (Object/Metadata/ // ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner // recovery loops submit those per object, so a full queue or a retry storm @@ -143,112 +113,6 @@ async fn pause_duplicate_admission_after_active_lock(request_id: &str) { type WorkloadSnapshotProviderRef = Arc; -/// Per-key bookkeeping for the queued-request dedup index: how many queued -/// requests hold the key, and the id of the first request that opened it — -/// the O(1) stand-in for the former heap scan when a merge receipt needs to -/// name a queued representative. -#[derive(Debug)] -struct DedupKeyEntry { - refcount: usize, - representative_request_id: String, -} - -/// Priority queue wrapper for heal requests -/// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items -#[derive(Debug)] -struct PriorityHealQueue { - /// Heap of (priority, sequence, request) tuples - heap: BinaryHeap, - /// Sequence counter for FIFO ordering within same priority - sequence: u64, - /// Deduplication index for queued requests - dedup_keys: HashMap, -} - -/// Wrapper for heap items to implement proper ordering -#[derive(Debug)] -struct PriorityQueueItem { - priority: HealPriority, - sequence: u64, - request: HealRequest, -} - -impl Eq for PriorityQueueItem {} - -impl PartialEq for PriorityQueueItem { - fn eq(&self, other: &Self) -> bool { - self.priority == other.priority && self.sequence == other.sequence - } -} - -impl Ord for PriorityQueueItem { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // First compare by priority (higher priority first) - match self.priority.cmp(&other.priority) { - std::cmp::Ordering::Equal => { - // If priorities are equal, use sequence for FIFO (lower sequence first) - other.sequence.cmp(&self.sequence) - } - ordering => ordering, - } - } -} - -impl PartialOrd for PriorityQueueItem { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum QueuePushOutcome { - Accepted, - Merged, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct ForegroundPressure { - class: WorkloadClass, - usage_pct: usize, - threshold_pct: usize, -} - -impl ForegroundPressure { - const fn reason(self) -> &'static str { - match self.class { - WorkloadClass::ForegroundRead => "foreground_read_pressure", - WorkloadClass::ForegroundWrite => "foreground_write_pressure", - _ => "foreground_pressure", - } - } -} - -#[derive(Debug, Clone)] -struct CompletedHealStatus { - heal_type: HealType, - status: HealTaskStatus, - result_items_truncated: bool, - completed_at: SystemTime, - /// Sequence-stamped retained window, archived with the completion so - /// incremental consumers keep their cursor across the transition (HS-06). - /// The un-stamped legacy view is derived from it on demand. - seqed_items: Vec<(u64, HealResultItem)>, - next_seq: u64, - min_seq: u64, -} - -#[derive(Debug, Clone)] -struct HealTaskAlias { - task_id: String, -} - -#[derive(Debug, Clone)] -struct RetryingHeal { - request: HealRequest, - error: String, - cancel_token: CancellationToken, -} - #[derive(Debug, Clone)] pub struct HealTaskReport { pub status: HealTaskStatus, @@ -379,337 +243,6 @@ fn usize_to_u64_saturated(value: usize) -> u64 { u64::try_from(value).unwrap_or(u64::MAX) } -impl PriorityHealQueue { - fn new() -> Self { - Self { - heap: BinaryHeap::new(), - sequence: 0, - dedup_keys: HashMap::new(), - } - } - - fn len(&self) -> usize { - self.heap.len() - } - - fn pop_next(&mut self) -> Option { - self.heap.pop().map(|item| { - let key = Self::make_dedup_key(&item.request); - Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); - item.request - }) - } - - fn is_empty(&self) -> bool { - self.heap.is_empty() - } - - fn push(&mut self, request: HealRequest) -> QueuePushOutcome { - let key = Self::make_dedup_key(&request); - - // Check for duplicates unless the caller explicitly forces admission. - if self.dedup_keys.contains_key(&key) && !request.force_start { - return QueuePushOutcome::Merged; - } - // Track dedup keys for both normal and forced requests so queued forced work - // also reserves the dedup key for later non-forced duplicates. The first - // request that opens the key becomes the named representative for merge - // receipts (taken before `request` moves into the heap). - self.dedup_keys - .entry(key) - .or_insert_with(|| DedupKeyEntry { - refcount: 0, - representative_request_id: request.id.clone(), - }) - .refcount += 1; - self.sequence += 1; - self.heap.push(PriorityQueueItem { - priority: request.priority, - sequence: self.sequence, - request, - }); - QueuePushOutcome::Accepted - } - - fn can_displace_lower_priority(&self, priority: HealPriority) -> bool { - self.heap.iter().any(|item| item.priority < priority) - } - - fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option { - let mut retained = BinaryHeap::new(); - let mut displaced: Option = None; - - while let Some(item) = self.heap.pop() { - if item.priority < request.priority { - let should_displace = displaced - .as_ref() - .map(|current| { - item.priority < current.priority - || (item.priority == current.priority && item.sequence > current.sequence) - }) - .unwrap_or(true); - if should_displace { - if let Some(current) = displaced.replace(item) { - retained.push(current); - } - } else { - retained.push(item); - } - } else { - retained.push(item); - } - } - - self.heap = retained; - - let displaced = displaced.map(|item| { - let key = Self::make_dedup_key(&item.request); - Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); - self.refresh_dedup_representative(&key); - item.request - }); - - if displaced.is_some() { - // The enqueue side effect must run in ALL builds. Do NOT fold `self.push(request)` - // into `debug_assert_eq!` — in release builds (`debug_assertions` off) the whole - // macro, including its argument expression, is compiled out, which would silently - // drop the new high-priority request after having already evicted a queued item. - let outcome = self.push(request); - debug_assert_eq!(outcome, QueuePushOutcome::Accepted); - } - - displaced - } - - /// Get statistics about queue contents by priority - fn get_priority_stats(&self) -> HashMap { - let mut stats = HashMap::new(); - for item in &self.heap { - *stats.entry(item.priority).or_insert(0) += 1; - } - stats - } - - fn operation_counts(&self) -> (HealPriorityCounts, HealSourceCounts) { - let mut priority = HealPriorityCounts::default(); - let mut source = HealSourceCounts::default(); - for item in &self.heap { - priority.increment(item.request.priority); - source.increment(item.request.source); - } - (priority, source) - } - - #[cfg(test)] - fn pop(&mut self) -> Option { - self.heap.pop().map(|item| { - let key = Self::make_dedup_key(&item.request); - Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); - item.request - }) - } - - #[cfg(test)] - fn pop_runnable(&mut self, can_run: F) -> Option - where - F: Fn(&HealRequest) -> bool, - { - self.pop_runnable_with_skips(can_run, |_| None).0 - } - - fn pop_runnable_with_skips(&mut self, can_run: F, skip_label: G) -> (Option, Vec) - where - F: Fn(&HealRequest) -> bool, - G: Fn(&HealRequest) -> Option, - { - let mut deferred = Vec::new(); - let mut selected = None; - let mut skipped = Vec::new(); - - while let Some(item) = self.heap.pop() { - if can_run(&item.request) { - selected = Some(item); - break; - } - if let Some(label) = skip_label(&item.request) { - skipped.push(label); - } - deferred.push(item); - } - - for item in deferred { - self.heap.push(item); - } - - ( - selected.map(|item| { - let key = Self::make_dedup_key(&item.request); - Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); - item.request - }), - skipped, - ) - } - - /// Create a deduplication key from a heal request - fn make_dedup_key(request: &HealRequest) -> String { - Self::make_dedup_key_for_type(&request.heal_type) - } - - fn make_dedup_key_for_type(heal_type: &HealType) -> String { - match heal_type { - HealType::Cluster => "cluster".to_string(), - HealType::Object { - bucket, - object, - version_id, - } => { - format!("object:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or("")) - } - HealType::Bucket { bucket } => { - format!("bucket:{bucket}") - } - HealType::Prefix { bucket, prefix } => { - format!("prefix:{bucket}/{prefix}") - } - HealType::ErasureSet { set_disk_id, .. } => { - format!("erasure_set:{set_disk_id}") - } - HealType::Metadata { bucket, object } => { - format!("metadata:{bucket}:{object}") - } - HealType::ECDecode { - bucket, - object, - version_id, - } => { - format!("ecdecode:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or("")) - } - } - } - - fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap, key: &str) { - if let Some(entry) = dedup_keys.get_mut(key) { - if entry.refcount <= 1 { - dedup_keys.remove(key); - } else { - entry.refcount -= 1; - } - } - } - /// Check if an erasure set heal request for a specific set_disk_id exists - fn contains_erasure_set(&self, set_disk_id: &str) -> bool { - let key = format!("erasure_set:{set_disk_id}"); - self.dedup_keys.contains_key(&key) - } - - /// Iterate queued requests (used by the admin overlap check). - fn requests(&self) -> impl Iterator { - self.heap.iter().map(|item| &item.request) - } - - fn contains_request_id(&self, request_id: &str) -> bool { - self.heap.iter().any(|item| item.request.id == request_id) - } - - fn contains_request_id_matching_path(&self, request_id: &str, heal_path: &str) -> bool { - self.heap - .iter() - .any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path)) - } - - fn queued_request_id_for_dedup_key(&self, key: &str) -> Option<&str> { - self.dedup_keys.get(key).map(|entry| entry.representative_request_id.as_str()) - } - - /// Re-elect the representative for `key` from the queue entries holding - /// it. Needed after a holder leaves the queue *without* becoming active - /// (canceled by id, or displaced): the former opener may be the request - /// that just left, and a merge receipt must never name an id that - /// resolves nowhere. The scheduler pop path does not need this — the - /// popped request surfaces in `active_heals` under the same id and the - /// duplicate pre-check consults active heals before the queue. No-op for - /// released keys; the survivor scan only runs when a key still has - /// holders, which under forced duplicates is the rare admin path. - fn refresh_dedup_representative(&mut self, key: &str) { - if !self.dedup_keys.contains_key(key) { - return; - } - if let Some(id) = self - .heap - .iter() - .find(|item| Self::make_dedup_key(&item.request) == key) - .map(|item| item.request.id.clone()) - && let Some(entry) = self.dedup_keys.get_mut(key) - { - entry.representative_request_id = id; - } - } - - fn contains_matching(&self, mut matches: F) -> bool - where - F: FnMut(&HealRequest) -> bool, - { - self.heap.iter().any(|item| matches(&item.request)) - } - - fn remove_request_id(&mut self, request_id: &str) -> Option { - let mut retained = BinaryHeap::new(); - let mut removed = None; - - while let Some(item) = self.heap.pop() { - if removed.is_none() && item.request.id == request_id { - let key = Self::make_dedup_key(&item.request); - Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); - removed = Some(item.request); - } else { - retained.push(item); - } - } - - self.heap = retained; - if let Some(removed) = removed.as_ref() { - self.refresh_dedup_representative(&Self::make_dedup_key(removed)); - } - removed - } - - fn remove_matching(&mut self, mut should_remove: F) -> Vec - where - F: FnMut(&HealRequest) -> bool, - { - let mut retained = BinaryHeap::new(); - let mut removed = Vec::new(); - let mut affected_keys = Vec::new(); - - while let Some(item) = self.heap.pop() { - if should_remove(&item.request) { - let key = Self::make_dedup_key(&item.request); - Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); - affected_keys.push(key); - removed.push(item.request); - } else { - retained.push(item); - } - } - - self.heap = retained; - for key in &affected_keys { - self.refresh_dedup_representative(key); - } - removed - } -} - -impl RetryingHeal { - fn status(&self) -> HealTaskStatus { - HealTaskStatus::Retrying { - error: self.error.clone(), - retry_attempt: self.request.retry_attempts, - } - } -} - fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool { let heal_path = heal_path.trim_matches('/'); if heal_path.is_empty() || heal_path == LEGACY_ROOT_HEAL_PATH { @@ -1560,345 +1093,6 @@ impl HealManager { Ok(()) } - /// Detect whether the previous run ended without a clean shutdown and, if so, - /// enqueue a full erasure-set heal for every local set. Also (re)writes the - /// marker for the current run; [`super::clear_unclean_shutdown_markers`] - /// removes it again during graceful shutdown. Best-effort: failures only log. - async fn process_unclean_shutdown(&self) { - let mut unclean = false; - let mut set_disk_ids = HashSet::new(); - let mut replacement_intents = HashMap::, Vec, String)>::new(); - let mut replacement_restarts = HashMap::)>::new(); - let mut conflicted_replacement_sets = HashSet::new(); - - { - let local_disks = { - let local_disk_map = local_disk_map_read().await; - local_disk_map.values().flatten().cloned().collect::>() - }; - for disk in &local_disks { - let endpoint = disk.endpoint(); - match disk - .read_all(super::RUSTFS_META_BUCKET, super::UNCLEAN_SHUTDOWN_MARKER_PATH) - .await - { - Ok(_) => unclean = true, - Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {} - Err(err) => { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - endpoint = %endpoint, - error = ?err, - "Unclean-shutdown marker check failed" - ); - } - } - - let marker = SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs().to_string()) - .unwrap_or_default(); - if let Err(err) = disk - .write_all(super::RUSTFS_META_BUCKET, super::UNCLEAN_SHUTDOWN_MARKER_PATH, marker.into()) - .await - { - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - endpoint = %endpoint, - error = ?err, - "Unclean-shutdown marker write failed" - ); - } - - let disk_set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); - if let Some(set_disk_id) = &disk_set_disk_id { - set_disk_ids.insert(set_disk_id.clone()); - } - - // Legacy flat records are inspected only while starting. The - // periodic scanner lists the dedicated replacement directory. - if let Err(error) = ResumeUtils::migrate_legacy_replacement_records(disk).await { - if let Some(set_disk_id) = &disk_set_disk_id { - self.block_replacement_recovery_set(set_disk_id); - } - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - endpoint = %endpoint, - error = %error, - "Legacy replacement recovery migration failed" - ); - } - let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await { - Ok(task_ids) => task_ids, - Err(error) => { - if let Some(set_disk_id) = &disk_set_disk_id { - self.block_replacement_recovery_set(set_disk_id); - } - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - endpoint = %endpoint, - error = %error, - "Replacement recovery discovery failed" - ); - continue; - } - }; - for task_id in replacement_task_ids { - let manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { - Ok(manager) => manager, - Err(error) => { - if let Some(set_disk_id) = &disk_set_disk_id { - self.block_replacement_recovery_set(set_disk_id); - } - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - endpoint = %endpoint, - task_id, - error = %error, - "Replacement recovery intent load failed" - ); - continue; - } - }; - let state = manager.get_state().await; - let active_replacement = !state.completed - && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding); - let verified_replacement = state.completed - && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending); - if (active_replacement || verified_replacement) - && state.replacement_generation.as_deref() == Some(task_id.as_str()) - && !state.replacement_targets.is_empty() - { - if matches!(state.replacement_phase, ReplacementPhase::CleanupPending) { - replacement_intents.entry(task_id).or_insert(( - state.set_disk_id, - state.replacement_targets, - state.replacement_buckets, - endpoint.to_string(), - )); - continue; - } - match self.storage.replacement_target_identities(&state.replacement_targets).await { - Ok(identities) if identities == state.replacement_target_identities => { - let resume_endpoint = endpoint.to_string(); - match replacement_intents.entry(task_id) { - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(( - state.set_disk_id, - state.replacement_targets, - state.replacement_buckets, - resume_endpoint, - )); - } - std::collections::hash_map::Entry::Occupied(entry) => { - let (existing_set_disk_id, existing_targets, existing_buckets, existing_anchor) = - entry.get(); - if existing_set_disk_id != &state.set_disk_id - || existing_targets != &state.replacement_targets - || existing_buckets != &state.replacement_buckets - || existing_anchor != &resume_endpoint - { - conflicted_replacement_sets.insert(state.set_disk_id.clone()); - self.block_replacement_recovery_set(&state.set_disk_id); - } - } - } - } - Ok(_) => { - if manager.abandon_replacement_intent().await.is_ok() { - replacement_restarts - .entry(task_id) - .or_insert((state.set_disk_id, state.replacement_targets)); - } - } - Err(_) => {} - } - } - } - } - } - - if !unclean && replacement_intents.is_empty() && replacement_restarts.is_empty() { - return; - } - - let mut recovery_by_set = HashMap::, Vec, Vec, Option)>>::new(); - for (task_id, (set_disk_id, heal_endpoints, buckets, resume_endpoint)) in replacement_intents { - recovery_by_set - .entry(set_disk_id) - .or_default() - .push((Some(task_id), heal_endpoints, buckets, Some(resume_endpoint))); - } - for (_abandoned_task_id, (set_disk_id, heal_endpoints)) in replacement_restarts { - recovery_by_set - .entry(set_disk_id) - .or_default() - .push((None, heal_endpoints, Vec::new(), None)); - } - - for (set_disk_id, mut recoveries) in recovery_by_set { - let Ok((pool_index, set_index)) = crate::heal::utils::parse_set_disk_id(&set_disk_id) else { - continue; - }; - if self.replacement_recovery_set_is_blocked(&set_disk_id) { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - set_disk_id, - recovery_count = recoveries.len(), - "Replacement recovery deferred because durable recovery validation is blocked" - ); - continue; - } - if conflicted_replacement_sets.contains(&set_disk_id) || recoveries.len() != 1 { - self.block_replacement_recovery_set(&set_disk_id); - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - set_disk_id, - recovery_count = recoveries.len(), - "Replacement recovery deferred because multiple durable generations exist" - ); - continue; - } - let reuse_single_generation = recoveries.len() == 1 && recoveries[0].0.is_some(); - let mut heal_endpoints = recoveries - .iter_mut() - .flat_map(|(_, targets, _, _)| std::mem::take(targets)) - .collect::>(); - heal_endpoints.sort_unstable(); - heal_endpoints.dedup(); - let buckets = if reuse_single_generation { - std::mem::take(&mut recoveries[0].2) - } else { - Vec::new() - }; - let mut req = HealRequest::new( - HealType::ErasureSet { - buckets, - set_disk_id: set_disk_id.clone(), - }, - HealOptions { - pool_index: Some(pool_index), - set_index: Some(set_index), - timeout: None, - ..HealOptions::default() - }, - HealPriority::Low, - ); - if reuse_single_generation && let Some(task_id) = recoveries[0].0.take() { - req.id = task_id; - } - let recovery_anchor = reuse_single_generation.then(|| recoveries[0].3.take()).flatten(); - req.source = HealRequestSource::AutoHeal; - req.heal_endpoints = heal_endpoints; - let request_id = req.id.clone(); - if let Some(anchor) = &recovery_anchor { - self.replacement_recovery_anchors - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .insert(request_id.clone(), anchor.clone()); - } - match self.submit_heal_request(req).await { - Ok(HealAdmissionResult::Accepted) => {} - Ok(_) => { - self.replacement_recovery_anchors - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .remove(&request_id); - } - Err(err) => { - self.replacement_recovery_anchors - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .remove(&request_id); - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - set_disk_id, - error = %err, - "Replacement recovery enqueue failed" - ); - } - } - } - - if !unclean || set_disk_ids.is_empty() { - return; - } - - info!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - set_count = set_disk_ids.len(), - "Unclean shutdown detected; scheduling erasure-set heal for local sets" - ); - - let buckets = match self.storage.list_buckets().await { - Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::>(), - Err(err) => { - error!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - error = %err, - "Unclean-shutdown heal skipped: bucket listing failed" - ); - return; - } - }; - - for set_disk_id in set_disk_ids { - let mut req = HealRequest::new( - HealType::ErasureSet { - buckets: buckets.clone(), - set_disk_id: set_disk_id.clone(), - }, - HealOptions { - timeout: None, - ..HealOptions::default() - }, - HealPriority::Low, - ); - req.source = HealRequestSource::AutoHeal; - if let Err(err) = self.submit_heal_request(req).await { - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_UNCLEAN_SHUTDOWN, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - set_disk_id, - error = %err, - "Unclean-shutdown heal enqueue failed" - ); - } - } - } - /// Stop HealManager pub async fn stop(&self) -> Result<()> { info!( @@ -2611,1044 +1805,6 @@ impl HealManager { snapshot.refresh_estimated_completion_time(); Some(snapshot) } - - /// Start scheduler - async fn start_scheduler(&self) -> Result<()> { - let config = self.config.clone(); - let heal_queue = self.heal_queue.clone(); - let active_heals = self.active_heals.clone(); - let completed_heals = self.completed_heals.clone(); - let retrying_heals = self.retrying_heals.clone(); - let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); - let cancel_token = self.cancel_token.clone(); - let statistics = self.statistics.clone(); - let storage = self.storage.clone(); - let notify = self.notify.clone(); - let workload_provider = self.workload_provider.clone(); - - tokio::spawn(async move { - let mut interval = interval(config.read().await.heal_interval); - - loop { - let event_driven_scheduler_enable = config.read().await.event_driven_scheduler_enable; - tokio::select! { - _ = cancel_token.cancelled() => { - info!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_SCHEDULER_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - state = "shutdown", - "Heal scheduler stopped" - ); - break; - } - _ = notify.notified(), if event_driven_scheduler_enable => { - Self::process_heal_queue(HealQueueContext { - heal_queue: &heal_queue, - active_heals: &active_heals, - completed_heals: &completed_heals, - retrying_heals: &retrying_heals, - replacement_recovery_anchors: &replacement_recovery_anchors, - config: &config, - statistics: &statistics, - storage: &storage, - notify: ¬ify, - cancel_token: &cancel_token, - workload_provider: &workload_provider, - }) - .await; - } - _ = interval.tick() => { - Self::process_heal_queue(HealQueueContext { - heal_queue: &heal_queue, - active_heals: &active_heals, - completed_heals: &completed_heals, - retrying_heals: &retrying_heals, - replacement_recovery_anchors: &replacement_recovery_anchors, - config: &config, - statistics: &statistics, - storage: &storage, - notify: ¬ify, - cancel_token: &cancel_token, - workload_provider: &workload_provider, - }) - .await; - } - } - } - }); - - Ok(()) - } - - /// Start background task to auto scan local disks and enqueue erasure set heal requests - async fn start_auto_disk_scanner(&self) -> Result<()> { - let config = self.config.clone(); - let heal_queue = self.heal_queue.clone(); - let active_heals = self.active_heals.clone(); - let storage = self.storage.clone(); - let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); - let replacement_recovery_blocked_sets = self.replacement_recovery_blocked_sets.clone(); - let cancel_token = self.cancel_token.clone(); - let notify = self.notify.clone(); - let mut duration = { - let config = config.read().await; - config.heal_interval - }; - if duration < Duration::from_secs(10) { - duration = Duration::from_secs(10); - } - info!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - state = "started", - interval = ?duration, - "Heal auto disk scanner started" - ); - - tokio::spawn(async move { - let mut interval = interval(duration); - - loop { - let mut candidate_count = 0usize; - let mut skipped_duplicate_count = 0usize; - let mut skipped_invalid_count = 0usize; - let mut enqueued_count = 0usize; - let mut not_enqueued_count = 0usize; - let mut dropped_count = 0usize; - let mut full_count = 0usize; - tokio::select! { - _ = cancel_token.cancelled() => { - info!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - state = "shutdown", - "Heal auto disk scanner stopped" - ); - break; - } - _ = interval.tick() => { - // Build list of endpoints that need healing - let mut endpoints = HashMap::>::new(); - let mut durable_recoveries = HashMap::, Vec, String)>::new(); - let mut conflicted_recovery_sets = HashSet::::new(); - let mut deferred_replacement_endpoints = HashSet::::new(); - let local_disks = { - let local_disk_map = local_disk_map_read().await; - local_disk_map.values().flatten().cloned().collect::>() - }; - let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::>(); - let blocked_sets = replacement_recovery_blocked_sets - .lock() - .expect("replacement recovery blocked set lock poisoned") - .clone(); - if !blocked_sets.is_empty() { - let mut retry_succeeded = HashSet::new(); - let mut retry_failed = HashSet::new(); - for disk in &local_disks { - let endpoint = disk.endpoint(); - let Some(set_disk_id) = - crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx) - else { - continue; - }; - if !blocked_sets.contains(&set_disk_id) { - continue; - } - match Self::validate_replacement_recovery_records(disk).await { - Ok(()) => { - retry_succeeded.insert(set_disk_id); - } - Err(error) => { - retry_failed.insert(set_disk_id.clone()); - conflicted_recovery_sets.insert(set_disk_id); - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - error = %error, - "Replacement recovery retry failed" - ); - } - } - } - let mut blocked = replacement_recovery_blocked_sets - .lock() - .expect("replacement recovery blocked set lock poisoned"); - unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed); - } - for disk in &local_disks { - let endpoint = disk.endpoint(); - let runtime_state = disk.runtime_state(); - let set_disk_id = - crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); - if set_disk_id.as_ref().is_some_and(|set_disk_id| { - replacement_recovery_blocked_sets - .lock() - .expect("replacement recovery blocked set lock poisoned") - .contains(set_disk_id) - }) { - skipped_invalid_count += 1; - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_DISK, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - set_disk_id = set_disk_id.as_deref().unwrap_or_default(), - disk_state = "replacement_recovery_blocked", - "Heal auto-scan replacement deferred because durable recovery is blocked" - ); - continue; - } - - // detect unformatted disk via get_disk_id() - match disk.get_disk_id().await { - Err(DiskError::UnformattedDisk) => { - if !super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks) - .await - { - deferred_replacement_endpoints.insert(endpoint.to_string()); - skipped_invalid_count += 1; - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_DISK, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - disk_state = "replacement_path_unavailable", - "Heal auto-scan replacement deferred" - ); - continue; - } - let Some(set_disk_id) = set_disk_id else { - skipped_invalid_count += 1; - continue; - }; - candidate_count += 1; - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_DISK, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - disk_state = "unformatted", - "Heal auto-scan candidate detected" - ); - endpoints.entry(set_disk_id).or_default().push(endpoint); - } - Err(e) => { - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_DISK, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - disk_state = "check_failed", - error = ?e, - "Heal auto-scan disk inspection failed" - ); - } - Ok(_) => { - if runtime_state.as_str() == "returning" && let Some(set_disk_id) = set_disk_id { - candidate_count += 1; - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_DISK, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - set_disk_id, - disk_state = "returning", - "Heal auto-scan returning disk candidate detected" - ); - endpoints.entry(set_disk_id).or_default().push(endpoint); - } - } - } - } - - // Once formatting succeeds a replacement is no longer - // discoverable as UnformattedDisk. Re-admit exactly one - // incomplete durable generation per set after bounded - // scheduler retries are exhausted, or re-admit its - // verified terminal cleanup. Multiple generations are a - // durable conflict: leave every marker/state intact and - // require reconciliation rather than choosing one. - for disk in &local_disks { - let endpoint = disk.endpoint(); - let disk_set_disk_id = - crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); - let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await { - Ok(task_ids) => task_ids, - Err(error) => { - let endpoint_string = endpoint.to_string(); - if replacement_discovery_error_is_expected_for_deferred_endpoint( - &error, - &endpoint_string, - &deferred_replacement_endpoints, - ) { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - disk_state = "replacement_path_unavailable", - result = "recovery_records_unavailable", - "Replacement recovery discovery skipped for deferred replacement" - ); - continue; - } - if let Some(set_disk_id) = &disk_set_disk_id { - conflicted_recovery_sets.insert(set_disk_id.clone()); - } - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - error = %error, - "Replacement recovery discovery failed" - ); - continue; - } - }; - for task_id in replacement_task_ids { - let resume_manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { - Ok(resume_manager) => resume_manager, - Err(error) => { - if let Some(set_disk_id) = &disk_set_disk_id { - conflicted_recovery_sets.insert(set_disk_id.clone()); - } - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint = %endpoint, - task_id, - error = %error, - "Replacement recovery intent load failed" - ); - continue; - } - }; - let state = resume_manager.get_state().await; - if !durable_replacement_recovery_is_due(&state, &task_id) { - continue; - } - if !matches!(state.replacement_phase, ReplacementPhase::CleanupPending) { - let Ok(identities) = storage.replacement_target_identities(&state.replacement_targets).await else { - continue; - }; - if identities != state.replacement_target_identities { - continue; - } - } - let targets = state - .replacement_targets - .iter() - .filter_map(|target| { - local_endpoints - .iter() - .find(|endpoint| endpoint.to_string() == *target) - .cloned() - }) - .collect::>(); - if targets.len() != state.replacement_targets.len() { - continue; - } - let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32( - targets[0].pool_idx, - targets[0].set_idx, - ) else { - continue; - }; - if targets.iter().any(|target| { - crate::heal::utils::format_set_disk_id_from_i32(target.pool_idx, target.set_idx) - .as_deref() - != Some(set_disk_id.as_str()) - }) { - continue; - } - let resume_endpoint = disk.endpoint().to_string(); - match durable_recoveries.get(&set_disk_id) { - Some((existing_task_id, _, _, existing_anchor)) - if existing_task_id != &task_id || existing_anchor != &resume_endpoint => { - replacement_recovery_blocked_sets - .lock() - .expect("replacement recovery blocked set lock poisoned") - .insert(set_disk_id.clone()); - conflicted_recovery_sets.insert(set_disk_id); - } - Some(_) => {} - None => { - durable_recoveries.insert( - set_disk_id, - (task_id, targets, state.replacement_buckets, resume_endpoint), - ); - } - } - } - } - - for set_disk_id in &conflicted_recovery_sets { - durable_recoveries.remove(set_disk_id); - endpoints.remove(set_disk_id); - warn!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - set_disk_id, - result = "durable_generation_conflict", - "Replacement recovery deferred because multiple durable generations exist" - ); - } - - for (set_disk_id, (_, targets, _, _)) in &durable_recoveries { - let expected = targets.iter().map(ToString::to_string).collect::>(); - let observed = endpoints - .get(set_disk_id) - .map(|endpoints| endpoints.iter().map(ToString::to_string).collect::>()) - .unwrap_or_default(); - if !observed.is_subset(&expected) { - replacement_recovery_blocked_sets - .lock() - .expect("replacement recovery blocked set lock poisoned") - .insert(set_disk_id.clone()); - conflicted_recovery_sets.insert(set_disk_id.clone()); - continue; - } - endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone()); - } - for set_disk_id in &conflicted_recovery_sets { - durable_recoveries.remove(set_disk_id); - endpoints.remove(set_disk_id); - } - - for target_endpoints in endpoints.values_mut() { - target_endpoints.sort_by_key(ToString::to_string); - target_endpoints.dedup_by(|left, right| left.to_string() == right.to_string()); - } - - if endpoints.is_empty() { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - state = "idle", - "Heal auto disk scanner idle" - ); - continue; - } - - // Admit one set task with every ready replacement target. Queue deduplication is - // set-scoped, so admitting endpoints independently would silently drop later targets. - for (set_disk_id, endpoints) in endpoints { - if replacement_recovery_blocked_sets - .lock() - .expect("replacement recovery blocked set lock poisoned") - .contains(&set_disk_id) - { - skipped_invalid_count += 1; - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - set_disk_id, - result = "replacement_recovery_blocked", - "Heal auto-scan replacement admission deferred because durable recovery is blocked" - ); - continue; - } - // skip if already queued or healing - // Use consistent lock order: queue first, then active_heals to avoid deadlock - let mut skip = false; - { - let queue = heal_queue.lock().await; - if queue.contains_erasure_set(&set_disk_id) { - skip = true; - } - } - if !skip { - let active = active_heals.lock().await; - if active.values().any(|task| { - matches!( - &task.heal_type, - crate::heal::task::HealType::ErasureSet { set_disk_id: active_id, .. } - if active_id == &set_disk_id - ) - }) { - skip = true; - } - } - - if skip { - skipped_duplicate_count += 1; - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint_count = endpoints.len(), - set_disk_id, - result = "skipped_duplicate", - "Heal auto-scan duplicate skipped" - ); - continue; - } - - // enqueue erasure set heal request for all ready replacements in this set - let recovery = durable_recoveries.remove(&set_disk_id); - let mut req = HealRequest::new( - HealType::ErasureSet { - buckets: recovery - .as_ref() - .map(|(_, _, buckets, _)| buckets.clone()) - .unwrap_or_default(), - set_disk_id: set_disk_id.clone(), - }, - HealOptions { - pool_index: endpoints - .first() - .and_then(|endpoint| usize::try_from(endpoint.pool_idx).ok()), - set_index: endpoints - .first() - .and_then(|endpoint| usize::try_from(endpoint.set_idx).ok()), - timeout: None, - ..HealOptions::default() - }, - HealPriority::Low, - ); - let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone()); - if let Some((task_id, _, _, _)) = recovery { - req.id = task_id; - } - req.source = HealRequestSource::AutoHeal; - req.heal_endpoints = endpoints.iter().map(ToString::to_string).collect(); - let request_id = req.id.clone(); - let endpoint_count = req.heal_endpoints.len(); - let config = config.read().await; - let mut queue = heal_queue.lock().await; - let admission = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan"); - let should_notify = - matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable; - if matches!(admission, HealAdmissionResult::Accepted) - && let Some(anchor) = recovery_anchor - { - replacement_recovery_anchors - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .insert(request_id, anchor); - } - drop(queue); - drop(config); - if matches!(admission, HealAdmissionResult::Accepted) { - if should_notify { - notify.notify_one(); - } - enqueued_count += 1; - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint_count, - set_disk_id, - bucket_count = 0, - result = "enqueued", - "Heal auto-scan task enqueued" - ); - } else { - if matches!(admission, HealAdmissionResult::Merged) { - skipped_duplicate_count += 1; - } else { - not_enqueued_count += 1; - } - if matches!(admission, HealAdmissionResult::Full) { - full_count += 1; - } - if matches!(admission, HealAdmissionResult::Dropped(_)) { - dropped_count += 1; - } - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - endpoint_count, - set_disk_id, - bucket_count = 0, - admission = admission.result_label(), - reason = admission.reason_label(), - result = "not_enqueued", - "Heal auto-scan task not enqueued" - ); - } - } - info!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_AUTO_SCAN_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_DISK_SCANNER, - state = "cycle_completed", - candidate_count, - enqueued_count, - not_enqueued_count, - dropped_count, - full_count, - skipped_duplicate_count, - skipped_invalid_count, - "Heal auto-scan cycle completed" - ); - } - } - } - }); - Ok(()) - } - - /// Process heal queue - /// Processes multiple tasks per cycle when capacity allows and queue has high-priority items - async fn process_heal_queue(context: HealQueueContext<'_>) { - let HealQueueContext { - heal_queue, - active_heals, - completed_heals, - retrying_heals, - replacement_recovery_anchors, - config, - statistics, - storage, - notify, - cancel_token, - workload_provider, - } = context; - - let config = config.read().await; - let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider); - let mut active_heals_guard = active_heals.lock().await; - publish_active_heal_count(&active_heals_guard); - - // Check if new heal tasks can be started - let active_count = active_heals_guard.len(); - if active_count >= config.max_concurrent_heals { - return; - } - - // Calculate how many tasks we can start this cycle - let available_slots = config.max_concurrent_heals - active_count; - - let mut queue = heal_queue.lock().await; - let queue_len = queue.len(); - publish_heal_queue_length(&queue); - - if queue_len == 0 { - return; - } - - let mut running_per_set = running_heal_set_counts(&active_heals_guard); - let mut tasks_started = 0usize; - let mut delayed_by_mainline_throttle = false; - - for _ in 0..available_slots { - let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() { - let max_concurrent_per_set = config.max_concurrent_per_set; - let (selected_request, skipped_sets) = queue.pop_runnable_with_skips( - |request| { - let set_allowed = !config.set_bulkhead_enable - || can_schedule_request(request, &running_per_set, max_concurrent_per_set); - let mainline_allowed = mainline_pressure.is_none() || Self::request_bypasses_mainline_throttle(request); - set_allowed && mainline_allowed - }, - |request| heal_request_set_key(request).map(|_| heal_request_set_metric_label(request)), - ); - for skipped_set in skipped_sets { - record_scheduler_skip(&skipped_set); - } - selected_request - } else { - queue.pop_next() - }; - - if let Some(mut request) = selected_request { - request.options.timeout.get_or_insert(config.task_timeout); - let task_priority = request.priority; - let task_type_label = heal_request_type_label(&request).to_string(); - let task_set_label = heal_request_set_metric_label(&request); - if config.set_bulkhead_enable - && let Some(set_key) = heal_request_set_key(&request) - { - *running_per_set.entry(set_key).or_insert(0) += 1; - } - let replacement_resume_endpoint = replacement_recovery_anchors - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .get(&request.id) - .cloned(); - let task = Arc::new(HealTask::from_replacement_recovery_request( - request, - storage.clone(), - replacement_resume_endpoint, - )); - let task_id = task.id.clone(); - active_heals_guard.insert(task_id.clone(), task.clone()); - publish_active_heal_count(&active_heals_guard); - update_task_running_metric_for_task(&active_heals_guard, task.as_ref()); - let active_heals_clone = active_heals.clone(); - let heal_queue_clone = heal_queue.clone(); - let completed_heals_clone = completed_heals.clone(); - let retrying_heals_clone = retrying_heals.clone(); - let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone(); - let statistics_clone = statistics.clone(); - let notify_clone = notify.clone(); - let manager_cancel_token = cancel_token.clone(); - let task_type_label_for_spawn = task_type_label.clone(); - let task_set_label_for_spawn = task_set_label.clone(); - let config_for_spawn = config.clone(); - - // start heal task - tokio::spawn(async move { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_SCHEDULER_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - task_id, - priority = ?task_priority, - heal_type = %task_type_label_for_spawn, - set = %task_set_label_for_spawn, - state = "task_started", - "Heal scheduler task started" - ); - let result = task.execute().await; - let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await; - match &result { - Ok(_) => { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_SCHEDULER_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - task_id, - heal_type = %task_type_label_for_spawn, - set = %task_set_label_for_spawn, - state = "task_completed", - "Heal scheduler task completed" - ); - } - Err(e) => { - let will_retry = retry_request.is_some(); - if will_retry { - demote_to_debug_when!(task.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", { - event = EVENT_HEAL_SCHEDULER_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - task_id, - heal_type = %task_type_label_for_spawn, - set = %task_set_label_for_spawn, - state = "task_retrying", - retry_attempt = task.retry_attempts.saturating_add(1), - error = %e, - "Heal scheduler task retrying" - }); - } else { - error!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_SCHEDULER_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - task_id, - heal_type = %task_type_label_for_spawn, - set = %task_set_label_for_spawn, - state = "task_failed", - error = %e, - "Heal scheduler task failed" - ); - } - } - } - let retry_request_for_status = retry_request.as_ref().map(|(request, _, error)| HealTaskStatus::Retrying { - error: error.clone(), - retry_attempt: request.retry_attempts, - }); - let retry_request_for_queue = retry_request; - let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new()); - if retry_request_for_queue.is_none() { - replacement_recovery_anchors_clone - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .remove(&task_id); - } - let mut active_heals_guard = active_heals_clone.lock().await; - // Keep retry ownership continuous: status snapshots acquire - // these locks in the same active -> retrying order. - let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) = - (retry_request_for_queue.as_ref(), retry_cancel_token.as_ref()) - { - let mut retrying = retrying_heals_clone.lock().await; - if active_heals_guard.contains_key(&task_id) { - retrying.insert( - request.id.clone(), - RetryingHeal { - request: request.clone(), - error: error.clone(), - cancel_token: cancel_token.clone(), - }, - ); - #[cfg(test)] - pause_retry_ownership_transition(&task_id, false).await; - } - Some(retrying) - } else { - None - }; - let completed_task = active_heals_guard.remove(&task_id); - if let Some(completed_task) = completed_task.as_ref() { - publish_active_heal_count(&active_heals_guard); - update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref()); - } - let active_count = active_heals_guard.len(); - drop(retrying_heals_guard.take()); - drop(active_heals_guard); - - if let Some(completed_task) = completed_task { - let completed_status = if let Some(status) = retry_request_for_status { - status - } else { - completed_task.get_status().await - }; - let completed_progress = completed_task.get_progress().await; - // Single snapshot of the retained window: the task is - // finished and already off the active map, so there is - // no concurrent writer to race with. - let seqed_items = completed_task.get_seqed_result_items().await; - let (next_seq, min_seq) = completed_task.result_seq_cursors(); - let completed_status_entry = CompletedHealStatus { - heal_type: completed_task.heal_type.clone(), - status: completed_status.clone(), - result_items_truncated: completed_task.result_items_truncated(), - completed_at: SystemTime::now(), - seqed_items, - next_seq, - min_seq, - }; - let mut completed_heals_guard = completed_heals_clone.lock().await; - prune_completed_heal_statuses(&mut completed_heals_guard); - completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry)); - // update statistics - let mut stats = statistics_clone.write().await; - match completed_status { - HealTaskStatus::Completed => { - stats.update_task_completion(true); - stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed); - } - HealTaskStatus::Retrying { .. } => {} - _ => { - stats.update_task_completion(false); - } - } - stats.update_running_tasks(usize_to_u64_saturated(active_count)); - } - - if let (Some((retry_request, retry_delay, retry_error)), Some(retry_cancel_token)) = - (retry_request_for_queue, retry_cancel_token) - { - let retry_request_id = retry_request.id.clone(); - let retry_attempt = retry_request.retry_attempts; - let retry_key = PriorityHealQueue::make_dedup_key(&retry_request); - let retry_priority = retry_request.priority; - let retry_active_heals = active_heals_clone.clone(); - let retry_heal_queue = heal_queue_clone.clone(); - let retrying_heals_for_spawn = retrying_heals_clone.clone(); - let retry_completed_heals = completed_heals_clone.clone(); - let retry_notify = notify_clone.clone(); - let retry_manager_cancel_token = manager_cancel_token.clone(); - let retry_config = config_for_spawn.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = retry_cancel_token.cancelled() => { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_ADMISSION, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - request_id = %retry_request_id, - priority = ?retry_priority, - retry_attempt, - result = "retry_cancelled", - "Heal retry admission decided" - ); - return; - } - _ = retry_manager_cancel_token.cancelled() => { - retrying_heals_for_spawn.lock().await.remove(&retry_request_id); - return; - } - _ = sleep(retry_delay) => {} - } - - { - let retrying_heals_guard = retrying_heals_for_spawn.lock().await; - if !retrying_heals_guard.contains_key(&retry_request_id) { - return; - } - } - - let active_duplicate = { - let active_heals_guard = retry_active_heals.lock().await; - active_heals_contains_dedup_key(&active_heals_guard, &retry_key) - }; - if active_duplicate { - retrying_heals_for_spawn.lock().await.remove(&retry_request_id); - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_ADMISSION, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - request_id = %retry_request_id, - priority = ?retry_priority, - retry_attempt, - result = "retry_merged_active_duplicate", - "Heal retry admission decided" - ); - return; - } - - let mut queue = retry_heal_queue.lock().await; - let admission = - Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry"); - let should_notify = matches!(admission, HealAdmissionResult::Accepted) - && retry_config.event_driven_scheduler_enable; - match admission { - HealAdmissionResult::Accepted => { - // Transfer ownership while holding queue -> retrying, - // matching operations_snapshot's lock order. - #[cfg(test)] - pause_retry_ownership_transition(&retry_request_id, true).await; - retrying_heals_for_spawn.lock().await.remove(&retry_request_id); - drop(queue); - retry_completed_heals.lock().await.remove(&retry_request_id); - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_ADMISSION, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - request_id = %retry_request_id, - priority = ?retry_priority, - retry_attempt, - retry_delay_ms = retry_delay.as_millis(), - error = %retry_error, - result = "retry_enqueued", - "Heal retry admission decided" - ); - if should_notify { - retry_notify.notify_one(); - } - return; - } - HealAdmissionResult::Merged => { - retrying_heals_for_spawn.lock().await.remove(&retry_request_id); - drop(queue); - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_ADMISSION, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - request_id = %retry_request_id, - priority = ?retry_priority, - retry_attempt, - result = "retry_merged_duplicate", - "Heal retry admission decided" - ); - return; - } - HealAdmissionResult::Full => { - // admit_request_to_queue already logged the - // rejection (context = "retry"); this repeats - // every backoff cycle while the queue stays - // full, so keep it at debug!. - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_ADMISSION, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - request_id = %retry_request_id, - priority = ?retry_priority, - retry_attempt, - result = "retry_rejected_full", - "Heal retry admission decided" - ); - } - HealAdmissionResult::Dropped(reason) => { - debug!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_ADMISSION, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - request_id = %retry_request_id, - priority = ?retry_priority, - retry_attempt, - reason = reason.as_str(), - result = "retry_dropped", - "Heal retry admission decided" - ); - } - } - } - }); - } - notify_clone.notify_one(); - }); - tasks_started += 1; - } else { - delayed_by_mainline_throttle = mainline_pressure.is_some(); - break; - } - } - - // Update statistics for all started tasks - let mut stats = statistics.write().await; - stats.total_tasks += tasks_started as u64; - stats.update_running_tasks(active_heals_guard.len() as u64); - publish_active_heal_count(&active_heals_guard); - publish_heal_queue_length(&queue); - - if delayed_by_mainline_throttle && let Some(pressure) = mainline_pressure { - Self::record_mainline_throttle_delay(pressure, &config); - Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep); - } - - // Log queue status if items remain - if !queue.is_empty() { - let remaining = queue.len(); - if remaining > 10 { - info!( - target: "rustfs::heal::manager", - event = EVENT_HEAL_QUEUE_STATE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_MANAGER, - queue_len = remaining, - active_tasks = active_heals_guard.len(), - state = "backlog_high", - "Heal queue backlog high" - ); - } - } - } } impl std::fmt::Debug for HealManager { @@ -3662,3062 +1818,14 @@ impl std::fmt::Debug for HealManager { } } -fn heal_request_set_key(request: &HealRequest) -> Option { - match &request.heal_type { - HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()), - HealType::Object { .. } => heal_options_set_key(&request.options), - _ => None, - } -} +mod auto_scan; +mod queue; +mod scheduler; +mod unclean_shutdown; -fn heal_options_set_key(options: &HealOptions) -> Option { - match (options.pool_index, options.set_index) { - (Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")), - _ => None, - } -} - -fn heal_request_type_label(request: &HealRequest) -> &'static str { - match &request.heal_type { - HealType::Cluster => "cluster", - HealType::Object { .. } => "object", - HealType::Bucket { .. } => "bucket", - HealType::Prefix { .. } => "prefix", - HealType::ErasureSet { .. } => "erasure_set", - HealType::Metadata { .. } => "metadata", - HealType::ECDecode { .. } => "ec_decode", - } -} - -fn heal_request_set_metric_label(request: &HealRequest) -> String { - heal_request_set_key(request).unwrap_or_else(|| match (request.options.pool_index, request.options.set_index) { - (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"), - _ => "global".to_string(), - }) -} - -fn record_scheduler_skip(set_label: &str) { - counter!( - "rustfs_heal_scheduler_skip_total", - "reason" => "set_limit".to_string(), - "set" => set_label.to_string() - ) - .increment(1); -} - -fn update_task_running_metric_for_task(active_heals: &HashMap>, task: &HealTask) { - let type_label = task.metric_type_label(); - let set_label = task.metric_set_label(); - let count = active_heals - .values() - .filter(|active_task| active_task.metric_type_label() == type_label && active_task.metric_set_label() == set_label) - .count(); - - gauge!( - "rustfs_heal_task_running", - "type" => type_label.to_string(), - "set" => set_label - ) - .set(count as f64); -} - -fn running_heal_set_counts(active_heals: &HashMap>) -> HashMap { - let mut running = HashMap::new(); - for task in active_heals.values() { - if let Some(set_key) = heal_request_set_key_for_task(task) { - *running.entry(set_key).or_insert(0) += 1; - } - } - running -} - -fn heal_request_set_key_for_task(task: &HealTask) -> Option { - match &task.heal_type { - HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()), - HealType::Object { .. } => heal_options_set_key(&task.options), - _ => None, - } -} - -fn prune_completed_heal_statuses(completed_heals: &mut HashMap>) { - let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else { - return; - }; - - completed_heals.retain(|_, completed| { - completed - .completed_at - .duration_since(SystemTime::UNIX_EPOCH) - .map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION) - .unwrap_or(false) - }); -} - -fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap, max_concurrent_per_set: usize) -> bool { - match heal_request_set_key(request) { - Some(set_key) => running_per_set.get(&set_key).copied().unwrap_or(0) < max_concurrent_per_set, - None => true, - } -} +use queue::*; +use scheduler::*; +use unclean_shutdown::*; #[cfg(test)] -mod tests { - use super::*; - use crate::heal::EcstoreError; - use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity}; - use crate::heal::storage::{HealObjectInfo, HealStorageAPI}; - use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType}; - use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; - use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot}; - use rustfs_madmin::heal_commands::HealResultItem; - use std::sync::Mutex as StdMutex; - use tempfile::TempDir; - - use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo}; - - #[tokio::test] - async fn auto_replacement_path_requires_a_non_root_mount() { - let temp = TempDir::new().expect("temporary replacement root should be created"); - let ready = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("replacement endpoint should parse"); - let missing = Endpoint::try_from(temp.path().join("missing").to_string_lossy().as_ref()) - .expect("missing replacement endpoint should parse"); - - let ready_disk = new_disk( - &ready, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("temporary disk should initialize"); - assert!( - !super::super::replacement_readiness::auto_replacement_target_ready(&ready_disk, std::slice::from_ref(&ready_disk),) - .await - ); - assert!( - matches!( - new_disk( - &missing, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await, - Err(DiskError::VolumeNotFound) - ), - "a missing replacement path must be rejected before admission" - ); - } - - #[derive(Debug)] - struct FixedWorkloadProvider { - class: WorkloadClass, - active: usize, - limit: usize, - state: AdmissionState, - } - - impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider { - fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot { - WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(self.class, self.state).with_counts( - Some(self.active), - None, - Some(self.limit), - )]) - } - } - - async fn process_manager_queue_once(manager: &HealManager) { - HealManager::process_heal_queue(HealQueueContext { - heal_queue: &manager.heal_queue, - active_heals: &manager.active_heals, - completed_heals: &manager.completed_heals, - retrying_heals: &manager.retrying_heals, - replacement_recovery_anchors: &manager.replacement_recovery_anchors, - config: &manager.config, - statistics: &manager.statistics, - storage: &manager.storage, - notify: &manager.notify, - cancel_token: &manager.cancel_token, - workload_provider: &manager.workload_provider, - }) - .await; - } - - struct MockStorage; - - #[async_trait::async_trait] - impl HealStorageAPI for MockStorage { - async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result> { - Ok(Vec::new()) - } - - async fn get_bucket_info(&self, _bucket: &str) -> Result> { - Ok(None) - } - - async fn list_buckets(&self) -> Result> { - if let Some(hook) = manager_recovery_test_hook() { - *hook.listed.lock().expect("manager recovery listed lock should not poison") = true; - } - Ok(Vec::new()) - } - - async fn object_exists(&self, bucket: &str, _object: &str) -> Result { - Ok(bucket == "retry-transition") - } - - async fn heal_object( - &self, - bucket: &str, - _object: &str, - _version_id: Option<&str>, - _opts: &HealOpts, - ) -> Result<(HealResultItem, Option)> { - if let Some(hook) = manager_recovery_test_hook() { - *hook - .heal_object_calls - .lock() - .expect("manager recovery object call lock should not poison") += 1; - } - if bucket == "retry-transition" { - return Ok(( - HealResultItem::default(), - Some(Error::Storage(EcstoreError::InsufficientReadQuorum( - bucket.to_string(), - "object".to_string(), - ))), - )); - } - Ok((HealResultItem::default(), None)) - } - - async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result { - if let Some(hook) = manager_recovery_test_hook() { - *hook - .bucket_heal_calls - .lock() - .expect("manager recovery bucket call lock should not poison") += 1; - } - Ok(HealResultItem::default()) - } - - async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option)> { - if let Some(hook) = manager_recovery_test_hook() { - *hook - .global_format_calls - .lock() - .expect("manager recovery global format call lock should not poison") += 1; - } - Ok((HealResultItem::default(), None)) - } - - async fn heal_replacement_format( - &self, - _dry_run: bool, - _pool_index: usize, - _set_index: usize, - _targets: &[String], - ) -> Result<(HealResultItem, Option)> { - if let Some(hook) = manager_recovery_test_hook() { - *hook - .replacement_format_calls - .lock() - .expect("manager recovery replacement format call lock should not poison") += 1; - } - Ok((HealResultItem::default(), None)) - } - - async fn list_objects_for_heal_page( - &self, - _bucket: &str, - _prefix: &str, - _continuation_token: Option<&str>, - _include_lifecycle_object_info: bool, - ) -> Result<(Vec, Option, bool)> { - Ok((Vec::new(), None, false)) - } - - async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result { - Err(Error::other("not implemented in tests")) - } - - async fn get_replacement_resume_disk( - &self, - _set_disk_id: &str, - _task_id: &str, - _excluded_targets: &[String], - ) -> Result { - let Some(hook) = manager_recovery_test_hook() else { - return Err(Error::other("not implemented in tests")); - }; - Ok(crate::heal::storage::ReplacementResumeDisk::Existing( - hook.replacement_resume_disk.clone(), - )) - } - } - - struct ManagerRecoveryTestHook { - replacement_resume_disk: DiskStore, - listed: StdMutex, - global_format_calls: StdMutex, - replacement_format_calls: StdMutex, - bucket_heal_calls: StdMutex, - heal_object_calls: StdMutex, - } - - static MANAGER_RECOVERY_TEST_HOOK: LazyLock>>> = - LazyLock::new(|| StdMutex::new(None)); - - struct ManagerRecoveryTestHookGuard; - - impl ManagerRecoveryTestHook { - fn install(replacement_resume_disk: DiskStore) -> (Arc, ManagerRecoveryTestHookGuard) { - let hook = Arc::new(Self { - replacement_resume_disk, - listed: StdMutex::new(false), - global_format_calls: StdMutex::new(0), - replacement_format_calls: StdMutex::new(0), - bucket_heal_calls: StdMutex::new(0), - heal_object_calls: StdMutex::new(0), - }); - let previous = MANAGER_RECOVERY_TEST_HOOK - .lock() - .expect("manager recovery hook lock should not poison") - .replace(hook.clone()); - assert!(previous.is_none(), "manager recovery hook already installed"); - (hook, ManagerRecoveryTestHookGuard) - } - } - - impl Drop for ManagerRecoveryTestHookGuard { - fn drop(&mut self) { - *MANAGER_RECOVERY_TEST_HOOK - .lock() - .expect("manager recovery hook lock should not poison") = None; - } - } - - fn manager_recovery_test_hook() -> Option> { - MANAGER_RECOVERY_TEST_HOOK - .lock() - .expect("manager recovery hook lock should not poison") - .clone() - } - - async fn make_manager_resume_disk(temp: &TempDir, name: &str) -> DiskStore { - let disk_path = temp.path().join(name); - std::fs::create_dir_all(&disk_path).expect("manager recovery disk directory should be created"); - let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).expect("manager recovery endpoint should parse"); - let disk = new_disk( - &endpoint, - &DiskOption { - cleanup: false, - health_check: false, - }, - ) - .await - .expect("manager recovery disk should initialize"); - let metadata_volume = disk.make_volume(super::super::RUSTFS_META_BUCKET).await; - assert!( - matches!(metadata_volume, Ok(()) | Err(DiskError::VolumeExists)), - "manager recovery metadata volume should exist: {metadata_volume:?}" - ); - disk - } - - fn bucket_request(bucket: &str, priority: HealPriority, source: HealRequestSource) -> HealRequest { - let mut request = HealRequest::new( - HealType::Bucket { - bucket: bucket.to_string(), - }, - HealOptions::default(), - priority, - ); - request.source = source; - request - } - - #[test] - fn test_push_displacing_lower_priority_actually_enqueues_new_request() { - // Regression for the release-build defect where the enqueue side effect lived inside - // `debug_assert_eq!(self.push(request), ...)` and was compiled out under - // `cargo test --release` (debug_assertions off), silently dropping the displacing - // high-priority request while still having evicted a queued item. - // - // Must run with --release to expose the original bug. - let mut queue = PriorityHealQueue::new(); - - let low = bucket_request("victim-bucket", HealPriority::Low, HealRequestSource::Scanner); - assert_eq!(queue.push(low), QueuePushOutcome::Accepted); - assert_eq!(queue.len(), 1); - - let high = bucket_request("admin-bucket", HealPriority::High, HealRequestSource::Admin); - let high_id = high.id.clone(); - assert!(queue.can_displace_lower_priority(high.priority)); - - let displaced = queue - .push_displacing_lower_priority(high) - .expect("a lower-priority item should have been displaced"); - assert_eq!(displaced.priority, HealPriority::Low); - - // The displacing high-priority request must actually be enqueued (pre-fix under - // --release, len() is 0 because self.push(request) was elided with debug_assert_eq!). - assert_eq!(queue.len(), 1, "displacing request must remain enqueued"); - let admitted = queue.pop_next().expect("displacing high-priority request must be enqueued"); - assert_eq!(admitted.priority, HealPriority::High); - assert_eq!(admitted.id, high_id); - assert_eq!(queue.len(), 0); - } - - #[test] - fn queued_request_id_for_dedup_key_tracks_the_representative() { - let mut queue = PriorityHealQueue::new(); - - let first = HealRequest::object("bucket".to_string(), "object".to_string(), None); - let first_id = first.id.clone(); - let first_key = PriorityHealQueue::make_dedup_key(&first); - assert_eq!(queue.push(first), QueuePushOutcome::Accepted); - - // A forced duplicate of the same target opens a second entry under - // the same key; the representative stays the request that opened it. - let mut second = HealRequest::object("bucket".to_string(), "object".to_string(), None); - second.force_start = true; - let second_id = second.id.clone(); - assert_eq!(queue.push(second), QueuePushOutcome::Accepted); - - let representative = queue - .queued_request_id_for_dedup_key(&first_key) - .expect("key must be reserved while either request is queued"); - assert_eq!(representative, first_id); - - // A holder leaving WITHOUT becoming active (canceled by id) must - // re-elect the representative to the surviving queued request, or a - // later merge receipt would name an id that resolves nowhere. The - // scheduler pop path needs no re-election: the popped request - // surfaces in active_heals under the same id and the duplicate - // pre-check consults active heals before the queue. - queue.remove_request_id(&first_id); - assert_eq!( - queue.queued_request_id_for_dedup_key(&first_key), - Some(second_id.as_str()), - "canceling the opener must re-elect the surviving queued holder" - ); - - // Pop the last holder: the key is released entirely. - let last = queue.pop_next().expect("second request must be queued"); - assert_eq!(last.id, second_id); - assert!(queue.queued_request_id_for_dedup_key(&first_key).is_none()); - } - - #[test] - fn test_priority_queue_ordering() { - let mut queue = PriorityHealQueue::new(); - - // Add requests with different priorities - let low_req = HealRequest::new( - HealType::Bucket { - bucket: "bucket1".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - - let normal_req = HealRequest::new( - HealType::Bucket { - bucket: "bucket2".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let high_req = HealRequest::new( - HealType::Bucket { - bucket: "bucket3".to_string(), - }, - HealOptions::default(), - HealPriority::High, - ); - - let urgent_req = HealRequest::new( - HealType::Bucket { - bucket: "bucket4".to_string(), - }, - HealOptions::default(), - HealPriority::Urgent, - ); - - // Add in random order: low, high, normal, urgent - assert_eq!(queue.push(low_req), QueuePushOutcome::Accepted); - assert_eq!(queue.push(high_req), QueuePushOutcome::Accepted); - assert_eq!(queue.push(normal_req), QueuePushOutcome::Accepted); - assert_eq!(queue.push(urgent_req), QueuePushOutcome::Accepted); - - assert_eq!(queue.len(), 4); - - // Should pop in priority order: urgent, high, normal, low - let popped1 = queue.pop().unwrap(); - assert_eq!(popped1.priority, HealPriority::Urgent); - - let popped2 = queue.pop().unwrap(); - assert_eq!(popped2.priority, HealPriority::High); - - let popped3 = queue.pop().unwrap(); - assert_eq!(popped3.priority, HealPriority::Normal); - - let popped4 = queue.pop().unwrap(); - assert_eq!(popped4.priority, HealPriority::Low); - - assert_eq!(queue.len(), 0); - } - - #[test] - fn test_priority_queue_fifo_same_priority() { - let mut queue = PriorityHealQueue::new(); - - // Add multiple requests with same priority - let req1 = HealRequest::new( - HealType::Bucket { - bucket: "bucket1".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let req2 = HealRequest::new( - HealType::Bucket { - bucket: "bucket2".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let req3 = HealRequest::new( - HealType::Bucket { - bucket: "bucket3".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let id1 = req1.id.clone(); - let id2 = req2.id.clone(); - let id3 = req3.id.clone(); - - assert_eq!(queue.push(req1), QueuePushOutcome::Accepted); - assert_eq!(queue.push(req2), QueuePushOutcome::Accepted); - assert_eq!(queue.push(req3), QueuePushOutcome::Accepted); - - // Should maintain FIFO order for same priority - let popped1 = queue.pop().unwrap(); - assert_eq!(popped1.id, id1); - - let popped2 = queue.pop().unwrap(); - assert_eq!(popped2.id, id2); - - let popped3 = queue.pop().unwrap(); - assert_eq!(popped3.id, id3); - } - - #[test] - fn test_priority_queue_deduplication() { - let mut queue = PriorityHealQueue::new(); - - let req1 = HealRequest::new( - HealType::Object { - bucket: "bucket1".to_string(), - object: "object1".to_string(), - version_id: None, - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let req2 = HealRequest::new( - HealType::Object { - bucket: "bucket1".to_string(), - object: "object1".to_string(), - version_id: None, - }, - HealOptions::default(), - HealPriority::High, - ); - - // First request should be added - assert_eq!(queue.push(req1), QueuePushOutcome::Accepted); - assert_eq!(queue.len(), 1); - - // Second request with same object should be rejected (duplicate) - assert_eq!(queue.push(req2), QueuePushOutcome::Merged); - assert_eq!(queue.len(), 1); - } - - #[test] - fn test_priority_queue_contains_erasure_set() { - let mut queue = PriorityHealQueue::new(); - - let req = HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket1".to_string()], - set_disk_id: "pool_0_set_1".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - assert_eq!(queue.push(req), QueuePushOutcome::Accepted); - assert!(queue.contains_erasure_set("pool_0_set_1")); - assert!(!queue.contains_erasure_set("pool_0_set_2")); - } - - #[test] - fn test_priority_queue_dedup_key_generation() { - // Test different heal types generate different keys - let obj_req = HealRequest::new( - HealType::Object { - bucket: "bucket1".to_string(), - object: "object1".to_string(), - version_id: None, - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let bucket_req = HealRequest::new( - HealType::Bucket { - bucket: "bucket1".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let erasure_req = HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket1".to_string()], - set_disk_id: "pool_0_set_1".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let obj_key = PriorityHealQueue::make_dedup_key(&obj_req); - let bucket_key = PriorityHealQueue::make_dedup_key(&bucket_req); - let erasure_key = PriorityHealQueue::make_dedup_key(&erasure_req); - - // All keys should be different - assert_ne!(obj_key, bucket_key); - assert_ne!(obj_key, erasure_key); - assert_ne!(bucket_key, erasure_key); - - assert!(obj_key.starts_with("object:")); - assert!(bucket_key.starts_with("bucket:")); - assert!(erasure_key.starts_with("erasure_set:")); - } - - #[test] - fn test_priority_queue_mixed_priorities_and_types() { - let mut queue = PriorityHealQueue::new(); - - // Add various requests - let requests = vec![ - ( - HealType::Object { - bucket: "b1".to_string(), - object: "o1".to_string(), - version_id: None, - }, - HealPriority::Low, - ), - ( - HealType::Bucket { - bucket: "b2".to_string(), - }, - HealPriority::Urgent, - ), - ( - HealType::ErasureSet { - buckets: vec!["b3".to_string()], - set_disk_id: "pool_0_set_1".to_string(), - }, - HealPriority::Normal, - ), - ( - HealType::Object { - bucket: "b4".to_string(), - object: "o4".to_string(), - version_id: None, - }, - HealPriority::High, - ), - ]; - - for (heal_type, priority) in requests { - let req = HealRequest::new(heal_type, HealOptions::default(), priority); - let outcome = queue.push(req); - assert_eq!(outcome, QueuePushOutcome::Accepted); - } - - assert_eq!(queue.len(), 4); - - // Check they come out in priority order - let priorities: Vec = (0..4).filter_map(|_| queue.pop().map(|r| r.priority)).collect(); - - assert_eq!( - priorities, - vec![ - HealPriority::Urgent, - HealPriority::High, - HealPriority::Normal, - HealPriority::Low, - ] - ); - } - - #[test] - fn test_priority_queue_stats() { - let mut queue = PriorityHealQueue::new(); - - // Add requests with different priorities - for _ in 0..3 { - assert_eq!( - queue.push(HealRequest::new( - HealType::Bucket { - bucket: format!("bucket-low-{}", queue.len()), - }, - HealOptions::default(), - HealPriority::Low, - )), - QueuePushOutcome::Accepted - ); - } - - for _ in 0..2 { - assert_eq!( - queue.push(HealRequest::new( - HealType::Bucket { - bucket: format!("bucket-normal-{}", queue.len()), - }, - HealOptions::default(), - HealPriority::Normal, - )), - QueuePushOutcome::Accepted - ); - } - - assert_eq!( - queue.push(HealRequest::new( - HealType::Bucket { - bucket: "bucket-high".to_string(), - }, - HealOptions::default(), - HealPriority::High, - )), - QueuePushOutcome::Accepted - ); - - let stats = queue.get_priority_stats(); - - assert_eq!(*stats.get(&HealPriority::Low).unwrap_or(&0), 3); - assert_eq!(*stats.get(&HealPriority::Normal).unwrap_or(&0), 2); - assert_eq!(*stats.get(&HealPriority::High).unwrap_or(&0), 1); - assert_eq!(*stats.get(&HealPriority::Urgent).unwrap_or(&0), 0); - } - - #[test] - fn test_priority_queue_is_empty() { - let mut queue = PriorityHealQueue::new(); - - assert!(queue.is_empty()); - - assert_eq!( - queue.push(HealRequest::new( - HealType::Bucket { - bucket: "test".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - )), - QueuePushOutcome::Accepted - ); - - assert!(!queue.is_empty()); - - queue.pop(); - - assert!(queue.is_empty()); - } - - #[test] - fn test_priority_queue_pop_runnable_skips_blocked_erasure_set() { - let mut queue = PriorityHealQueue::new(); - - let blocked = HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket-a".to_string()], - set_disk_id: "pool_0_set_1".to_string(), - }, - HealOptions::default(), - HealPriority::Urgent, - ); - let runnable = HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket-b".to_string()], - set_disk_id: "pool_0_set_2".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted); - assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted); - - let mut running = HashMap::new(); - running.insert("pool_0_set_1".to_string(), 1); - - let popped = queue - .pop_runnable(|request| can_schedule_request(request, &running, 1)) - .expect("should find runnable request"); - - assert!(matches!( - popped.heal_type, - HealType::ErasureSet { ref set_disk_id, .. } if set_disk_id == "pool_0_set_2" - )); - } - - #[test] - fn test_can_schedule_request_respects_per_set_limit() { - let request = HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket".to_string()], - set_disk_id: "pool_0_set_1".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - let mut running = HashMap::new(); - running.insert("pool_0_set_1".to_string(), 1); - - assert!(!can_schedule_request(&request, &running, 1)); - assert!(can_schedule_request(&request, &running, 2)); - } - - #[test] - fn test_can_schedule_scoped_object_request_respects_per_set_limit() { - let options = HealOptions { - pool_index: Some(0), - set_index: Some(1), - ..Default::default() - }; - let request = HealRequest::new( - HealType::Object { - bucket: "bucket".to_string(), - object: "object".to_string(), - version_id: None, - }, - options, - HealPriority::Normal, - ); - - let mut running = HashMap::new(); - running.insert("pool_0_set_1".to_string(), 1); - - assert!(!can_schedule_request(&request, &running, 1)); - assert!(can_schedule_request(&request, &running, 2)); - } - - #[tokio::test] - async fn test_submit_heal_request_returns_merged_for_duplicate() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let request = HealRequest::new( - HealType::Object { - bucket: "bucket".to_string(), - object: "object".to_string(), - version_id: None, - }, - HealOptions::default(), - HealPriority::Low, - ); - - assert_eq!( - manager - .submit_heal_request(request.clone()) - .await - .expect("first request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(request) - .await - .expect("duplicate request should produce admission result"), - HealAdmissionResult::Merged - ); - } - - #[tokio::test] - async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None); - original.source = HealRequestSource::Admin; - let original_id = original.id.clone(); - let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None); - duplicate.source = HealRequestSource::Admin; - let duplicate_id = duplicate.id.clone(); - - let accepted = manager - .submit_heal_request_with_receipt(original) - .await - .expect("first request should be accepted"); - let merged = manager - .submit_heal_request_with_receipt(duplicate) - .await - .expect("duplicate request should merge"); - - assert_eq!(accepted.result, HealAdmissionResult::Accepted); - assert_eq!(accepted.task_id, original_id); - assert_eq!(merged.result, HealAdmissionResult::Merged); - assert_eq!(merged.task_id, original_id); - assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id); - } - - #[tokio::test] - async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() { - let storage: Arc = Arc::new(MockStorage); - let manager = Arc::new(HealManager::new(storage.clone(), None)); - let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None); - original.source = HealRequestSource::Admin; - let original_id = original.id.clone(); - assert_eq!( - manager - .submit_heal_request(original) - .await - .expect("original request should be accepted"), - HealAdmissionResult::Accepted - ); - - let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None); - duplicate.source = HealRequestSource::Admin; - let hook = Arc::new(DuplicateAdmissionTestHook { - request_id: duplicate.id.clone(), - active_lock_reached: Notify::new(), - active_lock_release: Notify::new(), - }); - *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = Some(hook.clone()); - - let duplicate_manager = manager.clone(); - let duplicate_task = tokio::spawn(async move { duplicate_manager.submit_heal_request_with_receipt(duplicate).await }); - tokio::time::timeout(Duration::from_secs(1), hook.active_lock_reached.notified()) - .await - .expect("duplicate admission should reach the active lock hook"); - - let transition_manager = manager.clone(); - let transition_storage = storage.clone(); - let (attempting_active_tx, attempting_active_rx) = tokio::sync::oneshot::channel(); - let (active_acquired_tx, mut active_acquired_rx) = tokio::sync::oneshot::channel(); - let transition = tokio::spawn(async move { - let _ = attempting_active_tx.send(()); - let mut active = transition_manager.active_heals.lock().await; - let _ = active_acquired_tx.send(()); - let mut queue = transition_manager.heal_queue.lock().await; - let request = queue.pop_next().expect("original request should remain queued"); - let task = Arc::new(HealTask::from_request(request, transition_storage)); - active.insert(task.id.clone(), task); - }); - tokio::time::timeout(Duration::from_secs(1), attempting_active_rx) - .await - .expect("transition should attempt the active lock") - .expect("transition attempt signal should be delivered"); - assert!(matches!( - active_acquired_rx.try_recv(), - Err(tokio::sync::oneshot::error::TryRecvError::Empty) - )); - - hook.active_lock_release.notify_one(); - let receipt = tokio::time::timeout(Duration::from_secs(1), duplicate_task) - .await - .expect("duplicate admission should not hang") - .expect("duplicate task should join") - .expect("duplicate admission should succeed"); - tokio::time::timeout(Duration::from_secs(1), transition) - .await - .expect("queue to active transition should not hang") - .expect("queue to active transition should join"); - *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = None; - - assert_eq!(receipt.result, HealAdmissionResult::Merged); - assert_eq!(receipt.task_id, original_id); - assert_eq!(manager.get_queue_length().await, 0); - assert_eq!(manager.get_active_task_count().await, 1); - } - - #[tokio::test] - async fn test_submit_heal_request_returns_merged_for_active_duplicate() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); - let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - let active_task = Arc::new(HealTask::from_request(active_request, storage)); - manager.active_heals.lock().await.insert(active_task.id.clone(), active_task); - - let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - - assert_eq!( - manager - .submit_heal_request(duplicate_request) - .await - .expect("active duplicate should produce admission result"), - HealAdmissionResult::Merged - ); - assert_eq!(manager.get_queue_length().await, 0); - } - - #[tokio::test] - async fn test_active_duplicate_token_can_query_and_cancel_original_task() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); - let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - let active_task = Arc::new(HealTask::from_request(active_request, storage)); - let active_task_id = active_task.id.clone(); - manager.active_heals.lock().await.insert(active_task_id.clone(), active_task); - - let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - let duplicate_task_id = duplicate_request.id.clone(); - - assert_eq!( - manager - .submit_heal_request(duplicate_request) - .await - .expect("active duplicate should produce admission result"), - HealAdmissionResult::Merged - ); - assert_eq!( - manager - .get_task_status_for_path("bucket/object", &duplicate_task_id) - .await - .expect("duplicate token should query merged active task"), - HealTaskStatus::Pending - ); - - manager - .cancel_task(&duplicate_task_id) - .await - .expect("duplicate token should cancel merged active task"); - - assert!(manager.active_heals.lock().await.get(&active_task_id).is_none()); - assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. }))); - } - - #[tokio::test] - async fn test_queued_duplicate_token_can_query_and_cancel_original_request() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - let original_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - let original_task_id = original_request.id.clone(); - let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - let duplicate_task_id = duplicate_request.id.clone(); - - assert_eq!( - manager - .submit_heal_request(original_request) - .await - .expect("original request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(duplicate_request) - .await - .expect("queued duplicate should produce admission result"), - HealAdmissionResult::Merged - ); - assert_eq!( - manager - .get_task_status_for_path("bucket/object", &duplicate_task_id) - .await - .expect("duplicate token should query merged queued task"), - HealTaskStatus::Pending - ); - - manager - .cancel_task(&duplicate_task_id) - .await - .expect("duplicate token should cancel merged queued request"); - - assert!(matches!( - manager.get_task_status(&original_task_id).await, - Err(Error::TaskNotFound { .. }) - )); - } - - #[test] - fn test_retry_request_for_recoverable_lock_timeout() { - let storage: Arc = Arc::new(MockStorage); - let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); - let result = Err(Error::TaskExecutionFailed { - message: "Failed to heal object bucket/object: Lock acquisition timeout".to_string(), - }); - - let (retry_request, retry_delay, retry_error) = - retry_request_for_result(&task, &result).expect("lock timeout should be retryable"); - - assert_eq!(retry_request.id, task.id); - assert_eq!(retry_request.retry_attempts, 1); - assert_eq!(retry_request.priority, task.priority); - assert!(retry_delay > Duration::ZERO); - assert!(retry_error.contains("Lock acquisition timeout")); - } - - #[tokio::test] - async fn retry_request_for_result_preserves_remaining_timeout_budget() { - let storage: Arc = Arc::new(MockStorage); - let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); - request.options.timeout = Some(Duration::from_secs(60)); - let task = HealTask::from_request(request, storage); - let result = task.execute().await; - - let (retry_request, _, _) = retry_request_for_result_with_budget(&task, &result) - .await - .expect("read quorum failure should retain the unused timeout budget"); - let remaining = retry_request - .options - .timeout - .expect("configured timeout should remain present"); - assert!(remaining < Duration::from_secs(60)); - assert!(remaining > Duration::from_secs(59)); - } - - #[test] - fn test_retry_request_for_incomplete_heal_rename() { - let storage: Arc = Arc::new(MockStorage); - let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); - let result = Err(Error::TaskExecutionFailed { - message: "Failed to heal object bucket/object: heal rename incomplete: 1 of 2 targets committed".to_string(), - }); - - let (retry_request, retry_delay, retry_error) = - retry_request_for_result(&task, &result).expect("incomplete target rename should be retryable"); - - assert_eq!(retry_request.id, task.id); - assert_eq!(retry_request.retry_attempts, 1); - assert!(retry_delay > Duration::ZERO); - assert!(retry_error.contains("heal rename incomplete")); - } - - #[test] - fn test_retry_request_for_typed_read_quorum_error() { - let storage: Arc = Arc::new(MockStorage); - let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); - let result = Err(Error::Storage(EcstoreError::InsufficientReadQuorum( - "bucket".to_string(), - "object".to_string(), - ))); - - let (retry_request, retry_delay, retry_error) = - retry_request_for_result(&task, &result).expect("typed read quorum should be retryable"); - - assert_eq!(retry_request.id, task.id); - assert_eq!(retry_request.retry_attempts, 1); - assert!(retry_delay > Duration::ZERO); - assert!(retry_error.contains("Storage resources are insufficient")); - } - - #[test] - fn test_retry_request_for_durable_replacement_retry_signal() { - let storage: Arc = Arc::new(MockStorage); - let task = HealTask::from_request( - HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket".to_string()], - set_disk_id: "pool_0_set_0".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ), - storage, - ); - let result = Err(Error::transient_skip("Replacement erasure set heal incomplete; retry scheduled")); - - let (retry_request, retry_delay, retry_error) = - retry_request_for_result(&task, &result).expect("typed replacement retry signal must be scheduled"); - - assert_eq!(retry_request.id, task.id); - assert_eq!(retry_request.retry_attempts, 1); - assert!(retry_delay > Duration::ZERO); - assert!(retry_error.contains("Replacement erasure set heal incomplete")); - } - - #[test] - fn durable_replacement_recovery_re_admits_only_the_matching_generation() { - let task_id = "replacement-generation"; - let mut state = crate::heal::resume::ResumeState::new( - task_id.to_string(), - "erasure_set".to_string(), - "pool_0_set_0".to_string(), - vec!["bucket".to_string()], - ); - state.replacement_generation = Some(task_id.to_string()); - state.replacement_phase = ReplacementPhase::Intent; - state.replacement_targets = vec!["replacement-a".to_string()]; - assert!(!durable_replacement_recovery_is_due(&state, task_id)); - - state.retry_count = state.max_retries; - assert!(durable_replacement_recovery_is_due(&state, task_id)); - - state.completed = true; - state.retry_count = 0; - state.replacement_phase = ReplacementPhase::Verified; - assert!( - durable_replacement_recovery_is_due(&state, task_id), - "verified terminal cleanup must be re-admitted without re-running recovery" - ); - - state.replacement_phase = ReplacementPhase::CleanupPending; - assert!( - durable_replacement_recovery_is_due(&state, task_id), - "cleanup-pending terminal cleanup must be periodically re-admitted" - ); - - state.completed = false; - state.replacement_generation = Some("another-generation".to_string()); - assert!( - !durable_replacement_recovery_is_due(&state, task_id), - "a task must not adopt another generation's durable intent" - ); - - state.replacement_generation = Some(task_id.to_string()); - state.replacement_targets.clear(); - assert!( - !durable_replacement_recovery_is_due(&state, task_id), - "a durable retry without a target is not safe to re-admit" - ); - - state.replacement_targets = vec!["replacement-a".to_string()]; - state.replacement_phase = ReplacementPhase::Verified; - assert!( - !durable_replacement_recovery_is_due(&state, task_id), - "a task must not adopt another generation's terminal cleanup" - ); - } - - #[test] - fn replacement_recovery_blocker_is_set_scoped() { - let manager = HealManager::new(Arc::new(MockStorage), None); - - manager.block_replacement_recovery_set("pool_0_set_0"); - - assert!(manager.replacement_recovery_set_is_blocked("pool_0_set_0")); - assert!(!manager.replacement_recovery_set_is_blocked("pool_0_set_1")); - } - - #[test] - fn replacement_recovery_blocks_only_confirmed_conflicts() { - assert!(crate::heal::resume::replacement_recovery_error_requires_block( - &Error::TaskExecutionFailed { - message: "replacement recovery conflict: proof mismatch".to_string(), - } - )); - assert!(crate::heal::resume::replacement_recovery_error_requires_block( - &Error::TaskExecutionFailed { - message: "replacement recovery corruption: malformed legacy intent".to_string(), - } - )); - assert!(!crate::heal::resume::replacement_recovery_error_requires_block(&Error::Disk( - DiskError::Timeout - ))); - assert!(!crate::heal::resume::replacement_recovery_error_requires_block( - &Error::TaskExecutionFailed { - message: "Failed to list replacement recovery records: temporary I/O error".to_string(), - } - )); - } - - #[test] - fn replacement_recovery_discovery_unformatted_is_quiet_only_for_deferred_endpoint() { - let error = Error::Disk(DiskError::UnformattedDisk); - let deferred = HashSet::from(["endpoint-a".to_string()]); - - assert!(replacement_discovery_error_is_expected_for_deferred_endpoint( - &error, - "endpoint-a", - &deferred - )); - assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint( - &error, - "endpoint-b", - &deferred - )); - assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint( - &Error::Disk(DiskError::Timeout), - "endpoint-a", - &deferred - )); - } - - #[test] - fn replacement_recovery_retry_barrier_requires_all_set_records_to_validate() { - let mut blocked = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]); - let retry_succeeded = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]); - let retry_failed = HashSet::from(["pool_0_set_0".to_string()]); - - unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed); - - assert!( - blocked.contains("pool_0_set_0"), - "one failed disk record must keep the whole replacement set blocked" - ); - assert!( - !blocked.contains("pool_0_set_1"), - "a blocked set may resume only after every retried record validates" - ); - } - - #[tokio::test] - async fn scheduler_completes_cleanup_pending_recovery_from_manager_anchor() { - let temp = TempDir::new().expect("temporary manager recovery directory should be created"); - let anchor = make_manager_resume_disk(&temp, "anchor").await; - let task_id = ResumeUtils::generate_task_id(); - let target = "replacement-a".to_string(); - let identity = ReplacementTargetIdentity { - endpoint: target.clone(), - canonical_path: "/replacement/replacement-a".to_string(), - physical_device_ids: vec!["replacement-a".to_string()], - filesystem_identity: "identity-replacement-a".to_string(), - }; - let resume_manager = ResumeManager::new_replacement_intent( - anchor.clone(), - task_id.clone(), - "pool_0_set_0".to_string(), - vec!["bucket-a".to_string()], - vec![target.clone()], - vec![identity], - ) - .await - .expect("cleanup-pending replacement state should persist on the survivor anchor"); - resume_manager - .mark_replacement_completed_and_verified() - .await - .expect("completion proof should persist before cleanup"); - resume_manager - .mark_replacement_cleanup_pending() - .await - .expect("cleanup-pending state should persist before restart"); - CheckpointManager::new(anchor.clone(), task_id.clone()) - .await - .expect("checkpoint fixture should persist"); - - let (hook, _hook_guard) = ManagerRecoveryTestHook::install(anchor.clone()); - let storage = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); - let mut request = HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket-a".to_string()], - set_disk_id: "pool_0_set_0".to_string(), - }, - HealOptions { - pool_index: Some(0), - set_index: Some(0), - ..HealOptions::default() - }, - HealPriority::Low, - ); - request.id = task_id.clone(); - request.source = HealRequestSource::AutoHeal; - request.heal_endpoints = vec![target]; - assert_eq!( - manager - .submit_heal_request(request) - .await - .expect("durable recovery request should be admitted"), - HealAdmissionResult::Accepted - ); - manager - .replacement_recovery_anchors - .lock() - .expect("replacement recovery anchor lock should not poison") - .insert(task_id.clone(), anchor.endpoint().to_string()); - - process_manager_queue_once(&manager).await; - tokio::time::timeout(Duration::from_secs(2), async { - loop { - let resume_removed = !ResumeManager::has_resume_state(&anchor, &task_id).await; - let checkpoint_removed = !CheckpointManager::has_checkpoint(&anchor, &task_id).await; - let anchor_removed = !manager - .replacement_recovery_anchors - .lock() - .expect("replacement recovery anchor lock should not poison") - .contains_key(&task_id); - if resume_removed && checkpoint_removed && anchor_removed { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("cleanup-pending recovery should finish through the manager scheduler"); - - assert!( - !*hook.listed.lock().expect("manager recovery listed lock should not poison"), - "cleanup-pending recovery must not list buckets or restart object healing" - ); - assert_eq!( - *hook - .global_format_calls - .lock() - .expect("manager recovery global format call lock should not poison"), - 0 - ); - assert_eq!( - *hook - .replacement_format_calls - .lock() - .expect("manager recovery replacement format call lock should not poison"), - 0, - "manager-resumed terminal cleanup must not format replacement targets" - ); - assert_eq!( - *hook - .bucket_heal_calls - .lock() - .expect("manager recovery bucket call lock should not poison"), - 0 - ); - assert_eq!( - *hook - .heal_object_calls - .lock() - .expect("manager recovery object call lock should not poison"), - 0 - ); - } - - #[test] - fn test_retry_request_for_scoped_slowdown_preserves_scope() { - let storage: Arc = Arc::new(MockStorage); - let task = HealTask::from_request( - HealRequest::new( - HealType::Object { - bucket: "bucket".to_string(), - object: "object".to_string(), - version_id: None, - }, - HealOptions { - pool_index: Some(0), - set_index: Some(1), - ..Default::default() - }, - HealPriority::Normal, - ), - storage, - ); - let result = Err(Error::Storage(EcstoreError::SlowDown)); - - let (retry_request, retry_delay, _) = - retry_request_for_result(&task, &result).expect("SlowDown should defer scoped heal"); - - assert_eq!(retry_request.options.pool_index, Some(0)); - assert_eq!(retry_request.options.set_index, Some(1)); - assert_eq!(retry_request.retry_attempts, 1); - assert!(retry_delay > Duration::ZERO); - } - - #[test] - fn test_retry_request_for_typed_not_found_error_is_not_retryable() { - let storage: Arc = Arc::new(MockStorage); - let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); - let result = Err(Error::Storage(EcstoreError::ObjectNotFound("bucket".to_string(), "object".to_string()))); - - assert!(retry_request_for_result(&task, &result).is_none()); - } - - #[test] - fn test_retry_request_for_recoverable_error_stops_at_limit() { - let storage: Arc = Arc::new(MockStorage); - let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - request.retry_attempts = MAX_RECOVERABLE_HEAL_RETRIES; - let task = HealTask::from_request(request, storage); - let result = Err(Error::TaskExecutionFailed { - message: "Remote lock RPC timed out".to_string(), - }); - - assert!(retry_request_for_result(&task, &result).is_none()); - } - - #[tokio::test] - async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() { - let storage: Arc = Arc::new(MockStorage); - let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage); - let result = Err(task - .record_batch_failure(BatchHealFailure { - scope: "bucket:bucket".to_string(), - failed: 1, - retryable: 1, - permanent: 0, - first_object: "object".to_string(), - first_error: "Lock acquisition timeout".to_string(), - }) - .await); - - assert!(retry_request_for_result(&task, &result).is_none()); - } - - #[test] - fn test_heal_type_matches_path_normalizes_prefix_trailing_slash() { - let heal_type = HealType::Prefix { - bucket: "bucket".to_string(), - prefix: "logs/".to_string(), - }; - - assert!(heal_type_matches_path(&heal_type, "bucket")); - assert!(heal_type_matches_path(&heal_type, "bucket/logs")); - assert!(heal_type_matches_path(&heal_type, "bucket/logs/")); - } - - #[test] - fn test_heal_type_matches_path_normalizes_object_trailing_slash() { - let heal_type = HealType::Object { - bucket: "bucket".to_string(), - object: "object/".to_string(), - version_id: None, - }; - - assert!(heal_type_matches_path(&heal_type, "bucket/object")); - assert!(heal_type_matches_path(&heal_type, "bucket/object/")); - } - - async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> CancellationToken { - let task_id = request.id.clone(); - let cancel_token = CancellationToken::new(); - manager.retrying_heals.lock().await.insert( - task_id.clone(), - RetryingHeal { - request: request.clone(), - error: "Lock acquisition timeout".to_string(), - cancel_token: cancel_token.clone(), - }, - ); - manager.completed_heals.lock().await.insert( - task_id, - Arc::new(CompletedHealStatus { - heal_type: request.heal_type, - status: HealTaskStatus::Retrying { - error: "Lock acquisition timeout".to_string(), - retry_attempt: request.retry_attempts, - }, - result_items_truncated: false, - seqed_items: Vec::new(), - next_seq: 0, - min_seq: 0, - completed_at: SystemTime::now(), - }), - ); - cancel_token - } - - #[tokio::test] - async fn test_cancel_task_cancels_retrying_backoff() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - let mut request = HealRequest::bucket("bucket".to_string()); - request.retry_attempts = 1; - let task_id = request.id.clone(); - let cancel_token = insert_retrying_request(&manager, request).await; - - assert!(matches!( - manager - .get_task_status(&task_id) - .await - .expect("retrying task should be queryable"), - HealTaskStatus::Retrying { .. } - )); - - manager - .cancel_task(&task_id) - .await - .expect("retrying task should be cancellable by token"); - - assert!(cancel_token.is_cancelled()); - assert!(manager.retrying_heals.lock().await.get(&task_id).is_none()); - assert!(matches!(manager.get_task_status(&task_id).await, Err(Error::TaskNotFound { .. }))); - } - - #[tokio::test] - async fn test_cancel_tasks_for_path_cancels_retrying_backoff() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - let mut request = HealRequest::bucket("bucket".to_string()); - request.retry_attempts = 1; - let task_id = request.id.clone(); - let cancel_token = insert_retrying_request(&manager, request).await; - - assert_eq!( - manager - .cancel_tasks_for_path("bucket") - .await - .expect("retrying task should be cancellable by path"), - 1 - ); - - assert!(cancel_token.is_cancelled()); - assert!(manager.retrying_heals.lock().await.get(&task_id).is_none()); - assert!(matches!(manager.get_task_status(&task_id).await, Err(Error::TaskNotFound { .. }))); - } - - #[tokio::test] - async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); - let cluster_request_id = cluster_request.id.clone(); - let bucket_request = HealRequest::bucket("bucket".to_string()); - let bucket_request_id = bucket_request.id.clone(); - - manager - .submit_heal_request(cluster_request) - .await - .expect("cluster request should be accepted"); - manager - .submit_heal_request(bucket_request) - .await - .expect("bucket request should be accepted"); - - assert_eq!( - manager - .cancel_tasks_for_path("") - .await - .expect("root path should cancel queued cluster task"), - 1 - ); - assert!(matches!( - manager.get_task_status(&cluster_request_id).await, - Err(Error::TaskNotFound { .. }) - )); - assert_eq!( - manager - .get_task_status(&bucket_request_id) - .await - .expect("bucket request should not match root path"), - HealTaskStatus::Pending - ); - } - - #[tokio::test] - async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage.clone(), None); - - let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); - let cluster_request_id = cluster_request.id.clone(); - let bucket_request = HealRequest::bucket("bucket".to_string()); - let bucket_request_id = bucket_request.id.clone(); - - manager.active_heals.lock().await.insert( - cluster_request_id.clone(), - Arc::new(HealTask::from_request(cluster_request, storage.clone())), - ); - manager - .active_heals - .lock() - .await - .insert(bucket_request_id.clone(), Arc::new(HealTask::from_request(bucket_request, storage))); - - assert_eq!( - manager - .cancel_tasks_for_path("") - .await - .expect("root path should cancel active cluster task"), - 1 - ); - assert!(manager.active_heals.lock().await.get(&cluster_request_id).is_none()); - assert!(manager.active_heals.lock().await.get(&bucket_request_id).is_some()); - } - - #[tokio::test] - async fn test_cancel_tasks_for_empty_path_cancels_retrying_cluster_only() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let mut cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); - cluster_request.retry_attempts = 1; - let cluster_request_id = cluster_request.id.clone(); - let cluster_cancel_token = insert_retrying_request(&manager, cluster_request).await; - - let mut bucket_request = HealRequest::bucket("bucket".to_string()); - bucket_request.retry_attempts = 1; - let bucket_request_id = bucket_request.id.clone(); - let bucket_cancel_token = insert_retrying_request(&manager, bucket_request).await; - - assert_eq!( - manager - .cancel_tasks_for_path("") - .await - .expect("root path should cancel retrying cluster task"), - 1 - ); - assert!(cluster_cancel_token.is_cancelled()); - assert!(!bucket_cancel_token.is_cancelled()); - assert!(manager.retrying_heals.lock().await.get(&cluster_request_id).is_none()); - assert!(manager.retrying_heals.lock().await.get(&bucket_request_id).is_some()); - } - - #[test] - fn test_heal_type_matches_path_accepts_legacy_root() { - assert!(heal_type_matches_path(&HealType::Cluster, LEGACY_ROOT_HEAL_PATH)); - assert!(!heal_type_matches_path( - &HealType::Bucket { - bucket: "bucket".to_string(), - }, - LEGACY_ROOT_HEAL_PATH, - )); - } - - #[tokio::test] - async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - let mut original_request = HealRequest::bucket("bucket".to_string()); - original_request.retry_attempts = 1; - let original_task_id = original_request.id.clone(); - let cancel_token = insert_retrying_request(&manager, original_request).await; - - let duplicate_request = HealRequest::bucket("bucket".to_string()); - let duplicate_task_id = duplicate_request.id.clone(); - - assert_eq!( - manager - .submit_heal_request(duplicate_request) - .await - .expect("retrying duplicate should produce admission result"), - HealAdmissionResult::Merged - ); - assert!(matches!( - manager - .get_task_status_for_path("bucket", &duplicate_task_id) - .await - .expect("duplicate token should query merged retrying task"), - HealTaskStatus::Retrying { .. } - )); - - manager - .cancel_task(&duplicate_task_id) - .await - .expect("duplicate token should cancel merged retrying task"); - - assert!(cancel_token.is_cancelled()); - assert!(manager.retrying_heals.lock().await.get(&original_task_id).is_none()); - } - - #[tokio::test] - async fn test_get_task_status_reports_pending_for_queued_request() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let request = HealRequest::bucket("bucket".to_string()); - let request_id = request.id.clone(); - - assert_eq!( - manager - .submit_heal_request(request) - .await - .expect("request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .get_task_status(&request_id) - .await - .expect("queued request should have status"), - HealTaskStatus::Pending - ); - } - - #[tokio::test] - async fn test_operations_snapshot_counts_queue_by_source_and_priority() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let mut scanner_request = HealRequest::new( - HealType::Object { - bucket: "bucket-a".to_string(), - object: "object-a".to_string(), - version_id: None, - }, - HealOptions::default(), - HealPriority::Low, - ); - scanner_request.source = HealRequestSource::Scanner; - - let mut admin_request = HealRequest::bucket("bucket-b".to_string()); - admin_request.priority = HealPriority::High; - admin_request.source = HealRequestSource::Admin; - - let mut auto_request = HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket-c".to_string()], - set_disk_id: "0-0".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - auto_request.source = HealRequestSource::AutoHeal; - - manager - .submit_heal_request(scanner_request) - .await - .expect("scanner request should be accepted"); - manager - .submit_heal_request(admin_request) - .await - .expect("admin request should be accepted"); - manager - .submit_heal_request(auto_request) - .await - .expect("auto request should be accepted"); - - let snapshot = manager.operations_snapshot().await; - - assert_eq!(snapshot.queue_length, 3); - assert_eq!(snapshot.active_tasks, 0); - assert_eq!(snapshot.queued_by_priority.low, 1); - assert_eq!(snapshot.queued_by_priority.normal, 1); - assert_eq!(snapshot.queued_by_priority.high, 1); - assert_eq!(snapshot.queued_by_priority.urgent, 0); - assert_eq!(snapshot.queued_by_source.scanner, 1); - assert_eq!(snapshot.queued_by_source.admin, 1); - assert_eq!(snapshot.queued_by_source.auto_heal, 1); - assert_eq!(snapshot.queued_by_source.internal, 0); - } - - // HS-06 (backlog#1870): overlap policy + forceStart semantics. - fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager { - let storage: Arc = Arc::new(MockStorage); - HealManager::new( - storage, - Some(HealConfig { - overlap_policy: policy, - ..Default::default() - }), - ) - } - - fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest { - let mut request = HealRequest::new( - HealType::Prefix { - bucket: bucket.to_string(), - prefix: prefix.to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - request.source = HealRequestSource::Admin; - request - } - - async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String { - let task = Arc::new(HealTask::from_request(request, manager.storage.clone())); - let task_id = task.id.clone(); - manager.active_heals.lock().await.insert(task_id.clone(), task); - task_id - } - - #[tokio::test] - async fn overlap_policy_minio_error_rejects_same_and_containing_paths() { - let manager = manager_with_policy(HealOverlapPolicy::MinioError); - insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; - - // Same target: typed AlreadyRunning. - let same = manager - .submit_heal_request(admin_prefix_request("bucket-a", "logs/")) - .await - .expect("admission must decide"); - assert_eq!( - same, - HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning), - "an identical target must reject with already-running" - ); - - // Contained path: typed OverlappingPaths. - let nested = manager - .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/")) - .await - .expect("admission must decide"); - assert_eq!( - nested, - HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths), - "a path inside the active task's path must reject with overlapping-paths" - ); - - // Containing path (bucket-wide vs nested active): also overlapping. - let wide = manager - .submit_heal_request(admin_prefix_request("bucket-a", "")) - .await - .expect("admission must decide"); - assert_eq!( - wide, - HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths), - "a bucket-wide start overlapping a nested active heal must reject" - ); - - // Disjoint bucket: unaffected. - let disjoint = manager - .submit_heal_request(admin_prefix_request("bucket-b", "logs/")) - .await - .expect("admission must decide"); - assert_eq!(disjoint, HealAdmissionResult::Accepted); - } - - #[tokio::test] - async fn overlap_policy_default_merge_keeps_today_semantics() { - let manager = manager_with_policy(HealOverlapPolicy::Merge); - insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; - - // Different-dedup-key overlap still merges under the default policy: - // the nested path dedups to its own key but nothing rejects it. - let nested = manager - .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/")) - .await - .expect("admission must decide"); - assert_eq!(nested, HealAdmissionResult::Accepted, "default policy must not reject overlaps"); - - // Non-admin sources never get overlap rejections even under minio_error. - let manager = manager_with_policy(HealOverlapPolicy::MinioError); - insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; - let mut scanner_request = admin_prefix_request("bucket-a", "logs/app/"); - scanner_request.source = HealRequestSource::Scanner; - let admitted = manager - .submit_heal_request(scanner_request) - .await - .expect("admission must decide"); - assert_eq!(admitted, HealAdmissionResult::Accepted, "scanner sources must never be overlap-rejected"); - } - - #[tokio::test] - async fn admin_force_start_cancels_overlapping_active_task_first() { - let manager = manager_with_policy(HealOverlapPolicy::Merge); - let old_id = insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; - - let mut replacement = admin_prefix_request("bucket-a", "logs/"); - replacement.force_start = true; - let receipt = manager - .submit_heal_request_with_receipt(replacement) - .await - .expect("force-start submission must decide"); - - assert!(receipt.result.is_admitted(), "the new task must be admitted (Accepted or Merged)"); - let old_task_gone = { - let active_heals = manager.active_heals.lock().await; - !active_heals.contains_key(&old_id) - }; - assert!( - old_task_gone, - "the overlapping admin task must be cancelled (removed from the active table) before the new one starts" - ); - assert!( - matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })), - "a cancelled task must no longer resolve as an active heal" - ); - } - - #[tokio::test] - async fn test_operations_snapshot_counts_active_by_source_and_priority() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let mut request = HealRequest::bucket("bucket-a".to_string()); - request.priority = HealPriority::High; - request.source = HealRequestSource::Admin; - let task = Arc::new(HealTask::from_request(request, manager.storage.clone())); - let task_id = task.id.clone(); - - manager.active_heals.lock().await.insert(task_id, task); - - let snapshot = manager.operations_snapshot().await; - - assert_eq!(snapshot.queue_length, 0); - assert_eq!(snapshot.active_tasks, 1); - assert_eq!(snapshot.active_by_priority.high, 1); - assert_eq!(snapshot.active_by_source.admin, 1); - assert_eq!(snapshot.active_by_source.scanner, 0); - } - - #[tokio::test] - async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - let mut request = HealRequest::bucket("bucket-retry".to_string()); - request.priority = HealPriority::Urgent; - request.source = HealRequestSource::Admin; - let task_id = request.id.clone(); - manager.retrying_heals.lock().await.insert( - task_id, - RetryingHeal { - request, - error: "transient".to_string(), - cancel_token: CancellationToken::new(), - }, - ); - - let snapshot = manager.operations_snapshot().await; - - assert_eq!(snapshot.queue_length, 0); - assert_eq!(snapshot.active_tasks, 0); - assert_eq!(snapshot.retrying_tasks, 1); - assert_eq!(snapshot.retrying_by_priority.urgent, 1); - assert_eq!(snapshot.retrying_by_source.admin, 1); - } - - #[tokio::test] - async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() { - let storage: Arc = Arc::new(MockStorage); - let manager = Arc::new(HealManager::new(storage, None)); - { - let mut config = manager.config.write().await; - config.enable_auto_heal = false; - config.heal_interval = Duration::from_millis(10); - config.event_driven_scheduler_enable = true; - } - manager.start().await.expect("manager should start"); - - let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); - let task_id = request.id.clone(); - let hook = Arc::new(RetryOwnershipTestHook { - task_id: task_id.clone(), - active_to_retrying_reached: Notify::new(), - active_to_retrying_release: Notify::new(), - retrying_to_queue_reached: Notify::new(), - retrying_to_queue_release: Notify::new(), - }); - *RETRY_OWNERSHIP_TEST_HOOK.lock().await = Some(hook.clone()); - - manager - .submit_heal_request(request) - .await - .expect("retry test request should be accepted"); - tokio::time::timeout(Duration::from_secs(2), hook.active_to_retrying_reached.notified()) - .await - .expect("active to retrying transition should be reached"); - - let snapshot_manager = manager.clone(); - let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await }); - assert!( - tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(), - "snapshot must wait while active and retrying ownership locks are held" - ); - hook.active_to_retrying_release.notify_one(); - let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot) - .await - .expect("snapshot should resume after active to retrying handoff") - .expect("snapshot task should complete"); - assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1); - assert_eq!(snapshot.retrying_tasks, 1); - - tokio::time::timeout(Duration::from_secs(5), hook.retrying_to_queue_reached.notified()) - .await - .expect("retrying to queue transition should be reached"); - let snapshot_manager = manager.clone(); - let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await }); - assert!( - tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(), - "snapshot must wait while queue ownership is transferred" - ); - hook.retrying_to_queue_release.notify_one(); - // The scheduler may immediately execute the retried request again; - // leave a permit so a second test-only active handoff cannot stall it. - hook.active_to_retrying_release.notify_one(); - let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot) - .await - .expect("snapshot should resume after retrying to queue handoff") - .expect("snapshot task should complete"); - assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1); - - *RETRY_OWNERSHIP_TEST_HOOK.lock().await = None; - hook.active_to_retrying_release.notify_one(); - hook.retrying_to_queue_release.notify_one(); - tokio::time::timeout(Duration::from_secs(1), manager.stop()) - .await - .expect("manager stop should not stall") - .expect("manager should stop"); - } - - #[tokio::test] - async fn test_active_progress_snapshot_sums_active_task_progress() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let first = Arc::new(HealTask::from_request( - HealRequest::bucket("bucket-a".to_string()), - manager.storage.clone(), - )); - { - let mut progress = first.progress.write().await; - progress.start_time = Some(SystemTime::now() - Duration::from_secs(20)); - progress.set_total_baseline(12, 8192); - progress.update_progress(7, 3, 1, 4096); - } - - let second = Arc::new(HealTask::from_request( - HealRequest::bucket("bucket-b".to_string()), - manager.storage.clone(), - )); - { - let mut progress = second.progress.write().await; - progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); - progress.set_total_baseline(8, 4096); - progress.update_progress(11, 5, 2, 2048); - } - - manager.active_heals.lock().await.insert(first.id.clone(), first); - manager.active_heals.lock().await.insert(second.id.clone(), second); - - let progress = manager - .active_progress_snapshot() - .await - .expect("active progress should exist"); - - assert_eq!(progress.objects_scanned, 18); - assert_eq!(progress.objects_healed, 8); - assert_eq!(progress.objects_failed, 3); - assert_eq!(progress.objects_total_count, 20); - assert_eq!(progress.objects_total_size, 12288); - assert_eq!(progress.bytes_processed, 6144); - assert!((progress.progress_percentage - 50.0).abs() < 0.001); - assert!(progress.estimated_completion_time.is_some()); - } - - #[tokio::test] - async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - manager - .submit_heal_request(HealRequest::bucket("bucket".to_string())) - .await - .expect("request should be accepted"); - - assert!(matches!( - manager.get_task_status_for_path("bucket", "wrong-token").await, - Err(Error::InvalidClientToken) - )); - } - - #[tokio::test] - async fn test_get_task_status_for_path_rejects_token_from_other_active_path() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let bucket_request = HealRequest::bucket("bucket".to_string()); - let other_request = HealRequest::bucket("other".to_string()); - let other_request_id = other_request.id.clone(); - - manager - .submit_heal_request(bucket_request) - .await - .expect("bucket request should be accepted"); - manager - .submit_heal_request(other_request) - .await - .expect("other request should be accepted"); - - assert!(matches!( - manager.get_task_status_for_path("bucket", &other_request_id).await, - Err(Error::InvalidClientToken) - )); - } - - #[tokio::test] - async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let request = HealRequest::bucket("bucket".to_string()); - let request_id = request.id.clone(); - - manager - .submit_heal_request(request) - .await - .expect("request should be accepted"); - - assert!(matches!( - manager.get_task_status_for_path("other", &request_id).await, - Err(Error::TaskNotFound { .. }) - )); - } - - #[tokio::test] - async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - assert!(matches!( - manager.get_task_status_for_path("bucket", "old-token").await, - Err(Error::TaskNotFound { .. }) - )); - } - - #[tokio::test] - async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let request = HealRequest::bucket("bucket".to_string()); - let request_id = request.id.clone(); - - manager - .submit_heal_request(request) - .await - .expect("request should be accepted"); - - assert!(matches!( - manager.get_task_status_for_path("", &request_id).await, - Err(Error::TaskNotFound { .. }) - )); - assert!(matches!( - manager.get_task_status_for_path("", "wrong-token").await, - Err(Error::TaskNotFound { .. }) - )); - } - - #[tokio::test] - async fn test_get_task_report_queries_queued_task_by_token_without_path() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let request = HealRequest::new( - HealType::ErasureSet { - buckets: vec![], - set_disk_id: "pool_0_set_1".to_string(), - }, - HealOptions::default(), - HealPriority::High, - ); - let request_id = request.id.clone(); - - manager - .submit_heal_request(request) - .await - .expect("request should be accepted"); - - let report = manager - .get_task_report(&request_id) - .await - .expect("queued task should be queryable by token"); - - assert_eq!(report.status, HealTaskStatus::Pending); - assert!(report.result_items.is_empty()); - } - - #[tokio::test] - async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - // A completed entry recorded in a Retrying state for a task whose - // request is also (still) queued under the same id: the retrying - // completion must win the lookup, or the task would read back as - // Pending while it is actually waiting out a retry backoff. - let request = HealRequest::object("bucket".to_string(), "object".to_string(), None); - let task_id = request.id.clone(); - manager.completed_heals.lock().await.insert( - task_id.clone(), - Arc::new(CompletedHealStatus { - heal_type: request.heal_type.clone(), - status: HealTaskStatus::Retrying { - error: "transient disk failure".to_string(), - retry_attempt: 1, - }, - result_items_truncated: false, - seqed_items: Vec::new(), - next_seq: 0, - min_seq: 0, - completed_at: SystemTime::now(), - }), - ); - manager.heal_queue.lock().await.push(HealRequest { - id: task_id.clone(), - heal_type: request.heal_type, - ..request - }); - - assert_eq!( - manager.get_task_status(&task_id).await.expect("task must resolve"), - HealTaskStatus::Retrying { - error: "transient disk failure".to_string(), - retry_attempt: 1 - } - ); - } - - #[tokio::test] - async fn test_get_task_status_reads_recent_completed_status() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - manager.completed_heals.lock().await.insert( - "completed-token".to_string(), - Arc::new(CompletedHealStatus { - heal_type: HealType::Bucket { - bucket: "bucket".to_string(), - }, - status: HealTaskStatus::Completed, - result_items_truncated: false, - seqed_items: Vec::new(), - next_seq: 0, - min_seq: 0, - completed_at: SystemTime::now(), - }), - ); - - assert_eq!( - manager - .get_task_status_for_path("bucket", "completed-token") - .await - .expect("recent completed task should be queryable"), - HealTaskStatus::Completed - ); - } - - #[tokio::test] - async fn test_get_task_report_for_path_reads_completed_items() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - manager.completed_heals.lock().await.insert( - "completed-token".to_string(), - Arc::new(CompletedHealStatus { - heal_type: HealType::Object { - bucket: "bucket".to_string(), - object: "object".to_string(), - version_id: None, - }, - status: HealTaskStatus::Completed, - result_items_truncated: true, - seqed_items: vec![( - 1, - HealResultItem { - bucket: "bucket".to_string(), - object: "object".to_string(), - object_size: 1024, - ..Default::default() - }, - )], - next_seq: 2, - min_seq: 1, - completed_at: SystemTime::now(), - }), - ); - - let report = manager - .get_task_report_for_path("bucket/object", "completed-token") - .await - .expect("recent completed task report should be queryable"); - assert!(report.result_items_truncated); - - assert_eq!(report.status, HealTaskStatus::Completed); - assert_eq!(report.result_items.len(), 1); - assert_eq!(report.result_items[0].object_size, 1024); - // The archived cursors pass through to the report so an incremental - // consumer can resume against the next expected sequence. - assert_eq!(report.next_seq, 2); - assert_eq!(report.min_seq, 1); - } - - #[tokio::test] - async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - manager - .submit_heal_request(HealRequest::bucket("bucket".to_string())) - .await - .expect("request should be accepted"); - - assert!(matches!( - manager.get_task_report_for_path("", "wrong-token").await, - Err(Error::TaskNotFound { .. }) - )); - } - - #[tokio::test] - async fn test_cancel_task_removes_queued_request() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let request = HealRequest::bucket("bucket".to_string()); - let request_id = request.id.clone(); - - manager - .submit_heal_request(request) - .await - .expect("request should be accepted"); - manager - .cancel_task(&request_id) - .await - .expect("queued request should be cancelled"); - - assert!(matches!(manager.get_task_status(&request_id).await, Err(Error::TaskNotFound { .. }))); - } - - #[tokio::test] - async fn test_cancel_tasks_for_path_removes_matching_queued_requests() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new(storage, None); - - let bucket_request = HealRequest::bucket("bucket".to_string()); - let bucket_request_id = bucket_request.id.clone(); - let other_request = HealRequest::bucket("other".to_string()); - let other_request_id = other_request.id.clone(); - - manager - .submit_heal_request(bucket_request) - .await - .expect("bucket request should be accepted"); - manager - .submit_heal_request(other_request) - .await - .expect("other request should be accepted"); - - assert_eq!( - manager - .cancel_tasks_for_path("bucket") - .await - .expect("matching request should be cancelled"), - 1 - ); - assert!(matches!( - manager.get_task_status(&bucket_request_id).await, - Err(Error::TaskNotFound { .. }) - )); - assert_eq!( - manager - .get_task_status(&other_request_id) - .await - .expect("unmatched request should remain queued"), - HealTaskStatus::Pending - ); - } - - #[tokio::test] - async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 1, - ..HealConfig::default() - }), - ); - - let request = HealRequest::new( - HealType::Object { - bucket: "bucket".to_string(), - object: "object".to_string(), - version_id: None, - }, - HealOptions::default(), - HealPriority::Low, - ); - - assert_eq!( - manager - .submit_heal_request(request.clone()) - .await - .expect("first request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(request) - .await - .expect("duplicate request should merge even when queue is full"), - HealAdmissionResult::Merged - ); - } - - #[tokio::test] - async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 1, - low_priority_drop_when_full: true, - ..HealConfig::default() - }), - ); - - let accepted = HealRequest::new( - HealType::Bucket { - bucket: "bucket-a".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - let dropped = HealRequest::new( - HealType::Bucket { - bucket: "bucket-b".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - - assert_eq!( - manager - .submit_heal_request(accepted) - .await - .expect("first request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(dropped) - .await - .expect("low priority request should be dropped with explicit admission result"), - HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull) - ); - } - - #[tokio::test] - async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 1, - ..HealConfig::default() - }), - ); - - let accepted = HealRequest::new( - HealType::Bucket { - bucket: "bucket-a".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - let full = HealRequest::new( - HealType::Bucket { - bucket: "bucket-b".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ); - - assert_eq!( - manager - .submit_heal_request(accepted) - .await - .expect("first request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(full) - .await - .expect("normal priority request should surface full admission"), - HealAdmissionResult::Full - ); - } - - #[tokio::test] - async fn test_high_priority_request_displaces_lower_priority_when_queue_full() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 1, - ..HealConfig::default() - }), - ); - - let low = HealRequest::new( - HealType::Bucket { - bucket: "background-bucket".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - let low_id = low.id.clone(); - let high = HealRequest::new( - HealType::Bucket { - bucket: "manual-bucket".to_string(), - }, - HealOptions::default(), - HealPriority::High, - ); - let high_id = high.id.clone(); - - assert_eq!( - manager - .submit_heal_request(low) - .await - .expect("low priority request should be accepted first"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(high) - .await - .expect("high priority request should be admitted by displacing lower priority work"), - HealAdmissionResult::Accepted - ); - assert_eq!(manager.get_queue_length().await, 1); - assert!(matches!(manager.get_task_status(&low_id).await, Err(Error::TaskNotFound { .. }))); - assert_eq!( - manager - .get_task_status(&high_id) - .await - .expect("high priority request should remain queued"), - HealTaskStatus::Pending - ); - } - - #[tokio::test] - async fn test_submit_heal_request_drops_read_repair_under_pressure() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 10, - ..HealConfig::default() - }), - ); - - for index in 0..8 { - assert_eq!( - manager - .submit_heal_request(bucket_request( - &format!("queued-{index}"), - HealPriority::Normal, - HealRequestSource::Internal, - )) - .await - .expect("seed request should be accepted"), - HealAdmissionResult::Accepted - ); - } - - let admission = manager - .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)) - .await - .expect("read repair admission should return a result"); - - assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)); - assert_eq!(manager.get_queue_length().await, 8); - } - - #[tokio::test] - async fn test_submit_heal_request_drops_low_scanner_under_pressure() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 10, - ..HealConfig::default() - }), - ); - - for index in 0..8 { - assert_eq!( - manager - .submit_heal_request(bucket_request( - &format!("queued-{index}"), - HealPriority::Normal, - HealRequestSource::Internal, - )) - .await - .expect("seed request should be accepted"), - HealAdmissionResult::Accepted - ); - } - - let admission = manager - .submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)) - .await - .expect("scanner admission should return a result"); - - assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)); - assert_eq!(manager.get_queue_length().await, 8); - } - - #[tokio::test] - async fn test_submit_heal_request_accepts_admin_high_under_pressure() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 10, - ..HealConfig::default() - }), - ); - - for index in 0..8 { - assert_eq!( - manager - .submit_heal_request(bucket_request( - &format!("queued-{index}"), - HealPriority::Normal, - HealRequestSource::Internal, - )) - .await - .expect("seed request should be accepted"), - HealAdmissionResult::Accepted - ); - } - - let admission = manager - .submit_heal_request(bucket_request("admin", HealPriority::High, HealRequestSource::Admin)) - .await - .expect("admin admission should return a result"); - - assert_eq!(admission, HealAdmissionResult::Accepted); - assert_eq!(manager.get_queue_length().await, 9); - } - - #[tokio::test] - async fn test_mainline_throttle_delays_background_heal_start() { - let storage: Arc = Arc::new(MockStorage); - let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider { - class: WorkloadClass::ForegroundRead, - active: 8, - limit: 10, - state: AdmissionState::Open, - }); - let manager = HealManager::new_with_workload_provider( - storage, - Some(HealConfig { - max_concurrent_heals: 1, - mainline_throttle_enable: true, - mainline_read_utilization_high_percent: 80, - mainline_write_utilization_high_percent: 80, - mainline_max_sleep: Duration::from_millis(1), - ..HealConfig::default() - }), - Some(provider), - ); - - manager - .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)) - .await - .expect("read repair request should be queued"); - - process_manager_queue_once(&manager).await; - - assert_eq!(manager.get_queue_length().await, 1); - assert_eq!(manager.get_active_task_count().await, 0); - } - - #[tokio::test] - async fn test_mainline_throttle_delays_background_heal_start_under_write_pressure() { - let storage: Arc = Arc::new(MockStorage); - let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider { - class: WorkloadClass::ForegroundWrite, - active: 9, - limit: 10, - state: AdmissionState::Open, - }); - let manager = HealManager::new_with_workload_provider( - storage, - Some(HealConfig { - max_concurrent_heals: 1, - mainline_throttle_enable: true, - mainline_read_utilization_high_percent: 80, - mainline_write_utilization_high_percent: 80, - mainline_max_sleep: Duration::from_millis(1), - ..HealConfig::default() - }), - Some(provider), - ); - - manager - .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)) - .await - .expect("read repair request should be queued"); - - process_manager_queue_once(&manager).await; - - assert_eq!(manager.get_queue_length().await, 1); - assert_eq!(manager.get_active_task_count().await, 0); - } - - #[tokio::test] - async fn test_mainline_throttle_allows_admin_high_start() { - let storage: Arc = Arc::new(MockStorage); - let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider { - class: WorkloadClass::ForegroundRead, - active: 10, - limit: 10, - state: AdmissionState::Saturated, - }); - let manager = HealManager::new_with_workload_provider( - storage, - Some(HealConfig { - max_concurrent_heals: 1, - mainline_throttle_enable: true, - mainline_read_utilization_high_percent: 80, - mainline_write_utilization_high_percent: 80, - mainline_max_sleep: Duration::from_millis(1), - ..HealConfig::default() - }), - Some(provider), - ); - - manager - .submit_heal_request(bucket_request("admin", HealPriority::High, HealRequestSource::Admin)) - .await - .expect("admin request should be queued"); - - process_manager_queue_once(&manager).await; - - assert_eq!(manager.get_queue_length().await, 0); - } - - #[tokio::test] - async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - max_concurrent_heals: 1, - task_timeout: Duration::ZERO, - ..HealConfig::default() - }), - ); - - let mut defaulted = bucket_request("defaulted-timeout", HealPriority::Normal, HealRequestSource::Admin); - defaulted.options.timeout = None; - let defaulted_id = defaulted.id.clone(); - manager - .submit_heal_request(defaulted) - .await - .expect("request without timeout should be queued"); - process_manager_queue_once(&manager).await; - let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async { - loop { - if let Ok(status @ HealTaskStatus::Timeout) = manager.get_task_status(&defaulted_id).await { - break status; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("configured timeout should finish the task"); - assert_eq!(defaulted_status, HealTaskStatus::Timeout); - assert!(manager.retrying_heals.lock().await.get(&defaulted_id).is_none()); - - let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin); - explicit.options.timeout = Some(Duration::from_secs(60)); - let explicit_id = explicit.id.clone(); - manager - .submit_heal_request(explicit) - .await - .expect("request with explicit timeout should be queued"); - process_manager_queue_once(&manager).await; - let explicit_status = tokio::time::timeout(Duration::from_secs(1), async { - loop { - if let Ok(status @ HealTaskStatus::Failed { .. }) = manager.get_task_status(&explicit_id).await { - break status; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("explicit timeout request should finish without using the zero default"); - assert!(matches!(explicit_status, HealTaskStatus::Failed { .. })); - } - - #[tokio::test] - async fn test_force_start_bypasses_duplicate_and_full_admission() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 1, - low_priority_drop_when_full: true, - ..HealConfig::default() - }), - ); - - let normal = HealRequest::new( - HealType::Bucket { - bucket: "bucket".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - let mut forced_duplicate = HealRequest::new( - HealType::Bucket { - bucket: "bucket".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - forced_duplicate.force_start = true; - - let subsequent_duplicate = HealRequest::new( - HealType::Bucket { - bucket: "bucket".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - - assert_eq!( - manager - .submit_heal_request(normal) - .await - .expect("first request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(forced_duplicate) - .await - .expect("force start should bypass duplicate/full policy"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(subsequent_duplicate) - .await - .expect("subsequent non-force duplicate should be merged"), - HealAdmissionResult::Merged - ); - } - - #[tokio::test] - async fn test_force_start_marks_dedup_key_for_future_duplicates() { - let storage: Arc = Arc::new(MockStorage); - let manager = HealManager::new( - storage, - Some(HealConfig { - queue_size: 1, - ..HealConfig::default() - }), - ); - - let normal = HealRequest::new( - HealType::Bucket { - bucket: "bucket".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - let mut forced = HealRequest::new( - HealType::Bucket { - bucket: "bucket".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - forced.force_start = true; - let duplicate = HealRequest::new( - HealType::Bucket { - bucket: "bucket".to_string(), - }, - HealOptions::default(), - HealPriority::Low, - ); - - assert_eq!( - manager - .submit_heal_request(normal) - .await - .expect("first request should be accepted"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(forced) - .await - .expect("forced request should bypass duplicate/full admission"), - HealAdmissionResult::Accepted - ); - assert_eq!( - manager - .submit_heal_request(duplicate) - .await - .expect("non-forced duplicate should merge while forced request is queued"), - HealAdmissionResult::Merged - ); - } - - #[test] - fn test_running_heal_set_counts_groups_set_scoped_tasks() { - let storage: Arc = Arc::new(MockStorage); - let erasure_task = Arc::new(HealTask::from_request( - HealRequest::new( - HealType::ErasureSet { - buckets: vec!["bucket".to_string()], - set_disk_id: "pool_0_set_1".to_string(), - }, - HealOptions::default(), - HealPriority::Normal, - ), - storage.clone(), - )); - let scoped_options = HealOptions { - pool_index: Some(0), - set_index: Some(1), - ..Default::default() - }; - let scoped_object_task = Arc::new(HealTask::from_request( - HealRequest::new( - HealType::Object { - bucket: "bucket".to_string(), - object: "scoped-object".to_string(), - version_id: None, - }, - scoped_options, - HealPriority::Normal, - ), - storage.clone(), - )); - let object_task = Arc::new(HealTask::from_request( - HealRequest::new( - HealType::Object { - bucket: "bucket".to_string(), - object: "object".to_string(), - version_id: None, - }, - HealOptions::default(), - HealPriority::Normal, - ), - storage, - )); - - let mut active = HashMap::new(); - active.insert(erasure_task.id.clone(), erasure_task); - active.insert(scoped_object_task.id.clone(), scoped_object_task); - active.insert(object_task.id.clone(), object_task); - - let counts = running_heal_set_counts(&active); - assert_eq!(counts.get("pool_0_set_1"), Some(&2)); - assert_eq!(counts.len(), 1); - } - - #[test] - fn test_heal_config_respects_feature_flags() { - temp_env::with_vars( - [ - (rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE, Some("false")), - (rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE, Some("false")), - (rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("false")), - ], - || { - let config = HealConfig::default(); - assert!(!config.event_driven_scheduler_enable); - assert!(!config.set_bulkhead_enable); - assert!(!config.page_parallel_enable); - }, - ); - } -} +mod tests; diff --git a/crates/heal/src/heal/manager/auto_scan.rs b/crates/heal/src/heal/manager/auto_scan.rs new file mode 100644 index 000000000..d828b446c --- /dev/null +++ b/crates/heal/src/heal/manager/auto_scan.rs @@ -0,0 +1,555 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// The automatic disk scanner: replacement discovery and unformatted-disk enqueue. +use super::*; + +impl HealManager { + /// Start background task to auto scan local disks and enqueue erasure set heal requests + pub(super) async fn start_auto_disk_scanner(&self) -> Result<()> { + let config = self.config.clone(); + let heal_queue = self.heal_queue.clone(); + let active_heals = self.active_heals.clone(); + let storage = self.storage.clone(); + let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); + let replacement_recovery_blocked_sets = self.replacement_recovery_blocked_sets.clone(); + let cancel_token = self.cancel_token.clone(); + let notify = self.notify.clone(); + let mut duration = { + let config = config.read().await; + config.heal_interval + }; + if duration < Duration::from_secs(10) { + duration = Duration::from_secs(10); + } + info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + state = "started", + interval = ?duration, + "Heal auto disk scanner started" + ); + + tokio::spawn(async move { + let mut interval = interval(duration); + + loop { + let mut candidate_count = 0usize; + let mut skipped_duplicate_count = 0usize; + let mut skipped_invalid_count = 0usize; + let mut enqueued_count = 0usize; + let mut not_enqueued_count = 0usize; + let mut dropped_count = 0usize; + let mut full_count = 0usize; + tokio::select! { + _ = cancel_token.cancelled() => { + info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + state = "shutdown", + "Heal auto disk scanner stopped" + ); + break; + } + _ = interval.tick() => { + // Build list of endpoints that need healing + let mut endpoints = HashMap::>::new(); + let mut durable_recoveries = HashMap::, Vec, String)>::new(); + let mut conflicted_recovery_sets = HashSet::::new(); + let mut deferred_replacement_endpoints = HashSet::::new(); + let local_disks = { + let local_disk_map = local_disk_map_read().await; + local_disk_map.values().flatten().cloned().collect::>() + }; + let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::>(); + let blocked_sets = replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .clone(); + if !blocked_sets.is_empty() { + let mut retry_succeeded = HashSet::new(); + let mut retry_failed = HashSet::new(); + for disk in &local_disks { + let endpoint = disk.endpoint(); + let Some(set_disk_id) = + crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx) + else { + continue; + }; + if !blocked_sets.contains(&set_disk_id) { + continue; + } + match Self::validate_replacement_recovery_records(disk).await { + Ok(()) => { + retry_succeeded.insert(set_disk_id); + } + Err(error) => { + retry_failed.insert(set_disk_id.clone()); + conflicted_recovery_sets.insert(set_disk_id); + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + error = %error, + "Replacement recovery retry failed" + ); + } + } + } + let mut blocked = replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned"); + unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed); + } + for disk in &local_disks { + let endpoint = disk.endpoint(); + let runtime_state = disk.runtime_state(); + let set_disk_id = + crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); + if set_disk_id.as_ref().is_some_and(|set_disk_id| { + replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .contains(set_disk_id) + }) { + skipped_invalid_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_DISK, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + set_disk_id = set_disk_id.as_deref().unwrap_or_default(), + disk_state = "replacement_recovery_blocked", + "Heal auto-scan replacement deferred because durable recovery is blocked" + ); + continue; + } + + // detect unformatted disk via get_disk_id() + match disk.get_disk_id().await { + Err(DiskError::UnformattedDisk) => { + if !super::super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks) + .await + { + deferred_replacement_endpoints.insert(endpoint.to_string()); + skipped_invalid_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_DISK, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + disk_state = "replacement_path_unavailable", + "Heal auto-scan replacement deferred" + ); + continue; + } + let Some(set_disk_id) = set_disk_id else { + skipped_invalid_count += 1; + continue; + }; + candidate_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_DISK, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + disk_state = "unformatted", + "Heal auto-scan candidate detected" + ); + endpoints.entry(set_disk_id).or_default().push(endpoint); + } + Err(e) => { + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_DISK, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + disk_state = "check_failed", + error = ?e, + "Heal auto-scan disk inspection failed" + ); + } + Ok(_) => { + if runtime_state.as_str() == "returning" && let Some(set_disk_id) = set_disk_id { + candidate_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_DISK, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + set_disk_id, + disk_state = "returning", + "Heal auto-scan returning disk candidate detected" + ); + endpoints.entry(set_disk_id).or_default().push(endpoint); + } + } + } + } + + // Once formatting succeeds a replacement is no longer + // discoverable as UnformattedDisk. Re-admit exactly one + // incomplete durable generation per set after bounded + // scheduler retries are exhausted, or re-admit its + // verified terminal cleanup. Multiple generations are a + // durable conflict: leave every marker/state intact and + // require reconciliation rather than choosing one. + for disk in &local_disks { + let endpoint = disk.endpoint(); + let disk_set_disk_id = + crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); + let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await { + Ok(task_ids) => task_ids, + Err(error) => { + let endpoint_string = endpoint.to_string(); + if replacement_discovery_error_is_expected_for_deferred_endpoint( + &error, + &endpoint_string, + &deferred_replacement_endpoints, + ) { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + disk_state = "replacement_path_unavailable", + result = "recovery_records_unavailable", + "Replacement recovery discovery skipped for deferred replacement" + ); + continue; + } + if let Some(set_disk_id) = &disk_set_disk_id { + conflicted_recovery_sets.insert(set_disk_id.clone()); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + error = %error, + "Replacement recovery discovery failed" + ); + continue; + } + }; + for task_id in replacement_task_ids { + let resume_manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + Ok(resume_manager) => resume_manager, + Err(error) => { + if let Some(set_disk_id) = &disk_set_disk_id { + conflicted_recovery_sets.insert(set_disk_id.clone()); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint = %endpoint, + task_id, + error = %error, + "Replacement recovery intent load failed" + ); + continue; + } + }; + let state = resume_manager.get_state().await; + if !durable_replacement_recovery_is_due(&state, &task_id) { + continue; + } + if !matches!(state.replacement_phase, ReplacementPhase::CleanupPending) { + let Ok(identities) = storage.replacement_target_identities(&state.replacement_targets).await else { + continue; + }; + if identities != state.replacement_target_identities { + continue; + } + } + let targets = state + .replacement_targets + .iter() + .filter_map(|target| { + local_endpoints + .iter() + .find(|endpoint| endpoint.to_string() == *target) + .cloned() + }) + .collect::>(); + if targets.len() != state.replacement_targets.len() { + continue; + } + let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32( + targets[0].pool_idx, + targets[0].set_idx, + ) else { + continue; + }; + if targets.iter().any(|target| { + crate::heal::utils::format_set_disk_id_from_i32(target.pool_idx, target.set_idx) + .as_deref() + != Some(set_disk_id.as_str()) + }) { + continue; + } + let resume_endpoint = disk.endpoint().to_string(); + match durable_recoveries.get(&set_disk_id) { + Some((existing_task_id, _, _, existing_anchor)) + if existing_task_id != &task_id || existing_anchor != &resume_endpoint => { + replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .insert(set_disk_id.clone()); + conflicted_recovery_sets.insert(set_disk_id); + } + Some(_) => {} + None => { + durable_recoveries.insert( + set_disk_id, + (task_id, targets, state.replacement_buckets, resume_endpoint), + ); + } + } + } + } + + for set_disk_id in &conflicted_recovery_sets { + durable_recoveries.remove(set_disk_id); + endpoints.remove(set_disk_id); + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + set_disk_id, + result = "durable_generation_conflict", + "Replacement recovery deferred because multiple durable generations exist" + ); + } + + for (set_disk_id, (_, targets, _, _)) in &durable_recoveries { + let expected = targets.iter().map(ToString::to_string).collect::>(); + let observed = endpoints + .get(set_disk_id) + .map(|endpoints| endpoints.iter().map(ToString::to_string).collect::>()) + .unwrap_or_default(); + if !observed.is_subset(&expected) { + replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .insert(set_disk_id.clone()); + conflicted_recovery_sets.insert(set_disk_id.clone()); + continue; + } + endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone()); + } + for set_disk_id in &conflicted_recovery_sets { + durable_recoveries.remove(set_disk_id); + endpoints.remove(set_disk_id); + } + + for target_endpoints in endpoints.values_mut() { + target_endpoints.sort_by_key(ToString::to_string); + target_endpoints.dedup_by(|left, right| left.to_string() == right.to_string()); + } + + if endpoints.is_empty() { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + state = "idle", + "Heal auto disk scanner idle" + ); + continue; + } + + // Admit one set task with every ready replacement target. Queue deduplication is + // set-scoped, so admitting endpoints independently would silently drop later targets. + for (set_disk_id, endpoints) in endpoints { + if replacement_recovery_blocked_sets + .lock() + .expect("replacement recovery blocked set lock poisoned") + .contains(&set_disk_id) + { + skipped_invalid_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + set_disk_id, + result = "replacement_recovery_blocked", + "Heal auto-scan replacement admission deferred because durable recovery is blocked" + ); + continue; + } + // skip if already queued or healing + // Use consistent lock order: queue first, then active_heals to avoid deadlock + let mut skip = false; + { + let queue = heal_queue.lock().await; + if queue.contains_erasure_set(&set_disk_id) { + skip = true; + } + } + if !skip { + let active = active_heals.lock().await; + if active.values().any(|task| { + matches!( + &task.heal_type, + crate::heal::task::HealType::ErasureSet { set_disk_id: active_id, .. } + if active_id == &set_disk_id + ) + }) { + skip = true; + } + } + + if skip { + skipped_duplicate_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint_count = endpoints.len(), + set_disk_id, + result = "skipped_duplicate", + "Heal auto-scan duplicate skipped" + ); + continue; + } + + // enqueue erasure set heal request for all ready replacements in this set + let recovery = durable_recoveries.remove(&set_disk_id); + let mut req = HealRequest::new( + HealType::ErasureSet { + buckets: recovery + .as_ref() + .map(|(_, _, buckets, _)| buckets.clone()) + .unwrap_or_default(), + set_disk_id: set_disk_id.clone(), + }, + HealOptions { + pool_index: endpoints + .first() + .and_then(|endpoint| usize::try_from(endpoint.pool_idx).ok()), + set_index: endpoints + .first() + .and_then(|endpoint| usize::try_from(endpoint.set_idx).ok()), + timeout: None, + ..HealOptions::default() + }, + HealPriority::Low, + ); + let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone()); + if let Some((task_id, _, _, _)) = recovery { + req.id = task_id; + } + req.source = HealRequestSource::AutoHeal; + req.heal_endpoints = endpoints.iter().map(ToString::to_string).collect(); + let request_id = req.id.clone(); + let endpoint_count = req.heal_endpoints.len(); + let config = config.read().await; + let mut queue = heal_queue.lock().await; + let admission = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan"); + let should_notify = + matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable; + if matches!(admission, HealAdmissionResult::Accepted) + && let Some(anchor) = recovery_anchor + { + replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(request_id, anchor); + } + drop(queue); + drop(config); + if matches!(admission, HealAdmissionResult::Accepted) { + if should_notify { + notify.notify_one(); + } + enqueued_count += 1; + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint_count, + set_disk_id, + bucket_count = 0, + result = "enqueued", + "Heal auto-scan task enqueued" + ); + } else { + if matches!(admission, HealAdmissionResult::Merged) { + skipped_duplicate_count += 1; + } else { + not_enqueued_count += 1; + } + if matches!(admission, HealAdmissionResult::Full) { + full_count += 1; + } + if matches!(admission, HealAdmissionResult::Dropped(_)) { + dropped_count += 1; + } + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_ENQUEUE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + endpoint_count, + set_disk_id, + bucket_count = 0, + admission = admission.result_label(), + reason = admission.reason_label(), + result = "not_enqueued", + "Heal auto-scan task not enqueued" + ); + } + } + info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_AUTO_SCAN_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_DISK_SCANNER, + state = "cycle_completed", + candidate_count, + enqueued_count, + not_enqueued_count, + dropped_count, + full_count, + skipped_duplicate_count, + skipped_invalid_count, + "Heal auto-scan cycle completed" + ); + } + } + } + }); + Ok(()) + } +} diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs new file mode 100644 index 000000000..9d33c660d --- /dev/null +++ b/crates/heal/src/heal/manager/queue.rs @@ -0,0 +1,452 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// The priority heal queue and its per-key dedup index. +use super::*; + +/// Per-key bookkeeping for the queued-request dedup index: how many queued +/// requests hold the key, and the id of the first request that opened it — +/// the O(1) stand-in for the former heap scan when a merge receipt needs to +/// name a queued representative. +#[derive(Debug)] +pub(super) struct DedupKeyEntry { + pub(super) refcount: usize, + pub(super) representative_request_id: String, +} + +/// Priority queue wrapper for heal requests +/// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items +#[derive(Debug)] +pub(super) struct PriorityHealQueue { + /// Heap of (priority, sequence, request) tuples + pub(super) heap: BinaryHeap, + /// Sequence counter for FIFO ordering within same priority + pub(super) sequence: u64, + /// Deduplication index for queued requests + pub(super) dedup_keys: HashMap, +} + +/// Wrapper for heap items to implement proper ordering +#[derive(Debug)] +pub(super) struct PriorityQueueItem { + pub(super) priority: HealPriority, + pub(super) sequence: u64, + pub(super) request: HealRequest, +} + +impl Eq for PriorityQueueItem {} + +impl PartialEq for PriorityQueueItem { + fn eq(&self, other: &Self) -> bool { + self.priority == other.priority && self.sequence == other.sequence + } +} + +impl Ord for PriorityQueueItem { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // First compare by priority (higher priority first) + match self.priority.cmp(&other.priority) { + std::cmp::Ordering::Equal => { + // If priorities are equal, use sequence for FIFO (lower sequence first) + other.sequence.cmp(&self.sequence) + } + ordering => ordering, + } + } +} + +impl PartialOrd for PriorityQueueItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum QueuePushOutcome { + Accepted, + Merged, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ForegroundPressure { + pub(super) class: WorkloadClass, + pub(super) usage_pct: usize, + pub(super) threshold_pct: usize, +} + +impl ForegroundPressure { + pub(super) const fn reason(self) -> &'static str { + match self.class { + WorkloadClass::ForegroundRead => "foreground_read_pressure", + WorkloadClass::ForegroundWrite => "foreground_write_pressure", + _ => "foreground_pressure", + } + } +} + +#[derive(Debug, Clone)] +pub(super) struct CompletedHealStatus { + pub(super) heal_type: HealType, + pub(super) status: HealTaskStatus, + pub(super) result_items_truncated: bool, + pub(super) completed_at: SystemTime, + /// Sequence-stamped retained window, archived with the completion so + /// incremental consumers keep their cursor across the transition (HS-06). + /// The un-stamped legacy view is derived from it on demand. + pub(super) seqed_items: Vec<(u64, HealResultItem)>, + pub(super) next_seq: u64, + pub(super) min_seq: u64, +} + +#[derive(Debug, Clone)] +pub(super) struct HealTaskAlias { + pub(super) task_id: String, +} + +#[derive(Debug, Clone)] +pub(super) struct RetryingHeal { + pub(super) request: HealRequest, + pub(super) error: String, + pub(super) cancel_token: CancellationToken, +} + +impl PriorityHealQueue { + pub(super) fn new() -> Self { + Self { + heap: BinaryHeap::new(), + sequence: 0, + dedup_keys: HashMap::new(), + } + } + + pub(super) fn len(&self) -> usize { + self.heap.len() + } + + pub(super) fn pop_next(&mut self) -> Option { + self.heap.pop().map(|item| { + let key = Self::make_dedup_key(&item.request); + Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + item.request + }) + } + + pub(super) fn is_empty(&self) -> bool { + self.heap.is_empty() + } + + pub(super) fn push(&mut self, request: HealRequest) -> QueuePushOutcome { + let key = Self::make_dedup_key(&request); + + // Check for duplicates unless the caller explicitly forces admission. + if self.dedup_keys.contains_key(&key) && !request.force_start { + return QueuePushOutcome::Merged; + } + // Track dedup keys for both normal and forced requests so queued forced work + // also reserves the dedup key for later non-forced duplicates. The first + // request that opens the key becomes the named representative for merge + // receipts (taken before `request` moves into the heap). + self.dedup_keys + .entry(key) + .or_insert_with(|| DedupKeyEntry { + refcount: 0, + representative_request_id: request.id.clone(), + }) + .refcount += 1; + self.sequence += 1; + self.heap.push(PriorityQueueItem { + priority: request.priority, + sequence: self.sequence, + request, + }); + QueuePushOutcome::Accepted + } + + pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool { + self.heap.iter().any(|item| item.priority < priority) + } + + pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option { + let mut retained = BinaryHeap::new(); + let mut displaced: Option = None; + + while let Some(item) = self.heap.pop() { + if item.priority < request.priority { + let should_displace = displaced + .as_ref() + .map(|current| { + item.priority < current.priority + || (item.priority == current.priority && item.sequence > current.sequence) + }) + .unwrap_or(true); + if should_displace { + if let Some(current) = displaced.replace(item) { + retained.push(current); + } + } else { + retained.push(item); + } + } else { + retained.push(item); + } + } + + self.heap = retained; + + let displaced = displaced.map(|item| { + let key = Self::make_dedup_key(&item.request); + Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + self.refresh_dedup_representative(&key); + item.request + }); + + if displaced.is_some() { + // The enqueue side effect must run in ALL builds. Do NOT fold `self.push(request)` + // into `debug_assert_eq!` — in release builds (`debug_assertions` off) the whole + // macro, including its argument expression, is compiled out, which would silently + // drop the new high-priority request after having already evicted a queued item. + let outcome = self.push(request); + debug_assert_eq!(outcome, QueuePushOutcome::Accepted); + } + + displaced + } + + /// Get statistics about queue contents by priority + pub(super) fn get_priority_stats(&self) -> HashMap { + let mut stats = HashMap::new(); + for item in &self.heap { + *stats.entry(item.priority).or_insert(0) += 1; + } + stats + } + + pub(super) fn operation_counts(&self) -> (HealPriorityCounts, HealSourceCounts) { + let mut priority = HealPriorityCounts::default(); + let mut source = HealSourceCounts::default(); + for item in &self.heap { + priority.increment(item.request.priority); + source.increment(item.request.source); + } + (priority, source) + } + + #[cfg(test)] + pub(super) fn pop(&mut self) -> Option { + self.heap.pop().map(|item| { + let key = Self::make_dedup_key(&item.request); + Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + item.request + }) + } + + #[cfg(test)] + pub(super) fn pop_runnable(&mut self, can_run: F) -> Option + where + F: Fn(&HealRequest) -> bool, + { + self.pop_runnable_with_skips(can_run, |_| None).0 + } + + pub(super) fn pop_runnable_with_skips(&mut self, can_run: F, skip_label: G) -> (Option, Vec) + where + F: Fn(&HealRequest) -> bool, + G: Fn(&HealRequest) -> Option, + { + let mut deferred = Vec::new(); + let mut selected = None; + let mut skipped = Vec::new(); + + while let Some(item) = self.heap.pop() { + if can_run(&item.request) { + selected = Some(item); + break; + } + if let Some(label) = skip_label(&item.request) { + skipped.push(label); + } + deferred.push(item); + } + + for item in deferred { + self.heap.push(item); + } + + ( + selected.map(|item| { + let key = Self::make_dedup_key(&item.request); + Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + item.request + }), + skipped, + ) + } + + /// Create a deduplication key from a heal request + pub(super) fn make_dedup_key(request: &HealRequest) -> String { + Self::make_dedup_key_for_type(&request.heal_type) + } + + pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String { + match heal_type { + HealType::Cluster => "cluster".to_string(), + HealType::Object { + bucket, + object, + version_id, + } => { + format!("object:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or("")) + } + HealType::Bucket { bucket } => { + format!("bucket:{bucket}") + } + HealType::Prefix { bucket, prefix } => { + format!("prefix:{bucket}/{prefix}") + } + HealType::ErasureSet { set_disk_id, .. } => { + format!("erasure_set:{set_disk_id}") + } + HealType::Metadata { bucket, object } => { + format!("metadata:{bucket}:{object}") + } + HealType::ECDecode { + bucket, + object, + version_id, + } => { + format!("ecdecode:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or("")) + } + } + } + + pub(super) fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap, key: &str) { + if let Some(entry) = dedup_keys.get_mut(key) { + if entry.refcount <= 1 { + dedup_keys.remove(key); + } else { + entry.refcount -= 1; + } + } + } + /// Check if an erasure set heal request for a specific set_disk_id exists + pub(super) fn contains_erasure_set(&self, set_disk_id: &str) -> bool { + let key = format!("erasure_set:{set_disk_id}"); + self.dedup_keys.contains_key(&key) + } + + /// Iterate queued requests (used by the admin overlap check). + pub(super) fn requests(&self) -> impl Iterator { + self.heap.iter().map(|item| &item.request) + } + + pub(super) fn contains_request_id(&self, request_id: &str) -> bool { + self.heap.iter().any(|item| item.request.id == request_id) + } + + pub(super) fn contains_request_id_matching_path(&self, request_id: &str, heal_path: &str) -> bool { + self.heap + .iter() + .any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path)) + } + + pub(super) fn queued_request_id_for_dedup_key(&self, key: &str) -> Option<&str> { + self.dedup_keys.get(key).map(|entry| entry.representative_request_id.as_str()) + } + + /// Re-elect the representative for `key` from the queue entries holding + /// it. Needed after a holder leaves the queue *without* becoming active + /// (canceled by id, or displaced): the former opener may be the request + /// that just left, and a merge receipt must never name an id that + /// resolves nowhere. The scheduler pop path does not need this — the + /// popped request surfaces in `active_heals` under the same id and the + /// duplicate pre-check consults active heals before the queue. No-op for + /// released keys; the survivor scan only runs when a key still has + /// holders, which under forced duplicates is the rare admin path. + pub(super) fn refresh_dedup_representative(&mut self, key: &str) { + if !self.dedup_keys.contains_key(key) { + return; + } + if let Some(id) = self + .heap + .iter() + .find(|item| Self::make_dedup_key(&item.request) == key) + .map(|item| item.request.id.clone()) + && let Some(entry) = self.dedup_keys.get_mut(key) + { + entry.representative_request_id = id; + } + } + + pub(super) fn contains_matching(&self, mut matches: F) -> bool + where + F: FnMut(&HealRequest) -> bool, + { + self.heap.iter().any(|item| matches(&item.request)) + } + + pub(super) fn remove_request_id(&mut self, request_id: &str) -> Option { + let mut retained = BinaryHeap::new(); + let mut removed = None; + + while let Some(item) = self.heap.pop() { + if removed.is_none() && item.request.id == request_id { + let key = Self::make_dedup_key(&item.request); + Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + removed = Some(item.request); + } else { + retained.push(item); + } + } + + self.heap = retained; + if let Some(removed) = removed.as_ref() { + self.refresh_dedup_representative(&Self::make_dedup_key(removed)); + } + removed + } + + pub(super) fn remove_matching(&mut self, mut should_remove: F) -> Vec + where + F: FnMut(&HealRequest) -> bool, + { + let mut retained = BinaryHeap::new(); + let mut removed = Vec::new(); + let mut affected_keys = Vec::new(); + + while let Some(item) = self.heap.pop() { + if should_remove(&item.request) { + let key = Self::make_dedup_key(&item.request); + Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + affected_keys.push(key); + removed.push(item.request); + } else { + retained.push(item); + } + } + + self.heap = retained; + for key in &affected_keys { + self.refresh_dedup_representative(key); + } + removed + } +} + +impl RetryingHeal { + pub(super) fn status(&self) -> HealTaskStatus { + HealTaskStatus::Retrying { + error: self.error.clone(), + retry_attempt: self.request.retry_attempts, + } + } +} diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs new file mode 100644 index 000000000..a22bee4ad --- /dev/null +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -0,0 +1,618 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// The heal scheduler: queue consumption loop and its skip/metric helpers. +use super::*; + +impl HealManager { + /// Start scheduler + pub(super) async fn start_scheduler(&self) -> Result<()> { + let config = self.config.clone(); + let heal_queue = self.heal_queue.clone(); + let active_heals = self.active_heals.clone(); + let completed_heals = self.completed_heals.clone(); + let retrying_heals = self.retrying_heals.clone(); + let replacement_recovery_anchors = self.replacement_recovery_anchors.clone(); + let cancel_token = self.cancel_token.clone(); + let statistics = self.statistics.clone(); + let storage = self.storage.clone(); + let notify = self.notify.clone(); + let workload_provider = self.workload_provider.clone(); + + tokio::spawn(async move { + let mut interval = interval(config.read().await.heal_interval); + + loop { + let event_driven_scheduler_enable = config.read().await.event_driven_scheduler_enable; + tokio::select! { + _ = cancel_token.cancelled() => { + info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_SCHEDULER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + state = "shutdown", + "Heal scheduler stopped" + ); + break; + } + _ = notify.notified(), if event_driven_scheduler_enable => { + Self::process_heal_queue(HealQueueContext { + heal_queue: &heal_queue, + active_heals: &active_heals, + completed_heals: &completed_heals, + retrying_heals: &retrying_heals, + replacement_recovery_anchors: &replacement_recovery_anchors, + config: &config, + statistics: &statistics, + storage: &storage, + notify: ¬ify, + cancel_token: &cancel_token, + workload_provider: &workload_provider, + }) + .await; + } + _ = interval.tick() => { + Self::process_heal_queue(HealQueueContext { + heal_queue: &heal_queue, + active_heals: &active_heals, + completed_heals: &completed_heals, + retrying_heals: &retrying_heals, + replacement_recovery_anchors: &replacement_recovery_anchors, + config: &config, + statistics: &statistics, + storage: &storage, + notify: ¬ify, + cancel_token: &cancel_token, + workload_provider: &workload_provider, + }) + .await; + } + } + } + }); + + Ok(()) + } + + /// Process heal queue + /// Processes multiple tasks per cycle when capacity allows and queue has high-priority items + pub(super) async fn process_heal_queue(context: HealQueueContext<'_>) { + let HealQueueContext { + heal_queue, + active_heals, + completed_heals, + retrying_heals, + replacement_recovery_anchors, + config, + statistics, + storage, + notify, + cancel_token, + workload_provider, + } = context; + + let config = config.read().await; + let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider); + let mut active_heals_guard = active_heals.lock().await; + publish_active_heal_count(&active_heals_guard); + + // Check if new heal tasks can be started + let active_count = active_heals_guard.len(); + if active_count >= config.max_concurrent_heals { + return; + } + + // Calculate how many tasks we can start this cycle + let available_slots = config.max_concurrent_heals - active_count; + + let mut queue = heal_queue.lock().await; + let queue_len = queue.len(); + publish_heal_queue_length(&queue); + + if queue_len == 0 { + return; + } + + let mut running_per_set = running_heal_set_counts(&active_heals_guard); + let mut tasks_started = 0usize; + let mut delayed_by_mainline_throttle = false; + + for _ in 0..available_slots { + let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() { + let max_concurrent_per_set = config.max_concurrent_per_set; + let (selected_request, skipped_sets) = queue.pop_runnable_with_skips( + |request| { + let set_allowed = !config.set_bulkhead_enable + || can_schedule_request(request, &running_per_set, max_concurrent_per_set); + let mainline_allowed = mainline_pressure.is_none() || Self::request_bypasses_mainline_throttle(request); + set_allowed && mainline_allowed + }, + |request| heal_request_set_key(request).map(|_| heal_request_set_metric_label(request)), + ); + for skipped_set in skipped_sets { + record_scheduler_skip(&skipped_set); + } + selected_request + } else { + queue.pop_next() + }; + + if let Some(mut request) = selected_request { + request.options.timeout.get_or_insert(config.task_timeout); + let task_priority = request.priority; + let task_type_label = heal_request_type_label(&request).to_string(); + let task_set_label = heal_request_set_metric_label(&request); + if config.set_bulkhead_enable + && let Some(set_key) = heal_request_set_key(&request) + { + *running_per_set.entry(set_key).or_insert(0) += 1; + } + let replacement_resume_endpoint = replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(&request.id) + .cloned(); + let task = Arc::new(HealTask::from_replacement_recovery_request( + request, + storage.clone(), + replacement_resume_endpoint, + )); + let task_id = task.id.clone(); + active_heals_guard.insert(task_id.clone(), task.clone()); + publish_active_heal_count(&active_heals_guard); + update_task_running_metric_for_task(&active_heals_guard, task.as_ref()); + let active_heals_clone = active_heals.clone(); + let heal_queue_clone = heal_queue.clone(); + let completed_heals_clone = completed_heals.clone(); + let retrying_heals_clone = retrying_heals.clone(); + let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone(); + let statistics_clone = statistics.clone(); + let notify_clone = notify.clone(); + let manager_cancel_token = cancel_token.clone(); + let task_type_label_for_spawn = task_type_label.clone(); + let task_set_label_for_spawn = task_set_label.clone(); + let config_for_spawn = config.clone(); + + // start heal task + tokio::spawn(async move { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_SCHEDULER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + task_id, + priority = ?task_priority, + heal_type = %task_type_label_for_spawn, + set = %task_set_label_for_spawn, + state = "task_started", + "Heal scheduler task started" + ); + let result = task.execute().await; + let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await; + match &result { + Ok(_) => { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_SCHEDULER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + task_id, + heal_type = %task_type_label_for_spawn, + set = %task_set_label_for_spawn, + state = "task_completed", + "Heal scheduler task completed" + ); + } + Err(e) => { + let will_retry = retry_request.is_some(); + if will_retry { + demote_to_debug_when!(task.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", { + event = EVENT_HEAL_SCHEDULER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + task_id, + heal_type = %task_type_label_for_spawn, + set = %task_set_label_for_spawn, + state = "task_retrying", + retry_attempt = task.retry_attempts.saturating_add(1), + error = %e, + "Heal scheduler task retrying" + }); + } else { + error!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_SCHEDULER_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + task_id, + heal_type = %task_type_label_for_spawn, + set = %task_set_label_for_spawn, + state = "task_failed", + error = %e, + "Heal scheduler task failed" + ); + } + } + } + let retry_request_for_status = retry_request.as_ref().map(|(request, _, error)| HealTaskStatus::Retrying { + error: error.clone(), + retry_attempt: request.retry_attempts, + }); + let retry_request_for_queue = retry_request; + let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new()); + if retry_request_for_queue.is_none() { + replacement_recovery_anchors_clone + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&task_id); + } + let mut active_heals_guard = active_heals_clone.lock().await; + // Keep retry ownership continuous: status snapshots acquire + // these locks in the same active -> retrying order. + let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) = + (retry_request_for_queue.as_ref(), retry_cancel_token.as_ref()) + { + let mut retrying = retrying_heals_clone.lock().await; + if active_heals_guard.contains_key(&task_id) { + retrying.insert( + request.id.clone(), + RetryingHeal { + request: request.clone(), + error: error.clone(), + cancel_token: cancel_token.clone(), + }, + ); + #[cfg(test)] + pause_retry_ownership_transition(&task_id, false).await; + } + Some(retrying) + } else { + None + }; + let completed_task = active_heals_guard.remove(&task_id); + if let Some(completed_task) = completed_task.as_ref() { + publish_active_heal_count(&active_heals_guard); + update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref()); + } + let active_count = active_heals_guard.len(); + drop(retrying_heals_guard.take()); + drop(active_heals_guard); + + if let Some(completed_task) = completed_task { + let completed_status = if let Some(status) = retry_request_for_status { + status + } else { + completed_task.get_status().await + }; + let completed_progress = completed_task.get_progress().await; + // Single snapshot of the retained window: the task is + // finished and already off the active map, so there is + // no concurrent writer to race with. + let seqed_items = completed_task.get_seqed_result_items().await; + let (next_seq, min_seq) = completed_task.result_seq_cursors(); + let completed_status_entry = CompletedHealStatus { + heal_type: completed_task.heal_type.clone(), + status: completed_status.clone(), + result_items_truncated: completed_task.result_items_truncated(), + completed_at: SystemTime::now(), + seqed_items, + next_seq, + min_seq, + }; + let mut completed_heals_guard = completed_heals_clone.lock().await; + prune_completed_heal_statuses(&mut completed_heals_guard); + completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry)); + // update statistics + let mut stats = statistics_clone.write().await; + match completed_status { + HealTaskStatus::Completed => { + stats.update_task_completion(true); + stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed); + } + HealTaskStatus::Retrying { .. } => {} + _ => { + stats.update_task_completion(false); + } + } + stats.update_running_tasks(usize_to_u64_saturated(active_count)); + } + + if let (Some((retry_request, retry_delay, retry_error)), Some(retry_cancel_token)) = + (retry_request_for_queue, retry_cancel_token) + { + let retry_request_id = retry_request.id.clone(); + let retry_attempt = retry_request.retry_attempts; + let retry_key = PriorityHealQueue::make_dedup_key(&retry_request); + let retry_priority = retry_request.priority; + let retry_active_heals = active_heals_clone.clone(); + let retry_heal_queue = heal_queue_clone.clone(); + let retrying_heals_for_spawn = retrying_heals_clone.clone(); + let retry_completed_heals = completed_heals_clone.clone(); + let retry_notify = notify_clone.clone(); + let retry_manager_cancel_token = manager_cancel_token.clone(); + let retry_config = config_for_spawn.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = retry_cancel_token.cancelled() => { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %retry_request_id, + priority = ?retry_priority, + retry_attempt, + result = "retry_cancelled", + "Heal retry admission decided" + ); + return; + } + _ = retry_manager_cancel_token.cancelled() => { + retrying_heals_for_spawn.lock().await.remove(&retry_request_id); + return; + } + _ = sleep(retry_delay) => {} + } + + { + let retrying_heals_guard = retrying_heals_for_spawn.lock().await; + if !retrying_heals_guard.contains_key(&retry_request_id) { + return; + } + } + + let active_duplicate = { + let active_heals_guard = retry_active_heals.lock().await; + active_heals_contains_dedup_key(&active_heals_guard, &retry_key) + }; + if active_duplicate { + retrying_heals_for_spawn.lock().await.remove(&retry_request_id); + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %retry_request_id, + priority = ?retry_priority, + retry_attempt, + result = "retry_merged_active_duplicate", + "Heal retry admission decided" + ); + return; + } + + let mut queue = retry_heal_queue.lock().await; + let admission = + Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry"); + let should_notify = matches!(admission, HealAdmissionResult::Accepted) + && retry_config.event_driven_scheduler_enable; + match admission { + HealAdmissionResult::Accepted => { + // Transfer ownership while holding queue -> retrying, + // matching operations_snapshot's lock order. + #[cfg(test)] + pause_retry_ownership_transition(&retry_request_id, true).await; + retrying_heals_for_spawn.lock().await.remove(&retry_request_id); + drop(queue); + retry_completed_heals.lock().await.remove(&retry_request_id); + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %retry_request_id, + priority = ?retry_priority, + retry_attempt, + retry_delay_ms = retry_delay.as_millis(), + error = %retry_error, + result = "retry_enqueued", + "Heal retry admission decided" + ); + if should_notify { + retry_notify.notify_one(); + } + return; + } + HealAdmissionResult::Merged => { + retrying_heals_for_spawn.lock().await.remove(&retry_request_id); + drop(queue); + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %retry_request_id, + priority = ?retry_priority, + retry_attempt, + result = "retry_merged_duplicate", + "Heal retry admission decided" + ); + return; + } + HealAdmissionResult::Full => { + // admit_request_to_queue already logged the + // rejection (context = "retry"); this repeats + // every backoff cycle while the queue stays + // full, so keep it at debug!. + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %retry_request_id, + priority = ?retry_priority, + retry_attempt, + result = "retry_rejected_full", + "Heal retry admission decided" + ); + } + HealAdmissionResult::Dropped(reason) => { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_ADMISSION, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + request_id = %retry_request_id, + priority = ?retry_priority, + retry_attempt, + reason = reason.as_str(), + result = "retry_dropped", + "Heal retry admission decided" + ); + } + } + } + }); + } + notify_clone.notify_one(); + }); + tasks_started += 1; + } else { + delayed_by_mainline_throttle = mainline_pressure.is_some(); + break; + } + } + + // Update statistics for all started tasks + let mut stats = statistics.write().await; + stats.total_tasks += tasks_started as u64; + stats.update_running_tasks(active_heals_guard.len() as u64); + publish_active_heal_count(&active_heals_guard); + publish_heal_queue_length(&queue); + + if delayed_by_mainline_throttle && let Some(pressure) = mainline_pressure { + Self::record_mainline_throttle_delay(pressure, &config); + Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep); + } + + // Log queue status if items remain + if !queue.is_empty() { + let remaining = queue.len(); + if remaining > 10 { + info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_QUEUE_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + queue_len = remaining, + active_tasks = active_heals_guard.len(), + state = "backlog_high", + "Heal queue backlog high" + ); + } + } + } +} +pub(super) fn heal_request_set_key(request: &HealRequest) -> Option { + match &request.heal_type { + HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()), + HealType::Object { .. } => heal_options_set_key(&request.options), + _ => None, + } +} + +pub(super) fn heal_options_set_key(options: &HealOptions) -> Option { + match (options.pool_index, options.set_index) { + (Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")), + _ => None, + } +} + +pub(super) fn heal_request_type_label(request: &HealRequest) -> &'static str { + match &request.heal_type { + HealType::Cluster => "cluster", + HealType::Object { .. } => "object", + HealType::Bucket { .. } => "bucket", + HealType::Prefix { .. } => "prefix", + HealType::ErasureSet { .. } => "erasure_set", + HealType::Metadata { .. } => "metadata", + HealType::ECDecode { .. } => "ec_decode", + } +} + +pub(super) fn heal_request_set_metric_label(request: &HealRequest) -> String { + heal_request_set_key(request).unwrap_or_else(|| match (request.options.pool_index, request.options.set_index) { + (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"), + _ => "global".to_string(), + }) +} + +pub(super) fn record_scheduler_skip(set_label: &str) { + counter!( + "rustfs_heal_scheduler_skip_total", + "reason" => "set_limit".to_string(), + "set" => set_label.to_string() + ) + .increment(1); +} + +pub(super) fn update_task_running_metric_for_task(active_heals: &HashMap>, task: &HealTask) { + let type_label = task.metric_type_label(); + let set_label = task.metric_set_label(); + let count = active_heals + .values() + .filter(|active_task| active_task.metric_type_label() == type_label && active_task.metric_set_label() == set_label) + .count(); + + gauge!( + "rustfs_heal_task_running", + "type" => type_label.to_string(), + "set" => set_label + ) + .set(count as f64); +} + +pub(super) fn running_heal_set_counts(active_heals: &HashMap>) -> HashMap { + let mut running = HashMap::new(); + for task in active_heals.values() { + if let Some(set_key) = heal_request_set_key_for_task(task) { + *running.entry(set_key).or_insert(0) += 1; + } + } + running +} + +pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option { + match &task.heal_type { + HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()), + HealType::Object { .. } => heal_options_set_key(&task.options), + _ => None, + } +} + +pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap>) { + let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else { + return; + }; + + completed_heals.retain(|_, completed| { + completed + .completed_at + .duration_since(SystemTime::UNIX_EPOCH) + .map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION) + .unwrap_or(false) + }); +} + +pub(super) fn can_schedule_request( + request: &HealRequest, + running_per_set: &HashMap, + max_concurrent_per_set: usize, +) -> bool { + match heal_request_set_key(request) { + Some(set_key) => running_per_set.get(&set_key).copied().unwrap_or(0) < max_concurrent_per_set, + None => true, + } +} diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs new file mode 100644 index 000000000..0587371fc --- /dev/null +++ b/crates/heal/src/heal/manager/tests.rs @@ -0,0 +1,2971 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::heal::EcstoreError; +use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity}; +use crate::heal::storage::{HealObjectInfo, HealStorageAPI}; +use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType}; +use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; +use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot}; +use rustfs_madmin::heal_commands::HealResultItem; +use std::sync::Mutex as StdMutex; +use tempfile::TempDir; + +use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo}; + +#[tokio::test] +async fn auto_replacement_path_requires_a_non_root_mount() { + let temp = TempDir::new().expect("temporary replacement root should be created"); + let ready = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("replacement endpoint should parse"); + let missing = Endpoint::try_from(temp.path().join("missing").to_string_lossy().as_ref()) + .expect("missing replacement endpoint should parse"); + + let ready_disk = new_disk( + &ready, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("temporary disk should initialize"); + assert!( + !super::super::replacement_readiness::auto_replacement_target_ready(&ready_disk, std::slice::from_ref(&ready_disk),) + .await + ); + assert!( + matches!( + new_disk( + &missing, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await, + Err(DiskError::VolumeNotFound) + ), + "a missing replacement path must be rejected before admission" + ); +} + +#[derive(Debug)] +struct FixedWorkloadProvider { + class: WorkloadClass, + active: usize, + limit: usize, + state: AdmissionState, +} + +impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider { + fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot { + WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(self.class, self.state).with_counts( + Some(self.active), + None, + Some(self.limit), + )]) + } +} + +async fn process_manager_queue_once(manager: &HealManager) { + HealManager::process_heal_queue(HealQueueContext { + heal_queue: &manager.heal_queue, + active_heals: &manager.active_heals, + completed_heals: &manager.completed_heals, + retrying_heals: &manager.retrying_heals, + replacement_recovery_anchors: &manager.replacement_recovery_anchors, + config: &manager.config, + statistics: &manager.statistics, + storage: &manager.storage, + notify: &manager.notify, + cancel_token: &manager.cancel_token, + workload_provider: &manager.workload_provider, + }) + .await; +} + +struct MockStorage; + +#[async_trait::async_trait] +impl HealStorageAPI for MockStorage { + async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result> { + Ok(None) + } + + async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result> { + Ok(Vec::new()) + } + + async fn get_bucket_info(&self, _bucket: &str) -> Result> { + Ok(None) + } + + async fn list_buckets(&self) -> Result> { + if let Some(hook) = manager_recovery_test_hook() { + *hook.listed.lock().expect("manager recovery listed lock should not poison") = true; + } + Ok(Vec::new()) + } + + async fn object_exists(&self, bucket: &str, _object: &str) -> Result { + Ok(bucket == "retry-transition") + } + + async fn heal_object( + &self, + bucket: &str, + _object: &str, + _version_id: Option<&str>, + _opts: &HealOpts, + ) -> Result<(HealResultItem, Option)> { + if let Some(hook) = manager_recovery_test_hook() { + *hook + .heal_object_calls + .lock() + .expect("manager recovery object call lock should not poison") += 1; + } + if bucket == "retry-transition" { + return Ok(( + HealResultItem::default(), + Some(Error::Storage(EcstoreError::InsufficientReadQuorum( + bucket.to_string(), + "object".to_string(), + ))), + )); + } + Ok((HealResultItem::default(), None)) + } + + async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result { + if let Some(hook) = manager_recovery_test_hook() { + *hook + .bucket_heal_calls + .lock() + .expect("manager recovery bucket call lock should not poison") += 1; + } + Ok(HealResultItem::default()) + } + + async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option)> { + if let Some(hook) = manager_recovery_test_hook() { + *hook + .global_format_calls + .lock() + .expect("manager recovery global format call lock should not poison") += 1; + } + Ok((HealResultItem::default(), None)) + } + + async fn heal_replacement_format( + &self, + _dry_run: bool, + _pool_index: usize, + _set_index: usize, + _targets: &[String], + ) -> Result<(HealResultItem, Option)> { + if let Some(hook) = manager_recovery_test_hook() { + *hook + .replacement_format_calls + .lock() + .expect("manager recovery replacement format call lock should not poison") += 1; + } + Ok((HealResultItem::default(), None)) + } + + async fn list_objects_for_heal_page( + &self, + _bucket: &str, + _prefix: &str, + _continuation_token: Option<&str>, + _include_lifecycle_object_info: bool, + ) -> Result<(Vec, Option, bool)> { + Ok((Vec::new(), None, false)) + } + + async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result { + Err(Error::other("not implemented in tests")) + } + + async fn get_replacement_resume_disk( + &self, + _set_disk_id: &str, + _task_id: &str, + _excluded_targets: &[String], + ) -> Result { + let Some(hook) = manager_recovery_test_hook() else { + return Err(Error::other("not implemented in tests")); + }; + Ok(crate::heal::storage::ReplacementResumeDisk::Existing( + hook.replacement_resume_disk.clone(), + )) + } +} + +struct ManagerRecoveryTestHook { + replacement_resume_disk: DiskStore, + listed: StdMutex, + global_format_calls: StdMutex, + replacement_format_calls: StdMutex, + bucket_heal_calls: StdMutex, + heal_object_calls: StdMutex, +} + +static MANAGER_RECOVERY_TEST_HOOK: LazyLock>>> = + LazyLock::new(|| StdMutex::new(None)); + +struct ManagerRecoveryTestHookGuard; + +impl ManagerRecoveryTestHook { + fn install(replacement_resume_disk: DiskStore) -> (Arc, ManagerRecoveryTestHookGuard) { + let hook = Arc::new(Self { + replacement_resume_disk, + listed: StdMutex::new(false), + global_format_calls: StdMutex::new(0), + replacement_format_calls: StdMutex::new(0), + bucket_heal_calls: StdMutex::new(0), + heal_object_calls: StdMutex::new(0), + }); + let previous = MANAGER_RECOVERY_TEST_HOOK + .lock() + .expect("manager recovery hook lock should not poison") + .replace(hook.clone()); + assert!(previous.is_none(), "manager recovery hook already installed"); + (hook, ManagerRecoveryTestHookGuard) + } +} + +impl Drop for ManagerRecoveryTestHookGuard { + fn drop(&mut self) { + *MANAGER_RECOVERY_TEST_HOOK + .lock() + .expect("manager recovery hook lock should not poison") = None; + } +} + +fn manager_recovery_test_hook() -> Option> { + MANAGER_RECOVERY_TEST_HOOK + .lock() + .expect("manager recovery hook lock should not poison") + .clone() +} + +async fn make_manager_resume_disk(temp: &TempDir, name: &str) -> DiskStore { + let disk_path = temp.path().join(name); + std::fs::create_dir_all(&disk_path).expect("manager recovery disk directory should be created"); + let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).expect("manager recovery endpoint should parse"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("manager recovery disk should initialize"); + let metadata_volume = disk.make_volume(super::super::RUSTFS_META_BUCKET).await; + assert!( + matches!(metadata_volume, Ok(()) | Err(DiskError::VolumeExists)), + "manager recovery metadata volume should exist: {metadata_volume:?}" + ); + disk +} + +fn bucket_request(bucket: &str, priority: HealPriority, source: HealRequestSource) -> HealRequest { + let mut request = HealRequest::new( + HealType::Bucket { + bucket: bucket.to_string(), + }, + HealOptions::default(), + priority, + ); + request.source = source; + request +} + +#[test] +fn test_push_displacing_lower_priority_actually_enqueues_new_request() { + // Regression for the release-build defect where the enqueue side effect lived inside + // `debug_assert_eq!(self.push(request), ...)` and was compiled out under + // `cargo test --release` (debug_assertions off), silently dropping the displacing + // high-priority request while still having evicted a queued item. + // + // Must run with --release to expose the original bug. + let mut queue = PriorityHealQueue::new(); + + let low = bucket_request("victim-bucket", HealPriority::Low, HealRequestSource::Scanner); + assert_eq!(queue.push(low), QueuePushOutcome::Accepted); + assert_eq!(queue.len(), 1); + + let high = bucket_request("admin-bucket", HealPriority::High, HealRequestSource::Admin); + let high_id = high.id.clone(); + assert!(queue.can_displace_lower_priority(high.priority)); + + let displaced = queue + .push_displacing_lower_priority(high) + .expect("a lower-priority item should have been displaced"); + assert_eq!(displaced.priority, HealPriority::Low); + + // The displacing high-priority request must actually be enqueued (pre-fix under + // --release, len() is 0 because self.push(request) was elided with debug_assert_eq!). + assert_eq!(queue.len(), 1, "displacing request must remain enqueued"); + let admitted = queue.pop_next().expect("displacing high-priority request must be enqueued"); + assert_eq!(admitted.priority, HealPriority::High); + assert_eq!(admitted.id, high_id); + assert_eq!(queue.len(), 0); +} + +#[test] +fn queued_request_id_for_dedup_key_tracks_the_representative() { + let mut queue = PriorityHealQueue::new(); + + let first = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let first_id = first.id.clone(); + let first_key = PriorityHealQueue::make_dedup_key(&first); + assert_eq!(queue.push(first), QueuePushOutcome::Accepted); + + // A forced duplicate of the same target opens a second entry under + // the same key; the representative stays the request that opened it. + let mut second = HealRequest::object("bucket".to_string(), "object".to_string(), None); + second.force_start = true; + let second_id = second.id.clone(); + assert_eq!(queue.push(second), QueuePushOutcome::Accepted); + + let representative = queue + .queued_request_id_for_dedup_key(&first_key) + .expect("key must be reserved while either request is queued"); + assert_eq!(representative, first_id); + + // A holder leaving WITHOUT becoming active (canceled by id) must + // re-elect the representative to the surviving queued request, or a + // later merge receipt would name an id that resolves nowhere. The + // scheduler pop path needs no re-election: the popped request + // surfaces in active_heals under the same id and the duplicate + // pre-check consults active heals before the queue. + queue.remove_request_id(&first_id); + assert_eq!( + queue.queued_request_id_for_dedup_key(&first_key), + Some(second_id.as_str()), + "canceling the opener must re-elect the surviving queued holder" + ); + + // Pop the last holder: the key is released entirely. + let last = queue.pop_next().expect("second request must be queued"); + assert_eq!(last.id, second_id); + assert!(queue.queued_request_id_for_dedup_key(&first_key).is_none()); +} + +#[test] +fn test_priority_queue_ordering() { + let mut queue = PriorityHealQueue::new(); + + // Add requests with different priorities + let low_req = HealRequest::new( + HealType::Bucket { + bucket: "bucket1".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + + let normal_req = HealRequest::new( + HealType::Bucket { + bucket: "bucket2".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let high_req = HealRequest::new( + HealType::Bucket { + bucket: "bucket3".to_string(), + }, + HealOptions::default(), + HealPriority::High, + ); + + let urgent_req = HealRequest::new( + HealType::Bucket { + bucket: "bucket4".to_string(), + }, + HealOptions::default(), + HealPriority::Urgent, + ); + + // Add in random order: low, high, normal, urgent + assert_eq!(queue.push(low_req), QueuePushOutcome::Accepted); + assert_eq!(queue.push(high_req), QueuePushOutcome::Accepted); + assert_eq!(queue.push(normal_req), QueuePushOutcome::Accepted); + assert_eq!(queue.push(urgent_req), QueuePushOutcome::Accepted); + + assert_eq!(queue.len(), 4); + + // Should pop in priority order: urgent, high, normal, low + let popped1 = queue.pop().unwrap(); + assert_eq!(popped1.priority, HealPriority::Urgent); + + let popped2 = queue.pop().unwrap(); + assert_eq!(popped2.priority, HealPriority::High); + + let popped3 = queue.pop().unwrap(); + assert_eq!(popped3.priority, HealPriority::Normal); + + let popped4 = queue.pop().unwrap(); + assert_eq!(popped4.priority, HealPriority::Low); + + assert_eq!(queue.len(), 0); +} + +#[test] +fn test_priority_queue_fifo_same_priority() { + let mut queue = PriorityHealQueue::new(); + + // Add multiple requests with same priority + let req1 = HealRequest::new( + HealType::Bucket { + bucket: "bucket1".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let req2 = HealRequest::new( + HealType::Bucket { + bucket: "bucket2".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let req3 = HealRequest::new( + HealType::Bucket { + bucket: "bucket3".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let id1 = req1.id.clone(); + let id2 = req2.id.clone(); + let id3 = req3.id.clone(); + + assert_eq!(queue.push(req1), QueuePushOutcome::Accepted); + assert_eq!(queue.push(req2), QueuePushOutcome::Accepted); + assert_eq!(queue.push(req3), QueuePushOutcome::Accepted); + + // Should maintain FIFO order for same priority + let popped1 = queue.pop().unwrap(); + assert_eq!(popped1.id, id1); + + let popped2 = queue.pop().unwrap(); + assert_eq!(popped2.id, id2); + + let popped3 = queue.pop().unwrap(); + assert_eq!(popped3.id, id3); +} + +#[test] +fn test_priority_queue_deduplication() { + let mut queue = PriorityHealQueue::new(); + + let req1 = HealRequest::new( + HealType::Object { + bucket: "bucket1".to_string(), + object: "object1".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let req2 = HealRequest::new( + HealType::Object { + bucket: "bucket1".to_string(), + object: "object1".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::High, + ); + + // First request should be added + assert_eq!(queue.push(req1), QueuePushOutcome::Accepted); + assert_eq!(queue.len(), 1); + + // Second request with same object should be rejected (duplicate) + assert_eq!(queue.push(req2), QueuePushOutcome::Merged); + assert_eq!(queue.len(), 1); +} + +#[test] +fn test_priority_queue_contains_erasure_set() { + let mut queue = PriorityHealQueue::new(); + + let req = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket1".to_string()], + set_disk_id: "pool_0_set_1".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + assert_eq!(queue.push(req), QueuePushOutcome::Accepted); + assert!(queue.contains_erasure_set("pool_0_set_1")); + assert!(!queue.contains_erasure_set("pool_0_set_2")); +} + +#[test] +fn test_priority_queue_dedup_key_generation() { + // Test different heal types generate different keys + let obj_req = HealRequest::new( + HealType::Object { + bucket: "bucket1".to_string(), + object: "object1".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let bucket_req = HealRequest::new( + HealType::Bucket { + bucket: "bucket1".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let erasure_req = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket1".to_string()], + set_disk_id: "pool_0_set_1".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let obj_key = PriorityHealQueue::make_dedup_key(&obj_req); + let bucket_key = PriorityHealQueue::make_dedup_key(&bucket_req); + let erasure_key = PriorityHealQueue::make_dedup_key(&erasure_req); + + // All keys should be different + assert_ne!(obj_key, bucket_key); + assert_ne!(obj_key, erasure_key); + assert_ne!(bucket_key, erasure_key); + + assert!(obj_key.starts_with("object:")); + assert!(bucket_key.starts_with("bucket:")); + assert!(erasure_key.starts_with("erasure_set:")); +} + +#[test] +fn test_priority_queue_mixed_priorities_and_types() { + let mut queue = PriorityHealQueue::new(); + + // Add various requests + let requests = vec![ + ( + HealType::Object { + bucket: "b1".to_string(), + object: "o1".to_string(), + version_id: None, + }, + HealPriority::Low, + ), + ( + HealType::Bucket { + bucket: "b2".to_string(), + }, + HealPriority::Urgent, + ), + ( + HealType::ErasureSet { + buckets: vec!["b3".to_string()], + set_disk_id: "pool_0_set_1".to_string(), + }, + HealPriority::Normal, + ), + ( + HealType::Object { + bucket: "b4".to_string(), + object: "o4".to_string(), + version_id: None, + }, + HealPriority::High, + ), + ]; + + for (heal_type, priority) in requests { + let req = HealRequest::new(heal_type, HealOptions::default(), priority); + let outcome = queue.push(req); + assert_eq!(outcome, QueuePushOutcome::Accepted); + } + + assert_eq!(queue.len(), 4); + + // Check they come out in priority order + let priorities: Vec = (0..4).filter_map(|_| queue.pop().map(|r| r.priority)).collect(); + + assert_eq!( + priorities, + vec![ + HealPriority::Urgent, + HealPriority::High, + HealPriority::Normal, + HealPriority::Low, + ] + ); +} + +#[test] +fn test_priority_queue_stats() { + let mut queue = PriorityHealQueue::new(); + + // Add requests with different priorities + for _ in 0..3 { + assert_eq!( + queue.push(HealRequest::new( + HealType::Bucket { + bucket: format!("bucket-low-{}", queue.len()), + }, + HealOptions::default(), + HealPriority::Low, + )), + QueuePushOutcome::Accepted + ); + } + + for _ in 0..2 { + assert_eq!( + queue.push(HealRequest::new( + HealType::Bucket { + bucket: format!("bucket-normal-{}", queue.len()), + }, + HealOptions::default(), + HealPriority::Normal, + )), + QueuePushOutcome::Accepted + ); + } + + assert_eq!( + queue.push(HealRequest::new( + HealType::Bucket { + bucket: "bucket-high".to_string(), + }, + HealOptions::default(), + HealPriority::High, + )), + QueuePushOutcome::Accepted + ); + + let stats = queue.get_priority_stats(); + + assert_eq!(*stats.get(&HealPriority::Low).unwrap_or(&0), 3); + assert_eq!(*stats.get(&HealPriority::Normal).unwrap_or(&0), 2); + assert_eq!(*stats.get(&HealPriority::High).unwrap_or(&0), 1); + assert_eq!(*stats.get(&HealPriority::Urgent).unwrap_or(&0), 0); +} + +#[test] +fn test_priority_queue_is_empty() { + let mut queue = PriorityHealQueue::new(); + + assert!(queue.is_empty()); + + assert_eq!( + queue.push(HealRequest::new( + HealType::Bucket { + bucket: "test".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + )), + QueuePushOutcome::Accepted + ); + + assert!(!queue.is_empty()); + + queue.pop(); + + assert!(queue.is_empty()); +} + +#[test] +fn test_priority_queue_pop_runnable_skips_blocked_erasure_set() { + let mut queue = PriorityHealQueue::new(); + + let blocked = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_1".to_string(), + }, + HealOptions::default(), + HealPriority::Urgent, + ); + let runnable = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-b".to_string()], + set_disk_id: "pool_0_set_2".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted); + assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted); + + let mut running = HashMap::new(); + running.insert("pool_0_set_1".to_string(), 1); + + let popped = queue + .pop_runnable(|request| can_schedule_request(request, &running, 1)) + .expect("should find runnable request"); + + assert!(matches!( + popped.heal_type, + HealType::ErasureSet { ref set_disk_id, .. } if set_disk_id == "pool_0_set_2" + )); +} + +#[test] +fn test_can_schedule_request_respects_per_set_limit() { + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket".to_string()], + set_disk_id: "pool_0_set_1".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + let mut running = HashMap::new(); + running.insert("pool_0_set_1".to_string(), 1); + + assert!(!can_schedule_request(&request, &running, 1)); + assert!(can_schedule_request(&request, &running, 2)); +} + +#[test] +fn test_can_schedule_scoped_object_request_respects_per_set_limit() { + let options = HealOptions { + pool_index: Some(0), + set_index: Some(1), + ..Default::default() + }; + let request = HealRequest::new( + HealType::Object { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + }, + options, + HealPriority::Normal, + ); + + let mut running = HashMap::new(); + running.insert("pool_0_set_1".to_string(), 1); + + assert!(!can_schedule_request(&request, &running, 1)); + assert!(can_schedule_request(&request, &running, 2)); +} + +#[tokio::test] +async fn test_submit_heal_request_returns_merged_for_duplicate() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let request = HealRequest::new( + HealType::Object { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::Low, + ); + + assert_eq!( + manager + .submit_heal_request(request.clone()) + .await + .expect("first request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(request) + .await + .expect("duplicate request should produce admission result"), + HealAdmissionResult::Merged + ); +} + +#[tokio::test] +async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None); + original.source = HealRequestSource::Admin; + let original_id = original.id.clone(); + let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None); + duplicate.source = HealRequestSource::Admin; + let duplicate_id = duplicate.id.clone(); + + let accepted = manager + .submit_heal_request_with_receipt(original) + .await + .expect("first request should be accepted"); + let merged = manager + .submit_heal_request_with_receipt(duplicate) + .await + .expect("duplicate request should merge"); + + assert_eq!(accepted.result, HealAdmissionResult::Accepted); + assert_eq!(accepted.task_id, original_id); + assert_eq!(merged.result, HealAdmissionResult::Merged); + assert_eq!(merged.task_id, original_id); + assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id); +} + +#[tokio::test] +async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() { + let storage: Arc = Arc::new(MockStorage); + let manager = Arc::new(HealManager::new(storage.clone(), None)); + let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None); + original.source = HealRequestSource::Admin; + let original_id = original.id.clone(); + assert_eq!( + manager + .submit_heal_request(original) + .await + .expect("original request should be accepted"), + HealAdmissionResult::Accepted + ); + + let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None); + duplicate.source = HealRequestSource::Admin; + let hook = Arc::new(DuplicateAdmissionTestHook { + request_id: duplicate.id.clone(), + active_lock_reached: Notify::new(), + active_lock_release: Notify::new(), + }); + *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = Some(hook.clone()); + + let duplicate_manager = manager.clone(); + let duplicate_task = tokio::spawn(async move { duplicate_manager.submit_heal_request_with_receipt(duplicate).await }); + tokio::time::timeout(Duration::from_secs(1), hook.active_lock_reached.notified()) + .await + .expect("duplicate admission should reach the active lock hook"); + + let transition_manager = manager.clone(); + let transition_storage = storage.clone(); + let (attempting_active_tx, attempting_active_rx) = tokio::sync::oneshot::channel(); + let (active_acquired_tx, mut active_acquired_rx) = tokio::sync::oneshot::channel(); + let transition = tokio::spawn(async move { + let _ = attempting_active_tx.send(()); + let mut active = transition_manager.active_heals.lock().await; + let _ = active_acquired_tx.send(()); + let mut queue = transition_manager.heal_queue.lock().await; + let request = queue.pop_next().expect("original request should remain queued"); + let task = Arc::new(HealTask::from_request(request, transition_storage)); + active.insert(task.id.clone(), task); + }); + tokio::time::timeout(Duration::from_secs(1), attempting_active_rx) + .await + .expect("transition should attempt the active lock") + .expect("transition attempt signal should be delivered"); + assert!(matches!( + active_acquired_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + + hook.active_lock_release.notify_one(); + let receipt = tokio::time::timeout(Duration::from_secs(1), duplicate_task) + .await + .expect("duplicate admission should not hang") + .expect("duplicate task should join") + .expect("duplicate admission should succeed"); + tokio::time::timeout(Duration::from_secs(1), transition) + .await + .expect("queue to active transition should not hang") + .expect("queue to active transition should join"); + *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = None; + + assert_eq!(receipt.result, HealAdmissionResult::Merged); + assert_eq!(receipt.task_id, original_id); + assert_eq!(manager.get_queue_length().await, 0); + assert_eq!(manager.get_active_task_count().await, 1); +} + +#[tokio::test] +async fn test_submit_heal_request_returns_merged_for_active_duplicate() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage.clone(), None); + let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let active_task = Arc::new(HealTask::from_request(active_request, storage)); + manager.active_heals.lock().await.insert(active_task.id.clone(), active_task); + + let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + + assert_eq!( + manager + .submit_heal_request(duplicate_request) + .await + .expect("active duplicate should produce admission result"), + HealAdmissionResult::Merged + ); + assert_eq!(manager.get_queue_length().await, 0); +} + +#[tokio::test] +async fn test_active_duplicate_token_can_query_and_cancel_original_task() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage.clone(), None); + let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let active_task = Arc::new(HealTask::from_request(active_request, storage)); + let active_task_id = active_task.id.clone(); + manager.active_heals.lock().await.insert(active_task_id.clone(), active_task); + + let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let duplicate_task_id = duplicate_request.id.clone(); + + assert_eq!( + manager + .submit_heal_request(duplicate_request) + .await + .expect("active duplicate should produce admission result"), + HealAdmissionResult::Merged + ); + assert_eq!( + manager + .get_task_status_for_path("bucket/object", &duplicate_task_id) + .await + .expect("duplicate token should query merged active task"), + HealTaskStatus::Pending + ); + + manager + .cancel_task(&duplicate_task_id) + .await + .expect("duplicate token should cancel merged active task"); + + assert!(manager.active_heals.lock().await.get(&active_task_id).is_none()); + assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. }))); +} + +#[tokio::test] +async fn test_queued_duplicate_token_can_query_and_cancel_original_request() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + let original_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let original_task_id = original_request.id.clone(); + let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let duplicate_task_id = duplicate_request.id.clone(); + + assert_eq!( + manager + .submit_heal_request(original_request) + .await + .expect("original request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(duplicate_request) + .await + .expect("queued duplicate should produce admission result"), + HealAdmissionResult::Merged + ); + assert_eq!( + manager + .get_task_status_for_path("bucket/object", &duplicate_task_id) + .await + .expect("duplicate token should query merged queued task"), + HealTaskStatus::Pending + ); + + manager + .cancel_task(&duplicate_task_id) + .await + .expect("duplicate token should cancel merged queued request"); + + assert!(matches!( + manager.get_task_status(&original_task_id).await, + Err(Error::TaskNotFound { .. }) + )); +} + +#[test] +fn test_retry_request_for_recoverable_lock_timeout() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); + let result = Err(Error::TaskExecutionFailed { + message: "Failed to heal object bucket/object: Lock acquisition timeout".to_string(), + }); + + let (retry_request, retry_delay, retry_error) = + retry_request_for_result(&task, &result).expect("lock timeout should be retryable"); + + assert_eq!(retry_request.id, task.id); + assert_eq!(retry_request.retry_attempts, 1); + assert_eq!(retry_request.priority, task.priority); + assert!(retry_delay > Duration::ZERO); + assert!(retry_error.contains("Lock acquisition timeout")); +} + +#[tokio::test] +async fn retry_request_for_result_preserves_remaining_timeout_budget() { + let storage: Arc = Arc::new(MockStorage); + let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); + request.options.timeout = Some(Duration::from_secs(60)); + let task = HealTask::from_request(request, storage); + let result = task.execute().await; + + let (retry_request, _, _) = retry_request_for_result_with_budget(&task, &result) + .await + .expect("read quorum failure should retain the unused timeout budget"); + let remaining = retry_request + .options + .timeout + .expect("configured timeout should remain present"); + assert!(remaining < Duration::from_secs(60)); + assert!(remaining > Duration::from_secs(59)); +} + +#[test] +fn test_retry_request_for_incomplete_heal_rename() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); + let result = Err(Error::TaskExecutionFailed { + message: "Failed to heal object bucket/object: heal rename incomplete: 1 of 2 targets committed".to_string(), + }); + + let (retry_request, retry_delay, retry_error) = + retry_request_for_result(&task, &result).expect("incomplete target rename should be retryable"); + + assert_eq!(retry_request.id, task.id); + assert_eq!(retry_request.retry_attempts, 1); + assert!(retry_delay > Duration::ZERO); + assert!(retry_error.contains("heal rename incomplete")); +} + +#[test] +fn test_retry_request_for_typed_read_quorum_error() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); + let result = Err(Error::Storage(EcstoreError::InsufficientReadQuorum( + "bucket".to_string(), + "object".to_string(), + ))); + + let (retry_request, retry_delay, retry_error) = + retry_request_for_result(&task, &result).expect("typed read quorum should be retryable"); + + assert_eq!(retry_request.id, task.id); + assert_eq!(retry_request.retry_attempts, 1); + assert!(retry_delay > Duration::ZERO); + assert!(retry_error.contains("Storage resources are insufficient")); +} + +#[test] +fn test_retry_request_for_durable_replacement_retry_signal() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request( + HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ), + storage, + ); + let result = Err(Error::transient_skip("Replacement erasure set heal incomplete; retry scheduled")); + + let (retry_request, retry_delay, retry_error) = + retry_request_for_result(&task, &result).expect("typed replacement retry signal must be scheduled"); + + assert_eq!(retry_request.id, task.id); + assert_eq!(retry_request.retry_attempts, 1); + assert!(retry_delay > Duration::ZERO); + assert!(retry_error.contains("Replacement erasure set heal incomplete")); +} + +#[test] +fn durable_replacement_recovery_re_admits_only_the_matching_generation() { + let task_id = "replacement-generation"; + let mut state = crate::heal::resume::ResumeState::new( + task_id.to_string(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ); + state.replacement_generation = Some(task_id.to_string()); + state.replacement_phase = ReplacementPhase::Intent; + state.replacement_targets = vec!["replacement-a".to_string()]; + assert!(!durable_replacement_recovery_is_due(&state, task_id)); + + state.retry_count = state.max_retries; + assert!(durable_replacement_recovery_is_due(&state, task_id)); + + state.completed = true; + state.retry_count = 0; + state.replacement_phase = ReplacementPhase::Verified; + assert!( + durable_replacement_recovery_is_due(&state, task_id), + "verified terminal cleanup must be re-admitted without re-running recovery" + ); + + state.replacement_phase = ReplacementPhase::CleanupPending; + assert!( + durable_replacement_recovery_is_due(&state, task_id), + "cleanup-pending terminal cleanup must be periodically re-admitted" + ); + + state.completed = false; + state.replacement_generation = Some("another-generation".to_string()); + assert!( + !durable_replacement_recovery_is_due(&state, task_id), + "a task must not adopt another generation's durable intent" + ); + + state.replacement_generation = Some(task_id.to_string()); + state.replacement_targets.clear(); + assert!( + !durable_replacement_recovery_is_due(&state, task_id), + "a durable retry without a target is not safe to re-admit" + ); + + state.replacement_targets = vec!["replacement-a".to_string()]; + state.replacement_phase = ReplacementPhase::Verified; + assert!( + !durable_replacement_recovery_is_due(&state, task_id), + "a task must not adopt another generation's terminal cleanup" + ); +} + +#[test] +fn replacement_recovery_blocker_is_set_scoped() { + let manager = HealManager::new(Arc::new(MockStorage), None); + + manager.block_replacement_recovery_set("pool_0_set_0"); + + assert!(manager.replacement_recovery_set_is_blocked("pool_0_set_0")); + assert!(!manager.replacement_recovery_set_is_blocked("pool_0_set_1")); +} + +#[test] +fn replacement_recovery_blocks_only_confirmed_conflicts() { + assert!(crate::heal::resume::replacement_recovery_error_requires_block( + &Error::TaskExecutionFailed { + message: "replacement recovery conflict: proof mismatch".to_string(), + } + )); + assert!(crate::heal::resume::replacement_recovery_error_requires_block( + &Error::TaskExecutionFailed { + message: "replacement recovery corruption: malformed legacy intent".to_string(), + } + )); + assert!(!crate::heal::resume::replacement_recovery_error_requires_block(&Error::Disk( + DiskError::Timeout + ))); + assert!(!crate::heal::resume::replacement_recovery_error_requires_block( + &Error::TaskExecutionFailed { + message: "Failed to list replacement recovery records: temporary I/O error".to_string(), + } + )); +} + +#[test] +fn replacement_recovery_discovery_unformatted_is_quiet_only_for_deferred_endpoint() { + let error = Error::Disk(DiskError::UnformattedDisk); + let deferred = HashSet::from(["endpoint-a".to_string()]); + + assert!(replacement_discovery_error_is_expected_for_deferred_endpoint( + &error, + "endpoint-a", + &deferred + )); + assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint( + &error, + "endpoint-b", + &deferred + )); + assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint( + &Error::Disk(DiskError::Timeout), + "endpoint-a", + &deferred + )); +} + +#[test] +fn replacement_recovery_retry_barrier_requires_all_set_records_to_validate() { + let mut blocked = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]); + let retry_succeeded = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]); + let retry_failed = HashSet::from(["pool_0_set_0".to_string()]); + + unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed); + + assert!( + blocked.contains("pool_0_set_0"), + "one failed disk record must keep the whole replacement set blocked" + ); + assert!( + !blocked.contains("pool_0_set_1"), + "a blocked set may resume only after every retried record validates" + ); +} + +#[tokio::test] +async fn scheduler_completes_cleanup_pending_recovery_from_manager_anchor() { + let temp = TempDir::new().expect("temporary manager recovery directory should be created"); + let anchor = make_manager_resume_disk(&temp, "anchor").await; + let task_id = ResumeUtils::generate_task_id(); + let target = "replacement-a".to_string(); + let identity = ReplacementTargetIdentity { + endpoint: target.clone(), + canonical_path: "/replacement/replacement-a".to_string(), + physical_device_ids: vec!["replacement-a".to_string()], + filesystem_identity: "identity-replacement-a".to_string(), + }; + let resume_manager = ResumeManager::new_replacement_intent( + anchor.clone(), + task_id.clone(), + "pool_0_set_0".to_string(), + vec!["bucket-a".to_string()], + vec![target.clone()], + vec![identity], + ) + .await + .expect("cleanup-pending replacement state should persist on the survivor anchor"); + resume_manager + .mark_replacement_completed_and_verified() + .await + .expect("completion proof should persist before cleanup"); + resume_manager + .mark_replacement_cleanup_pending() + .await + .expect("cleanup-pending state should persist before restart"); + CheckpointManager::new(anchor.clone(), task_id.clone()) + .await + .expect("checkpoint fixture should persist"); + + let (hook, _hook_guard) = ManagerRecoveryTestHook::install(anchor.clone()); + let storage = Arc::new(MockStorage); + let manager = HealManager::new(storage.clone(), None); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + pool_index: Some(0), + set_index: Some(0), + ..HealOptions::default() + }, + HealPriority::Low, + ); + request.id = task_id.clone(); + request.source = HealRequestSource::AutoHeal; + request.heal_endpoints = vec![target]; + assert_eq!( + manager + .submit_heal_request(request) + .await + .expect("durable recovery request should be admitted"), + HealAdmissionResult::Accepted + ); + manager + .replacement_recovery_anchors + .lock() + .expect("replacement recovery anchor lock should not poison") + .insert(task_id.clone(), anchor.endpoint().to_string()); + + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let resume_removed = !ResumeManager::has_resume_state(&anchor, &task_id).await; + let checkpoint_removed = !CheckpointManager::has_checkpoint(&anchor, &task_id).await; + let anchor_removed = !manager + .replacement_recovery_anchors + .lock() + .expect("replacement recovery anchor lock should not poison") + .contains_key(&task_id); + if resume_removed && checkpoint_removed && anchor_removed { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("cleanup-pending recovery should finish through the manager scheduler"); + + assert!( + !*hook.listed.lock().expect("manager recovery listed lock should not poison"), + "cleanup-pending recovery must not list buckets or restart object healing" + ); + assert_eq!( + *hook + .global_format_calls + .lock() + .expect("manager recovery global format call lock should not poison"), + 0 + ); + assert_eq!( + *hook + .replacement_format_calls + .lock() + .expect("manager recovery replacement format call lock should not poison"), + 0, + "manager-resumed terminal cleanup must not format replacement targets" + ); + assert_eq!( + *hook + .bucket_heal_calls + .lock() + .expect("manager recovery bucket call lock should not poison"), + 0 + ); + assert_eq!( + *hook + .heal_object_calls + .lock() + .expect("manager recovery object call lock should not poison"), + 0 + ); +} + +#[test] +fn test_retry_request_for_scoped_slowdown_preserves_scope() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request( + HealRequest::new( + HealType::Object { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + }, + HealOptions { + pool_index: Some(0), + set_index: Some(1), + ..Default::default() + }, + HealPriority::Normal, + ), + storage, + ); + let result = Err(Error::Storage(EcstoreError::SlowDown)); + + let (retry_request, retry_delay, _) = retry_request_for_result(&task, &result).expect("SlowDown should defer scoped heal"); + + assert_eq!(retry_request.options.pool_index, Some(0)); + assert_eq!(retry_request.options.set_index, Some(1)); + assert_eq!(retry_request.retry_attempts, 1); + assert!(retry_delay > Duration::ZERO); +} + +#[test] +fn test_retry_request_for_typed_not_found_error_is_not_retryable() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage); + let result = Err(Error::Storage(EcstoreError::ObjectNotFound("bucket".to_string(), "object".to_string()))); + + assert!(retry_request_for_result(&task, &result).is_none()); +} + +#[test] +fn test_retry_request_for_recoverable_error_stops_at_limit() { + let storage: Arc = Arc::new(MockStorage); + let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + request.retry_attempts = MAX_RECOVERABLE_HEAL_RETRIES; + let task = HealTask::from_request(request, storage); + let result = Err(Error::TaskExecutionFailed { + message: "Remote lock RPC timed out".to_string(), + }); + + assert!(retry_request_for_result(&task, &result).is_none()); +} + +#[tokio::test] +async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() { + let storage: Arc = Arc::new(MockStorage); + let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage); + let result = Err(task + .record_batch_failure(BatchHealFailure { + scope: "bucket:bucket".to_string(), + failed: 1, + retryable: 1, + permanent: 0, + first_object: "object".to_string(), + first_error: "Lock acquisition timeout".to_string(), + }) + .await); + + assert!(retry_request_for_result(&task, &result).is_none()); +} + +#[test] +fn test_heal_type_matches_path_normalizes_prefix_trailing_slash() { + let heal_type = HealType::Prefix { + bucket: "bucket".to_string(), + prefix: "logs/".to_string(), + }; + + assert!(heal_type_matches_path(&heal_type, "bucket")); + assert!(heal_type_matches_path(&heal_type, "bucket/logs")); + assert!(heal_type_matches_path(&heal_type, "bucket/logs/")); +} + +#[test] +fn test_heal_type_matches_path_normalizes_object_trailing_slash() { + let heal_type = HealType::Object { + bucket: "bucket".to_string(), + object: "object/".to_string(), + version_id: None, + }; + + assert!(heal_type_matches_path(&heal_type, "bucket/object")); + assert!(heal_type_matches_path(&heal_type, "bucket/object/")); +} + +async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> CancellationToken { + let task_id = request.id.clone(); + let cancel_token = CancellationToken::new(); + manager.retrying_heals.lock().await.insert( + task_id.clone(), + RetryingHeal { + request: request.clone(), + error: "Lock acquisition timeout".to_string(), + cancel_token: cancel_token.clone(), + }, + ); + manager.completed_heals.lock().await.insert( + task_id, + Arc::new(CompletedHealStatus { + heal_type: request.heal_type, + status: HealTaskStatus::Retrying { + error: "Lock acquisition timeout".to_string(), + retry_attempt: request.retry_attempts, + }, + result_items_truncated: false, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + completed_at: SystemTime::now(), + }), + ); + cancel_token +} + +#[tokio::test] +async fn test_cancel_task_cancels_retrying_backoff() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + let mut request = HealRequest::bucket("bucket".to_string()); + request.retry_attempts = 1; + let task_id = request.id.clone(); + let cancel_token = insert_retrying_request(&manager, request).await; + + assert!(matches!( + manager + .get_task_status(&task_id) + .await + .expect("retrying task should be queryable"), + HealTaskStatus::Retrying { .. } + )); + + manager + .cancel_task(&task_id) + .await + .expect("retrying task should be cancellable by token"); + + assert!(cancel_token.is_cancelled()); + assert!(manager.retrying_heals.lock().await.get(&task_id).is_none()); + assert!(matches!(manager.get_task_status(&task_id).await, Err(Error::TaskNotFound { .. }))); +} + +#[tokio::test] +async fn test_cancel_tasks_for_path_cancels_retrying_backoff() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + let mut request = HealRequest::bucket("bucket".to_string()); + request.retry_attempts = 1; + let task_id = request.id.clone(); + let cancel_token = insert_retrying_request(&manager, request).await; + + assert_eq!( + manager + .cancel_tasks_for_path("bucket") + .await + .expect("retrying task should be cancellable by path"), + 1 + ); + + assert!(cancel_token.is_cancelled()); + assert!(manager.retrying_heals.lock().await.get(&task_id).is_none()); + assert!(matches!(manager.get_task_status(&task_id).await, Err(Error::TaskNotFound { .. }))); +} + +#[tokio::test] +async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); + let cluster_request_id = cluster_request.id.clone(); + let bucket_request = HealRequest::bucket("bucket".to_string()); + let bucket_request_id = bucket_request.id.clone(); + + manager + .submit_heal_request(cluster_request) + .await + .expect("cluster request should be accepted"); + manager + .submit_heal_request(bucket_request) + .await + .expect("bucket request should be accepted"); + + assert_eq!( + manager + .cancel_tasks_for_path("") + .await + .expect("root path should cancel queued cluster task"), + 1 + ); + assert!(matches!( + manager.get_task_status(&cluster_request_id).await, + Err(Error::TaskNotFound { .. }) + )); + assert_eq!( + manager + .get_task_status(&bucket_request_id) + .await + .expect("bucket request should not match root path"), + HealTaskStatus::Pending + ); +} + +#[tokio::test] +async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage.clone(), None); + + let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); + let cluster_request_id = cluster_request.id.clone(); + let bucket_request = HealRequest::bucket("bucket".to_string()); + let bucket_request_id = bucket_request.id.clone(); + + manager.active_heals.lock().await.insert( + cluster_request_id.clone(), + Arc::new(HealTask::from_request(cluster_request, storage.clone())), + ); + manager + .active_heals + .lock() + .await + .insert(bucket_request_id.clone(), Arc::new(HealTask::from_request(bucket_request, storage))); + + assert_eq!( + manager + .cancel_tasks_for_path("") + .await + .expect("root path should cancel active cluster task"), + 1 + ); + assert!(manager.active_heals.lock().await.get(&cluster_request_id).is_none()); + assert!(manager.active_heals.lock().await.get(&bucket_request_id).is_some()); +} + +#[tokio::test] +async fn test_cancel_tasks_for_empty_path_cancels_retrying_cluster_only() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let mut cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High); + cluster_request.retry_attempts = 1; + let cluster_request_id = cluster_request.id.clone(); + let cluster_cancel_token = insert_retrying_request(&manager, cluster_request).await; + + let mut bucket_request = HealRequest::bucket("bucket".to_string()); + bucket_request.retry_attempts = 1; + let bucket_request_id = bucket_request.id.clone(); + let bucket_cancel_token = insert_retrying_request(&manager, bucket_request).await; + + assert_eq!( + manager + .cancel_tasks_for_path("") + .await + .expect("root path should cancel retrying cluster task"), + 1 + ); + assert!(cluster_cancel_token.is_cancelled()); + assert!(!bucket_cancel_token.is_cancelled()); + assert!(manager.retrying_heals.lock().await.get(&cluster_request_id).is_none()); + assert!(manager.retrying_heals.lock().await.get(&bucket_request_id).is_some()); +} + +#[test] +fn test_heal_type_matches_path_accepts_legacy_root() { + assert!(heal_type_matches_path(&HealType::Cluster, LEGACY_ROOT_HEAL_PATH)); + assert!(!heal_type_matches_path( + &HealType::Bucket { + bucket: "bucket".to_string(), + }, + LEGACY_ROOT_HEAL_PATH, + )); +} + +#[tokio::test] +async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + let mut original_request = HealRequest::bucket("bucket".to_string()); + original_request.retry_attempts = 1; + let original_task_id = original_request.id.clone(); + let cancel_token = insert_retrying_request(&manager, original_request).await; + + let duplicate_request = HealRequest::bucket("bucket".to_string()); + let duplicate_task_id = duplicate_request.id.clone(); + + assert_eq!( + manager + .submit_heal_request(duplicate_request) + .await + .expect("retrying duplicate should produce admission result"), + HealAdmissionResult::Merged + ); + assert!(matches!( + manager + .get_task_status_for_path("bucket", &duplicate_task_id) + .await + .expect("duplicate token should query merged retrying task"), + HealTaskStatus::Retrying { .. } + )); + + manager + .cancel_task(&duplicate_task_id) + .await + .expect("duplicate token should cancel merged retrying task"); + + assert!(cancel_token.is_cancelled()); + assert!(manager.retrying_heals.lock().await.get(&original_task_id).is_none()); +} + +#[tokio::test] +async fn test_get_task_status_reports_pending_for_queued_request() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let request = HealRequest::bucket("bucket".to_string()); + let request_id = request.id.clone(); + + assert_eq!( + manager + .submit_heal_request(request) + .await + .expect("request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .get_task_status(&request_id) + .await + .expect("queued request should have status"), + HealTaskStatus::Pending + ); +} + +#[tokio::test] +async fn test_operations_snapshot_counts_queue_by_source_and_priority() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let mut scanner_request = HealRequest::new( + HealType::Object { + bucket: "bucket-a".to_string(), + object: "object-a".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::Low, + ); + scanner_request.source = HealRequestSource::Scanner; + + let mut admin_request = HealRequest::bucket("bucket-b".to_string()); + admin_request.priority = HealPriority::High; + admin_request.source = HealRequestSource::Admin; + + let mut auto_request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-c".to_string()], + set_disk_id: "0-0".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + auto_request.source = HealRequestSource::AutoHeal; + + manager + .submit_heal_request(scanner_request) + .await + .expect("scanner request should be accepted"); + manager + .submit_heal_request(admin_request) + .await + .expect("admin request should be accepted"); + manager + .submit_heal_request(auto_request) + .await + .expect("auto request should be accepted"); + + let snapshot = manager.operations_snapshot().await; + + assert_eq!(snapshot.queue_length, 3); + assert_eq!(snapshot.active_tasks, 0); + assert_eq!(snapshot.queued_by_priority.low, 1); + assert_eq!(snapshot.queued_by_priority.normal, 1); + assert_eq!(snapshot.queued_by_priority.high, 1); + assert_eq!(snapshot.queued_by_priority.urgent, 0); + assert_eq!(snapshot.queued_by_source.scanner, 1); + assert_eq!(snapshot.queued_by_source.admin, 1); + assert_eq!(snapshot.queued_by_source.auto_heal, 1); + assert_eq!(snapshot.queued_by_source.internal, 0); +} + +// HS-06 (backlog#1870): overlap policy + forceStart semantics. +fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager { + let storage: Arc = Arc::new(MockStorage); + HealManager::new( + storage, + Some(HealConfig { + overlap_policy: policy, + ..Default::default() + }), + ) +} + +fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest { + let mut request = HealRequest::new( + HealType::Prefix { + bucket: bucket.to_string(), + prefix: prefix.to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + request.source = HealRequestSource::Admin; + request +} + +async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String { + let task = Arc::new(HealTask::from_request(request, manager.storage.clone())); + let task_id = task.id.clone(); + manager.active_heals.lock().await.insert(task_id.clone(), task); + task_id +} + +#[tokio::test] +async fn overlap_policy_minio_error_rejects_same_and_containing_paths() { + let manager = manager_with_policy(HealOverlapPolicy::MinioError); + insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + + // Same target: typed AlreadyRunning. + let same = manager + .submit_heal_request(admin_prefix_request("bucket-a", "logs/")) + .await + .expect("admission must decide"); + assert_eq!( + same, + HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning), + "an identical target must reject with already-running" + ); + + // Contained path: typed OverlappingPaths. + let nested = manager + .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/")) + .await + .expect("admission must decide"); + assert_eq!( + nested, + HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths), + "a path inside the active task's path must reject with overlapping-paths" + ); + + // Containing path (bucket-wide vs nested active): also overlapping. + let wide = manager + .submit_heal_request(admin_prefix_request("bucket-a", "")) + .await + .expect("admission must decide"); + assert_eq!( + wide, + HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths), + "a bucket-wide start overlapping a nested active heal must reject" + ); + + // Disjoint bucket: unaffected. + let disjoint = manager + .submit_heal_request(admin_prefix_request("bucket-b", "logs/")) + .await + .expect("admission must decide"); + assert_eq!(disjoint, HealAdmissionResult::Accepted); +} + +#[tokio::test] +async fn overlap_policy_default_merge_keeps_today_semantics() { + let manager = manager_with_policy(HealOverlapPolicy::Merge); + insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + + // Different-dedup-key overlap still merges under the default policy: + // the nested path dedups to its own key but nothing rejects it. + let nested = manager + .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/")) + .await + .expect("admission must decide"); + assert_eq!(nested, HealAdmissionResult::Accepted, "default policy must not reject overlaps"); + + // Non-admin sources never get overlap rejections even under minio_error. + let manager = manager_with_policy(HealOverlapPolicy::MinioError); + insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + let mut scanner_request = admin_prefix_request("bucket-a", "logs/app/"); + scanner_request.source = HealRequestSource::Scanner; + let admitted = manager + .submit_heal_request(scanner_request) + .await + .expect("admission must decide"); + assert_eq!(admitted, HealAdmissionResult::Accepted, "scanner sources must never be overlap-rejected"); +} + +#[tokio::test] +async fn admin_force_start_cancels_overlapping_active_task_first() { + let manager = manager_with_policy(HealOverlapPolicy::Merge); + let old_id = insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await; + + let mut replacement = admin_prefix_request("bucket-a", "logs/"); + replacement.force_start = true; + let receipt = manager + .submit_heal_request_with_receipt(replacement) + .await + .expect("force-start submission must decide"); + + assert!(receipt.result.is_admitted(), "the new task must be admitted (Accepted or Merged)"); + let old_task_gone = { + let active_heals = manager.active_heals.lock().await; + !active_heals.contains_key(&old_id) + }; + assert!( + old_task_gone, + "the overlapping admin task must be cancelled (removed from the active table) before the new one starts" + ); + assert!( + matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })), + "a cancelled task must no longer resolve as an active heal" + ); +} + +#[tokio::test] +async fn test_operations_snapshot_counts_active_by_source_and_priority() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let mut request = HealRequest::bucket("bucket-a".to_string()); + request.priority = HealPriority::High; + request.source = HealRequestSource::Admin; + let task = Arc::new(HealTask::from_request(request, manager.storage.clone())); + let task_id = task.id.clone(); + + manager.active_heals.lock().await.insert(task_id, task); + + let snapshot = manager.operations_snapshot().await; + + assert_eq!(snapshot.queue_length, 0); + assert_eq!(snapshot.active_tasks, 1); + assert_eq!(snapshot.active_by_priority.high, 1); + assert_eq!(snapshot.active_by_source.admin, 1); + assert_eq!(snapshot.active_by_source.scanner, 0); +} + +#[tokio::test] +async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + let mut request = HealRequest::bucket("bucket-retry".to_string()); + request.priority = HealPriority::Urgent; + request.source = HealRequestSource::Admin; + let task_id = request.id.clone(); + manager.retrying_heals.lock().await.insert( + task_id, + RetryingHeal { + request, + error: "transient".to_string(), + cancel_token: CancellationToken::new(), + }, + ); + + let snapshot = manager.operations_snapshot().await; + + assert_eq!(snapshot.queue_length, 0); + assert_eq!(snapshot.active_tasks, 0); + assert_eq!(snapshot.retrying_tasks, 1); + assert_eq!(snapshot.retrying_by_priority.urgent, 1); + assert_eq!(snapshot.retrying_by_source.admin, 1); +} + +#[tokio::test] +async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() { + let storage: Arc = Arc::new(MockStorage); + let manager = Arc::new(HealManager::new(storage, None)); + { + let mut config = manager.config.write().await; + config.enable_auto_heal = false; + config.heal_interval = Duration::from_millis(10); + config.event_driven_scheduler_enable = true; + } + manager.start().await.expect("manager should start"); + + let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None); + let task_id = request.id.clone(); + let hook = Arc::new(RetryOwnershipTestHook { + task_id: task_id.clone(), + active_to_retrying_reached: Notify::new(), + active_to_retrying_release: Notify::new(), + retrying_to_queue_reached: Notify::new(), + retrying_to_queue_release: Notify::new(), + }); + *RETRY_OWNERSHIP_TEST_HOOK.lock().await = Some(hook.clone()); + + manager + .submit_heal_request(request) + .await + .expect("retry test request should be accepted"); + tokio::time::timeout(Duration::from_secs(2), hook.active_to_retrying_reached.notified()) + .await + .expect("active to retrying transition should be reached"); + + let snapshot_manager = manager.clone(); + let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(), + "snapshot must wait while active and retrying ownership locks are held" + ); + hook.active_to_retrying_release.notify_one(); + let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot) + .await + .expect("snapshot should resume after active to retrying handoff") + .expect("snapshot task should complete"); + assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1); + assert_eq!(snapshot.retrying_tasks, 1); + + tokio::time::timeout(Duration::from_secs(5), hook.retrying_to_queue_reached.notified()) + .await + .expect("retrying to queue transition should be reached"); + let snapshot_manager = manager.clone(); + let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(), + "snapshot must wait while queue ownership is transferred" + ); + hook.retrying_to_queue_release.notify_one(); + // The scheduler may immediately execute the retried request again; + // leave a permit so a second test-only active handoff cannot stall it. + hook.active_to_retrying_release.notify_one(); + let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot) + .await + .expect("snapshot should resume after retrying to queue handoff") + .expect("snapshot task should complete"); + assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1); + + *RETRY_OWNERSHIP_TEST_HOOK.lock().await = None; + hook.active_to_retrying_release.notify_one(); + hook.retrying_to_queue_release.notify_one(); + tokio::time::timeout(Duration::from_secs(1), manager.stop()) + .await + .expect("manager stop should not stall") + .expect("manager should stop"); +} + +#[tokio::test] +async fn test_active_progress_snapshot_sums_active_task_progress() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let first = Arc::new(HealTask::from_request( + HealRequest::bucket("bucket-a".to_string()), + manager.storage.clone(), + )); + { + let mut progress = first.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(20)); + progress.set_total_baseline(12, 8192); + progress.update_progress(7, 3, 1, 4096); + } + + let second = Arc::new(HealTask::from_request( + HealRequest::bucket("bucket-b".to_string()), + manager.storage.clone(), + )); + { + let mut progress = second.progress.write().await; + progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); + progress.set_total_baseline(8, 4096); + progress.update_progress(11, 5, 2, 2048); + } + + manager.active_heals.lock().await.insert(first.id.clone(), first); + manager.active_heals.lock().await.insert(second.id.clone(), second); + + let progress = manager + .active_progress_snapshot() + .await + .expect("active progress should exist"); + + assert_eq!(progress.objects_scanned, 18); + assert_eq!(progress.objects_healed, 8); + assert_eq!(progress.objects_failed, 3); + assert_eq!(progress.objects_total_count, 20); + assert_eq!(progress.objects_total_size, 12288); + assert_eq!(progress.bytes_processed, 6144); + assert!((progress.progress_percentage - 50.0).abs() < 0.001); + assert!(progress.estimated_completion_time.is_some()); +} + +#[tokio::test] +async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + manager + .submit_heal_request(HealRequest::bucket("bucket".to_string())) + .await + .expect("request should be accepted"); + + assert!(matches!( + manager.get_task_status_for_path("bucket", "wrong-token").await, + Err(Error::InvalidClientToken) + )); +} + +#[tokio::test] +async fn test_get_task_status_for_path_rejects_token_from_other_active_path() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let bucket_request = HealRequest::bucket("bucket".to_string()); + let other_request = HealRequest::bucket("other".to_string()); + let other_request_id = other_request.id.clone(); + + manager + .submit_heal_request(bucket_request) + .await + .expect("bucket request should be accepted"); + manager + .submit_heal_request(other_request) + .await + .expect("other request should be accepted"); + + assert!(matches!( + manager.get_task_status_for_path("bucket", &other_request_id).await, + Err(Error::InvalidClientToken) + )); +} + +#[tokio::test] +async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let request = HealRequest::bucket("bucket".to_string()); + let request_id = request.id.clone(); + + manager + .submit_heal_request(request) + .await + .expect("request should be accepted"); + + assert!(matches!( + manager.get_task_status_for_path("other", &request_id).await, + Err(Error::TaskNotFound { .. }) + )); +} + +#[tokio::test] +async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + assert!(matches!( + manager.get_task_status_for_path("bucket", "old-token").await, + Err(Error::TaskNotFound { .. }) + )); +} + +#[tokio::test] +async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let request = HealRequest::bucket("bucket".to_string()); + let request_id = request.id.clone(); + + manager + .submit_heal_request(request) + .await + .expect("request should be accepted"); + + assert!(matches!( + manager.get_task_status_for_path("", &request_id).await, + Err(Error::TaskNotFound { .. }) + )); + assert!(matches!( + manager.get_task_status_for_path("", "wrong-token").await, + Err(Error::TaskNotFound { .. }) + )); +} + +#[tokio::test] +async fn test_get_task_report_queries_queued_task_by_token_without_path() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let request = HealRequest::new( + HealType::ErasureSet { + buckets: vec![], + set_disk_id: "pool_0_set_1".to_string(), + }, + HealOptions::default(), + HealPriority::High, + ); + let request_id = request.id.clone(); + + manager + .submit_heal_request(request) + .await + .expect("request should be accepted"); + + let report = manager + .get_task_report(&request_id) + .await + .expect("queued task should be queryable by token"); + + assert_eq!(report.status, HealTaskStatus::Pending); + assert!(report.result_items.is_empty()); +} + +#[tokio::test] +async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + // A completed entry recorded in a Retrying state for a task whose + // request is also (still) queued under the same id: the retrying + // completion must win the lookup, or the task would read back as + // Pending while it is actually waiting out a retry backoff. + let request = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let task_id = request.id.clone(); + manager.completed_heals.lock().await.insert( + task_id.clone(), + Arc::new(CompletedHealStatus { + heal_type: request.heal_type.clone(), + status: HealTaskStatus::Retrying { + error: "transient disk failure".to_string(), + retry_attempt: 1, + }, + result_items_truncated: false, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + completed_at: SystemTime::now(), + }), + ); + manager.heal_queue.lock().await.push(HealRequest { + id: task_id.clone(), + heal_type: request.heal_type, + ..request + }); + + assert_eq!( + manager.get_task_status(&task_id).await.expect("task must resolve"), + HealTaskStatus::Retrying { + error: "transient disk failure".to_string(), + retry_attempt: 1 + } + ); +} + +#[tokio::test] +async fn test_get_task_status_reads_recent_completed_status() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + manager.completed_heals.lock().await.insert( + "completed-token".to_string(), + Arc::new(CompletedHealStatus { + heal_type: HealType::Bucket { + bucket: "bucket".to_string(), + }, + status: HealTaskStatus::Completed, + result_items_truncated: false, + seqed_items: Vec::new(), + next_seq: 0, + min_seq: 0, + completed_at: SystemTime::now(), + }), + ); + + assert_eq!( + manager + .get_task_status_for_path("bucket", "completed-token") + .await + .expect("recent completed task should be queryable"), + HealTaskStatus::Completed + ); +} + +#[tokio::test] +async fn test_get_task_report_for_path_reads_completed_items() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + manager.completed_heals.lock().await.insert( + "completed-token".to_string(), + Arc::new(CompletedHealStatus { + heal_type: HealType::Object { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + }, + status: HealTaskStatus::Completed, + result_items_truncated: true, + seqed_items: vec![( + 1, + HealResultItem { + bucket: "bucket".to_string(), + object: "object".to_string(), + object_size: 1024, + ..Default::default() + }, + )], + next_seq: 2, + min_seq: 1, + completed_at: SystemTime::now(), + }), + ); + + let report = manager + .get_task_report_for_path("bucket/object", "completed-token") + .await + .expect("recent completed task report should be queryable"); + assert!(report.result_items_truncated); + + assert_eq!(report.status, HealTaskStatus::Completed); + assert_eq!(report.result_items.len(), 1); + assert_eq!(report.result_items[0].object_size, 1024); + // The archived cursors pass through to the report so an incremental + // consumer can resume against the next expected sequence. + assert_eq!(report.next_seq, 2); + assert_eq!(report.min_seq, 1); +} + +#[tokio::test] +async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + manager + .submit_heal_request(HealRequest::bucket("bucket".to_string())) + .await + .expect("request should be accepted"); + + assert!(matches!( + manager.get_task_report_for_path("", "wrong-token").await, + Err(Error::TaskNotFound { .. }) + )); +} + +#[tokio::test] +async fn test_cancel_task_removes_queued_request() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let request = HealRequest::bucket("bucket".to_string()); + let request_id = request.id.clone(); + + manager + .submit_heal_request(request) + .await + .expect("request should be accepted"); + manager + .cancel_task(&request_id) + .await + .expect("queued request should be cancelled"); + + assert!(matches!(manager.get_task_status(&request_id).await, Err(Error::TaskNotFound { .. }))); +} + +#[tokio::test] +async fn test_cancel_tasks_for_path_removes_matching_queued_requests() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new(storage, None); + + let bucket_request = HealRequest::bucket("bucket".to_string()); + let bucket_request_id = bucket_request.id.clone(); + let other_request = HealRequest::bucket("other".to_string()); + let other_request_id = other_request.id.clone(); + + manager + .submit_heal_request(bucket_request) + .await + .expect("bucket request should be accepted"); + manager + .submit_heal_request(other_request) + .await + .expect("other request should be accepted"); + + assert_eq!( + manager + .cancel_tasks_for_path("bucket") + .await + .expect("matching request should be cancelled"), + 1 + ); + assert!(matches!( + manager.get_task_status(&bucket_request_id).await, + Err(Error::TaskNotFound { .. }) + )); + assert_eq!( + manager + .get_task_status(&other_request_id) + .await + .expect("unmatched request should remain queued"), + HealTaskStatus::Pending + ); +} + +#[tokio::test] +async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + ); + + let request = HealRequest::new( + HealType::Object { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::Low, + ); + + assert_eq!( + manager + .submit_heal_request(request.clone()) + .await + .expect("first request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(request) + .await + .expect("duplicate request should merge even when queue is full"), + HealAdmissionResult::Merged + ); +} + +#[tokio::test] +async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 1, + low_priority_drop_when_full: true, + ..HealConfig::default() + }), + ); + + let accepted = HealRequest::new( + HealType::Bucket { + bucket: "bucket-a".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + let dropped = HealRequest::new( + HealType::Bucket { + bucket: "bucket-b".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + + assert_eq!( + manager + .submit_heal_request(accepted) + .await + .expect("first request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(dropped) + .await + .expect("low priority request should be dropped with explicit admission result"), + HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull) + ); +} + +#[tokio::test] +async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + ); + + let accepted = HealRequest::new( + HealType::Bucket { + bucket: "bucket-a".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + let full = HealRequest::new( + HealType::Bucket { + bucket: "bucket-b".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ); + + assert_eq!( + manager + .submit_heal_request(accepted) + .await + .expect("first request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(full) + .await + .expect("normal priority request should surface full admission"), + HealAdmissionResult::Full + ); +} + +#[tokio::test] +async fn test_high_priority_request_displaces_lower_priority_when_queue_full() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + ); + + let low = HealRequest::new( + HealType::Bucket { + bucket: "background-bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + let low_id = low.id.clone(); + let high = HealRequest::new( + HealType::Bucket { + bucket: "manual-bucket".to_string(), + }, + HealOptions::default(), + HealPriority::High, + ); + let high_id = high.id.clone(); + + assert_eq!( + manager + .submit_heal_request(low) + .await + .expect("low priority request should be accepted first"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(high) + .await + .expect("high priority request should be admitted by displacing lower priority work"), + HealAdmissionResult::Accepted + ); + assert_eq!(manager.get_queue_length().await, 1); + assert!(matches!(manager.get_task_status(&low_id).await, Err(Error::TaskNotFound { .. }))); + assert_eq!( + manager + .get_task_status(&high_id) + .await + .expect("high priority request should remain queued"), + HealTaskStatus::Pending + ); +} + +#[tokio::test] +async fn test_submit_heal_request_drops_read_repair_under_pressure() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 10, + ..HealConfig::default() + }), + ); + + for index in 0..8 { + assert_eq!( + manager + .submit_heal_request(bucket_request( + &format!("queued-{index}"), + HealPriority::Normal, + HealRequestSource::Internal, + )) + .await + .expect("seed request should be accepted"), + HealAdmissionResult::Accepted + ); + } + + let admission = manager + .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)) + .await + .expect("read repair admission should return a result"); + + assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)); + assert_eq!(manager.get_queue_length().await, 8); +} + +#[tokio::test] +async fn test_submit_heal_request_drops_low_scanner_under_pressure() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 10, + ..HealConfig::default() + }), + ); + + for index in 0..8 { + assert_eq!( + manager + .submit_heal_request(bucket_request( + &format!("queued-{index}"), + HealPriority::Normal, + HealRequestSource::Internal, + )) + .await + .expect("seed request should be accepted"), + HealAdmissionResult::Accepted + ); + } + + let admission = manager + .submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)) + .await + .expect("scanner admission should return a result"); + + assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)); + assert_eq!(manager.get_queue_length().await, 8); +} + +#[tokio::test] +async fn test_submit_heal_request_accepts_admin_high_under_pressure() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 10, + ..HealConfig::default() + }), + ); + + for index in 0..8 { + assert_eq!( + manager + .submit_heal_request(bucket_request( + &format!("queued-{index}"), + HealPriority::Normal, + HealRequestSource::Internal, + )) + .await + .expect("seed request should be accepted"), + HealAdmissionResult::Accepted + ); + } + + let admission = manager + .submit_heal_request(bucket_request("admin", HealPriority::High, HealRequestSource::Admin)) + .await + .expect("admin admission should return a result"); + + assert_eq!(admission, HealAdmissionResult::Accepted); + assert_eq!(manager.get_queue_length().await, 9); +} + +#[tokio::test] +async fn test_mainline_throttle_delays_background_heal_start() { + let storage: Arc = Arc::new(MockStorage); + let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider { + class: WorkloadClass::ForegroundRead, + active: 8, + limit: 10, + state: AdmissionState::Open, + }); + let manager = HealManager::new_with_workload_provider( + storage, + Some(HealConfig { + max_concurrent_heals: 1, + mainline_throttle_enable: true, + mainline_read_utilization_high_percent: 80, + mainline_write_utilization_high_percent: 80, + mainline_max_sleep: Duration::from_millis(1), + ..HealConfig::default() + }), + Some(provider), + ); + + manager + .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)) + .await + .expect("read repair request should be queued"); + + process_manager_queue_once(&manager).await; + + assert_eq!(manager.get_queue_length().await, 1); + assert_eq!(manager.get_active_task_count().await, 0); +} + +#[tokio::test] +async fn test_mainline_throttle_delays_background_heal_start_under_write_pressure() { + let storage: Arc = Arc::new(MockStorage); + let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider { + class: WorkloadClass::ForegroundWrite, + active: 9, + limit: 10, + state: AdmissionState::Open, + }); + let manager = HealManager::new_with_workload_provider( + storage, + Some(HealConfig { + max_concurrent_heals: 1, + mainline_throttle_enable: true, + mainline_read_utilization_high_percent: 80, + mainline_write_utilization_high_percent: 80, + mainline_max_sleep: Duration::from_millis(1), + ..HealConfig::default() + }), + Some(provider), + ); + + manager + .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)) + .await + .expect("read repair request should be queued"); + + process_manager_queue_once(&manager).await; + + assert_eq!(manager.get_queue_length().await, 1); + assert_eq!(manager.get_active_task_count().await, 0); +} + +#[tokio::test] +async fn test_mainline_throttle_allows_admin_high_start() { + let storage: Arc = Arc::new(MockStorage); + let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider { + class: WorkloadClass::ForegroundRead, + active: 10, + limit: 10, + state: AdmissionState::Saturated, + }); + let manager = HealManager::new_with_workload_provider( + storage, + Some(HealConfig { + max_concurrent_heals: 1, + mainline_throttle_enable: true, + mainline_read_utilization_high_percent: 80, + mainline_write_utilization_high_percent: 80, + mainline_max_sleep: Duration::from_millis(1), + ..HealConfig::default() + }), + Some(provider), + ); + + manager + .submit_heal_request(bucket_request("admin", HealPriority::High, HealRequestSource::Admin)) + .await + .expect("admin request should be queued"); + + process_manager_queue_once(&manager).await; + + assert_eq!(manager.get_queue_length().await, 0); +} + +#[tokio::test] +async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + max_concurrent_heals: 1, + task_timeout: Duration::ZERO, + ..HealConfig::default() + }), + ); + + let mut defaulted = bucket_request("defaulted-timeout", HealPriority::Normal, HealRequestSource::Admin); + defaulted.options.timeout = None; + let defaulted_id = defaulted.id.clone(); + manager + .submit_heal_request(defaulted) + .await + .expect("request without timeout should be queued"); + process_manager_queue_once(&manager).await; + let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async { + loop { + if let Ok(status @ HealTaskStatus::Timeout) = manager.get_task_status(&defaulted_id).await { + break status; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("configured timeout should finish the task"); + assert_eq!(defaulted_status, HealTaskStatus::Timeout); + assert!(manager.retrying_heals.lock().await.get(&defaulted_id).is_none()); + + let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin); + explicit.options.timeout = Some(Duration::from_secs(60)); + let explicit_id = explicit.id.clone(); + manager + .submit_heal_request(explicit) + .await + .expect("request with explicit timeout should be queued"); + process_manager_queue_once(&manager).await; + let explicit_status = tokio::time::timeout(Duration::from_secs(1), async { + loop { + if let Ok(status @ HealTaskStatus::Failed { .. }) = manager.get_task_status(&explicit_id).await { + break status; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("explicit timeout request should finish without using the zero default"); + assert!(matches!(explicit_status, HealTaskStatus::Failed { .. })); +} + +#[tokio::test] +async fn test_force_start_bypasses_duplicate_and_full_admission() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 1, + low_priority_drop_when_full: true, + ..HealConfig::default() + }), + ); + + let normal = HealRequest::new( + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + let mut forced_duplicate = HealRequest::new( + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + forced_duplicate.force_start = true; + + let subsequent_duplicate = HealRequest::new( + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + + assert_eq!( + manager + .submit_heal_request(normal) + .await + .expect("first request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(forced_duplicate) + .await + .expect("force start should bypass duplicate/full policy"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(subsequent_duplicate) + .await + .expect("subsequent non-force duplicate should be merged"), + HealAdmissionResult::Merged + ); +} + +#[tokio::test] +async fn test_force_start_marks_dedup_key_for_future_duplicates() { + let storage: Arc = Arc::new(MockStorage); + let manager = HealManager::new( + storage, + Some(HealConfig { + queue_size: 1, + ..HealConfig::default() + }), + ); + + let normal = HealRequest::new( + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + let mut forced = HealRequest::new( + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + forced.force_start = true; + let duplicate = HealRequest::new( + HealType::Bucket { + bucket: "bucket".to_string(), + }, + HealOptions::default(), + HealPriority::Low, + ); + + assert_eq!( + manager + .submit_heal_request(normal) + .await + .expect("first request should be accepted"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(forced) + .await + .expect("forced request should bypass duplicate/full admission"), + HealAdmissionResult::Accepted + ); + assert_eq!( + manager + .submit_heal_request(duplicate) + .await + .expect("non-forced duplicate should merge while forced request is queued"), + HealAdmissionResult::Merged + ); +} + +#[test] +fn test_running_heal_set_counts_groups_set_scoped_tasks() { + let storage: Arc = Arc::new(MockStorage); + let erasure_task = Arc::new(HealTask::from_request( + HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket".to_string()], + set_disk_id: "pool_0_set_1".to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ), + storage.clone(), + )); + let scoped_options = HealOptions { + pool_index: Some(0), + set_index: Some(1), + ..Default::default() + }; + let scoped_object_task = Arc::new(HealTask::from_request( + HealRequest::new( + HealType::Object { + bucket: "bucket".to_string(), + object: "scoped-object".to_string(), + version_id: None, + }, + scoped_options, + HealPriority::Normal, + ), + storage.clone(), + )); + let object_task = Arc::new(HealTask::from_request( + HealRequest::new( + HealType::Object { + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + }, + HealOptions::default(), + HealPriority::Normal, + ), + storage, + )); + + let mut active = HashMap::new(); + active.insert(erasure_task.id.clone(), erasure_task); + active.insert(scoped_object_task.id.clone(), scoped_object_task); + active.insert(object_task.id.clone(), object_task); + + let counts = running_heal_set_counts(&active); + assert_eq!(counts.get("pool_0_set_1"), Some(&2)); + assert_eq!(counts.len(), 1); +} + +#[test] +fn test_heal_config_respects_feature_flags() { + temp_env::with_vars( + [ + (rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE, Some("false")), + (rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE, Some("false")), + (rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("false")), + ], + || { + let config = HealConfig::default(); + assert!(!config.event_driven_scheduler_enable); + assert!(!config.set_bulkhead_enable); + assert!(!config.page_parallel_enable); + }, + ); +} diff --git a/crates/heal/src/heal/manager/unclean_shutdown.rs b/crates/heal/src/heal/manager/unclean_shutdown.rs new file mode 100644 index 000000000..9b954223d --- /dev/null +++ b/crates/heal/src/heal/manager/unclean_shutdown.rs @@ -0,0 +1,390 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/// Unclean-shutdown recovery: durable replacement-intent discovery and healing-marker rewrite. +use super::*; + +pub(super) fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bool { + state.replacement_generation.as_deref() == Some(task_id) + && !state.replacement_targets.is_empty() + && ((!state.completed + && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) + && state.retry_count >= state.max_retries) + || (state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending))) +} + +pub(super) fn replacement_discovery_error_is_expected_for_deferred_endpoint( + error: &Error, + endpoint: &str, + deferred_replacement_endpoints: &HashSet, +) -> bool { + matches!(error, Error::Disk(DiskError::UnformattedDisk)) && deferred_replacement_endpoints.contains(endpoint) +} + +pub(super) fn unblock_replacement_recovery_sets_after_validation( + blocked_sets: &mut HashSet, + retry_succeeded: HashSet, + retry_failed: &HashSet, +) { + for set_disk_id in retry_succeeded { + if !retry_failed.contains(&set_disk_id) { + blocked_sets.remove(&set_disk_id); + } + } +} + +impl HealManager { + /// Detect whether the previous run ended without a clean shutdown and, if so, + /// enqueue a full erasure-set heal for every local set. Also (re)writes the + /// marker for the current run; [`super::super::clear_unclean_shutdown_markers`] + /// removes it again during graceful shutdown. Best-effort: failures only log. + pub(super) async fn process_unclean_shutdown(&self) { + let mut unclean = false; + let mut set_disk_ids = HashSet::new(); + let mut replacement_intents = HashMap::, Vec, String)>::new(); + let mut replacement_restarts = HashMap::)>::new(); + let mut conflicted_replacement_sets = HashSet::new(); + + { + let local_disks = { + let local_disk_map = local_disk_map_read().await; + local_disk_map.values().flatten().cloned().collect::>() + }; + for disk in &local_disks { + let endpoint = disk.endpoint(); + match disk + .read_all(super::super::RUSTFS_META_BUCKET, super::super::UNCLEAN_SHUTDOWN_MARKER_PATH) + .await + { + Ok(_) => unclean = true, + Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {} + Err(err) => { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + error = ?err, + "Unclean-shutdown marker check failed" + ); + } + } + + let marker = SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs().to_string()) + .unwrap_or_default(); + if let Err(err) = disk + .write_all( + super::super::RUSTFS_META_BUCKET, + super::super::UNCLEAN_SHUTDOWN_MARKER_PATH, + marker.into(), + ) + .await + { + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + error = ?err, + "Unclean-shutdown marker write failed" + ); + } + + let disk_set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx); + if let Some(set_disk_id) = &disk_set_disk_id { + set_disk_ids.insert(set_disk_id.clone()); + } + + // Legacy flat records are inspected only while starting. The + // periodic scanner lists the dedicated replacement directory. + if let Err(error) = ResumeUtils::migrate_legacy_replacement_records(disk).await { + if let Some(set_disk_id) = &disk_set_disk_id { + self.block_replacement_recovery_set(set_disk_id); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + error = %error, + "Legacy replacement recovery migration failed" + ); + } + let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await { + Ok(task_ids) => task_ids, + Err(error) => { + if let Some(set_disk_id) = &disk_set_disk_id { + self.block_replacement_recovery_set(set_disk_id); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + error = %error, + "Replacement recovery discovery failed" + ); + continue; + } + }; + for task_id in replacement_task_ids { + let manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await { + Ok(manager) => manager, + Err(error) => { + if let Some(set_disk_id) = &disk_set_disk_id { + self.block_replacement_recovery_set(set_disk_id); + } + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + endpoint = %endpoint, + task_id, + error = %error, + "Replacement recovery intent load failed" + ); + continue; + } + }; + let state = manager.get_state().await; + let active_replacement = !state.completed + && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding); + let verified_replacement = state.completed + && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending); + if (active_replacement || verified_replacement) + && state.replacement_generation.as_deref() == Some(task_id.as_str()) + && !state.replacement_targets.is_empty() + { + if matches!(state.replacement_phase, ReplacementPhase::CleanupPending) { + replacement_intents.entry(task_id).or_insert(( + state.set_disk_id, + state.replacement_targets, + state.replacement_buckets, + endpoint.to_string(), + )); + continue; + } + match self.storage.replacement_target_identities(&state.replacement_targets).await { + Ok(identities) if identities == state.replacement_target_identities => { + let resume_endpoint = endpoint.to_string(); + match replacement_intents.entry(task_id) { + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(( + state.set_disk_id, + state.replacement_targets, + state.replacement_buckets, + resume_endpoint, + )); + } + std::collections::hash_map::Entry::Occupied(entry) => { + let (existing_set_disk_id, existing_targets, existing_buckets, existing_anchor) = + entry.get(); + if existing_set_disk_id != &state.set_disk_id + || existing_targets != &state.replacement_targets + || existing_buckets != &state.replacement_buckets + || existing_anchor != &resume_endpoint + { + conflicted_replacement_sets.insert(state.set_disk_id.clone()); + self.block_replacement_recovery_set(&state.set_disk_id); + } + } + } + } + Ok(_) => { + if manager.abandon_replacement_intent().await.is_ok() { + replacement_restarts + .entry(task_id) + .or_insert((state.set_disk_id, state.replacement_targets)); + } + } + Err(_) => {} + } + } + } + } + } + + if !unclean && replacement_intents.is_empty() && replacement_restarts.is_empty() { + return; + } + + let mut recovery_by_set = HashMap::, Vec, Vec, Option)>>::new(); + for (task_id, (set_disk_id, heal_endpoints, buckets, resume_endpoint)) in replacement_intents { + recovery_by_set + .entry(set_disk_id) + .or_default() + .push((Some(task_id), heal_endpoints, buckets, Some(resume_endpoint))); + } + for (_abandoned_task_id, (set_disk_id, heal_endpoints)) in replacement_restarts { + recovery_by_set + .entry(set_disk_id) + .or_default() + .push((None, heal_endpoints, Vec::new(), None)); + } + + for (set_disk_id, mut recoveries) in recovery_by_set { + let Ok((pool_index, set_index)) = crate::heal::utils::parse_set_disk_id(&set_disk_id) else { + continue; + }; + if self.replacement_recovery_set_is_blocked(&set_disk_id) { + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_disk_id, + recovery_count = recoveries.len(), + "Replacement recovery deferred because durable recovery validation is blocked" + ); + continue; + } + if conflicted_replacement_sets.contains(&set_disk_id) || recoveries.len() != 1 { + self.block_replacement_recovery_set(&set_disk_id); + debug!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_disk_id, + recovery_count = recoveries.len(), + "Replacement recovery deferred because multiple durable generations exist" + ); + continue; + } + let reuse_single_generation = recoveries.len() == 1 && recoveries[0].0.is_some(); + let mut heal_endpoints = recoveries + .iter_mut() + .flat_map(|(_, targets, _, _)| std::mem::take(targets)) + .collect::>(); + heal_endpoints.sort_unstable(); + heal_endpoints.dedup(); + let buckets = if reuse_single_generation { + std::mem::take(&mut recoveries[0].2) + } else { + Vec::new() + }; + let mut req = HealRequest::new( + HealType::ErasureSet { + buckets, + set_disk_id: set_disk_id.clone(), + }, + HealOptions { + pool_index: Some(pool_index), + set_index: Some(set_index), + timeout: None, + ..HealOptions::default() + }, + HealPriority::Low, + ); + if reuse_single_generation && let Some(task_id) = recoveries[0].0.take() { + req.id = task_id; + } + let recovery_anchor = reuse_single_generation.then(|| recoveries[0].3.take()).flatten(); + req.source = HealRequestSource::AutoHeal; + req.heal_endpoints = heal_endpoints; + let request_id = req.id.clone(); + if let Some(anchor) = &recovery_anchor { + self.replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(request_id.clone(), anchor.clone()); + } + match self.submit_heal_request(req).await { + Ok(HealAdmissionResult::Accepted) => {} + Ok(_) => { + self.replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&request_id); + } + Err(err) => { + self.replacement_recovery_anchors + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&request_id); + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_disk_id, + error = %err, + "Replacement recovery enqueue failed" + ); + } + } + } + + if !unclean || set_disk_ids.is_empty() { + return; + } + + info!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_count = set_disk_ids.len(), + "Unclean shutdown detected; scheduling erasure-set heal for local sets" + ); + + let buckets = match self.storage.list_buckets().await { + Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::>(), + Err(err) => { + error!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + error = %err, + "Unclean-shutdown heal skipped: bucket listing failed" + ); + return; + } + }; + + for set_disk_id in set_disk_ids { + let mut req = HealRequest::new( + HealType::ErasureSet { + buckets: buckets.clone(), + set_disk_id: set_disk_id.clone(), + }, + HealOptions { + timeout: None, + ..HealOptions::default() + }, + HealPriority::Low, + ); + req.source = HealRequestSource::AutoHeal; + if let Err(err) = self.submit_heal_request(req).await { + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_UNCLEAN_SHUTDOWN, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + set_disk_id, + error = %err, + "Unclean-shutdown heal enqueue failed" + ); + } + } + } +} diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index 9e4b59f7a..02efba891 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -1017,9 +1017,11 @@ if rg -n -F 'target: "rustfs::server::http"' rustfs/src/server/layer.rs >/dev/nu exit 1 fi -demoted_admission_sites="$(rg -c -F 'demote_to_debug_when!(' crates/heal/src/heal/manager.rs || echo 0)" +# manager.rs and its manager/ child modules are one logical module tree since +# the queue/scheduler split; count the demoted sites across the whole tree. +demoted_admission_sites="$(cat crates/heal/src/heal/manager.rs crates/heal/src/heal/manager/*.rs 2>/dev/null | rg -c -F 'demote_to_debug_when!(' || echo 0)" if [[ "$demoted_admission_sites" -lt 6 ]]; then - echo "❌ logging guardrail violation: heal queue admission/scheduler warns for per-object requests must stay level-split via demote_to_debug_when! (expected >= 6 sites in crates/heal/src/heal/manager.rs, found $demoted_admission_sites)" >&2 + echo "❌ logging guardrail violation: heal queue admission/scheduler warns for per-object requests must stay level-split via demote_to_debug_when! (expected >= 6 total sites in the manager module tree, found $demoted_admission_sites)" >&2 exit 1 fi