// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use crate::heal::{
    progress::{HealProgress, HealStatistics},
    resume::{ReplacementPhase, ResumeManager, ResumeState, ResumeUtils},
    storage::HealStorageAPI,
    task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when},
};
use crate::{Error, Result};
use metrics::{counter, gauge};
use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionReceipt, HealAdmissionResult, HealRequestSource};
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass};
use rustfs_madmin::heal_commands::HealResultItem;
#[cfg(test)]
use std::sync::LazyLock;
use std::{
    collections::{BinaryHeap, HashMap, HashSet},
    sync::Arc,
    time::{Duration, SystemTime},
};
use tokio::{
    sync::{Mutex, Notify, RwLock},
    time::{interval, sleep},
};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};

use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};

const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
const LOG_SUBSYSTEM_MANAGER: &str = "manager";
const EVENT_HEAL_AUTO_SCAN_STATE: &str = "heal_auto_scan_state";
const EVENT_HEAL_AUTO_SCAN_DISK: &str = "heal_auto_scan_disk";
const EVENT_HEAL_AUTO_SCAN_ENQUEUE: &str = "heal_auto_scan_enqueue";
const EVENT_HEAL_MANAGER_STATE: &str = "heal_manager_state";
const EVENT_HEAL_QUEUE_ADMISSION: &str = "heal_queue_admission";
const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle";
const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state";
const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state";
const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
const LEGACY_ROOT_HEAL_PATH: &str = ".";
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);

fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bool {
    state.replacement_generation.as_deref() == Some(task_id)
        && !state.replacement_targets.is_empty()
        && ((!state.completed
            && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding)
            && state.retry_count >= state.max_retries)
            || (state.completed
                && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)))
}

fn replacement_discovery_error_is_expected_for_deferred_endpoint(
    error: &Error,
    endpoint: &str,
    deferred_replacement_endpoints: &HashSet<String>,
) -> bool {
    matches!(error, Error::Disk(DiskError::UnformattedDisk)) && deferred_replacement_endpoints.contains(endpoint)
}

fn unblock_replacement_recovery_sets_after_validation(
    blocked_sets: &mut HashSet<String>,
    retry_succeeded: HashSet<String>,
    retry_failed: &HashSet<String>,
) {
    for set_disk_id in retry_succeeded {
        if !retry_failed.contains(&set_disk_id) {
            blocked_sets.remove(&set_disk_id);
        }
    }
}

// Admission/scheduler outcomes for per-object requests (Object/Metadata/MRF/
// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
// recovery loops submit those per object, so a full queue or a retry storm
// would otherwise emit one warn! per object (rustfs/rustfs#5716). The
// `rustfs_heal_admission_total` metric and the `heal_queue_state` backlog
// event keep the aggregate signal at operator-visible levels.

#[cfg(test)]
struct RetryOwnershipTestHook {
    task_id: String,
    active_to_retrying_reached: Notify,
    active_to_retrying_release: Notify,
    retrying_to_queue_reached: Notify,
    retrying_to_queue_release: Notify,
}

#[cfg(test)]
static RETRY_OWNERSHIP_TEST_HOOK: LazyLock<Mutex<Option<Arc<RetryOwnershipTestHook>>>> = LazyLock::new(|| Mutex::new(None));

#[cfg(test)]
struct DuplicateAdmissionTestHook {
    request_id: String,
    active_lock_reached: Notify,
    active_lock_release: Notify,
}

#[cfg(test)]
static DUPLICATE_ADMISSION_TEST_HOOK: LazyLock<Mutex<Option<Arc<DuplicateAdmissionTestHook>>>> =
    LazyLock::new(|| Mutex::new(None));

#[cfg(test)]
async fn pause_retry_ownership_transition(task_id: &str, to_queue: bool) {
    let hook = RETRY_OWNERSHIP_TEST_HOOK.lock().await.clone();
    let Some(hook) = hook.filter(|hook| hook.task_id == task_id) else {
        return;
    };
    if to_queue {
        hook.retrying_to_queue_reached.notify_one();
        hook.retrying_to_queue_release.notified().await;
    } else {
        hook.active_to_retrying_reached.notify_one();
        hook.active_to_retrying_release.notified().await;
    }
}

#[cfg(test)]
async fn pause_duplicate_admission_after_active_lock(request_id: &str) {
    let hook = DUPLICATE_ADMISSION_TEST_HOOK.lock().await.clone();
    let Some(hook) = hook.filter(|hook| hook.request_id == request_id) else {
        return;
    };
    hook.active_lock_reached.notify_one();
    hook.active_lock_release.notified().await;
}

type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;

/// Priority queue wrapper for heal requests
/// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items
#[derive(Debug)]
struct PriorityHealQueue {
    /// Heap of (priority, sequence, request) tuples
    heap: BinaryHeap<PriorityQueueItem>,
    /// Sequence counter for FIFO ordering within same priority
    sequence: u64,
    /// Deduplication key reference counts for queued requests
    dedup_keys: HashMap<String, usize>,
}

/// Wrapper for heap items to implement proper ordering
#[derive(Debug)]
struct PriorityQueueItem {
    priority: HealPriority,
    sequence: u64,
    request: HealRequest,
}

impl Eq for PriorityQueueItem {}

impl PartialEq for PriorityQueueItem {
    fn eq(&self, other: &Self) -> bool {
        self.priority == other.priority && self.sequence == other.sequence
    }
}

impl Ord for PriorityQueueItem {
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        // First compare by priority (higher priority first)
        match self.priority.cmp(&other.priority) {
            std::cmp::Ordering::Equal => {
                // If priorities are equal, use sequence for FIFO (lower sequence first)
                other.sequence.cmp(&self.sequence)
            }
            ordering => ordering,
        }
    }
}

impl PartialOrd for PriorityQueueItem {
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.cmp(other))
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum QueuePushOutcome {
    Accepted,
    Merged,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ForegroundPressure {
    class: WorkloadClass,
    usage_pct: usize,
    threshold_pct: usize,
}

impl ForegroundPressure {
    const fn reason(self) -> &'static str {
        match self.class {
            WorkloadClass::ForegroundRead => "foreground_read_pressure",
            WorkloadClass::ForegroundWrite => "foreground_write_pressure",
            _ => "foreground_pressure",
        }
    }
}

#[derive(Debug, Clone)]
struct CompletedHealStatus {
    heal_type: HealType,
    status: HealTaskStatus,
    result_items: Vec<HealResultItem>,
    result_items_truncated: bool,
    completed_at: SystemTime,
    /// Sequence-stamped retained window, archived with the completion so
    /// incremental consumers keep their cursor across the transition (HS-06).
    seqed_items: Vec<(u64, HealResultItem)>,
    next_seq: u64,
    min_seq: u64,
}

#[derive(Debug, Clone)]
struct HealTaskAlias {
    task_id: String,
}

#[derive(Debug, Clone)]
struct RetryingHeal {
    request: HealRequest,
    error: String,
    cancel_token: CancellationToken,
}

#[derive(Debug, Clone)]
pub struct HealTaskReport {
    pub status: HealTaskStatus,
    pub result_items: Vec<HealResultItem>,
    pub result_items_truncated: bool,
    pub progress: Option<HealProgress>,
    /// Cursor for incremental consumption: sequence number of the next item
    /// to be produced. `0` on reports from sources without sequencing.
    pub next_seq: u64,
    /// Oldest sequence still retained (`0` together with `next_seq` when
    /// sequencing is unavailable).
    pub min_seq: u64,
}

/// Report from a live task, honoring the client's incremental cursor.
async fn active_task_report(task: &HealTask, since: Option<u64>) -> HealTaskReport {
    let window = task.get_result_items_since(since).await;
    HealTaskReport {
        status: task.get_status().await,
        result_items: window.items,
        // The legacy flag stays set once anything was evicted; a lagging
        // incremental cursor additionally marks this response truncated so
        // the client knows to restart from `min_seq`.
        result_items_truncated: task.result_items_truncated() || window.lagged,
        progress: Some(task.get_progress().await),
        next_seq: window.next_seq,
        min_seq: window.min_seq,
    }
}

fn empty_task_report(status: HealTaskStatus) -> HealTaskReport {
    HealTaskReport {
        status,
        result_items: Vec::new(),
        result_items_truncated: false,
        progress: None,
        next_seq: 0,
        min_seq: 0,
    }
}

fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) -> HealTaskReport {
    let mut lagged = false;
    let result_items = match since {
        None => completed.result_items.clone(),
        Some(cursor) => {
            if cursor + 1 < completed.min_seq {
                lagged = true;
            }
            completed
                .seqed_items
                .iter()
                .filter(|(seq, _)| *seq > cursor)
                .map(|(_, item)| item.clone())
                .collect()
        }
    };
    HealTaskReport {
        status: completed.status.clone(),
        result_items,
        result_items_truncated: completed.result_items_truncated || lagged,
        progress: None,
        next_seq: completed.next_seq,
        min_seq: completed.min_seq,
    }
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct HealPriorityCounts {
    pub low: u64,
    pub normal: u64,
    pub high: u64,
    pub urgent: u64,
}

impl HealPriorityCounts {
    fn increment(&mut self, priority: HealPriority) {
        match priority {
            HealPriority::Low => self.low += 1,
            HealPriority::Normal => self.normal += 1,
            HealPriority::High => self.high += 1,
            HealPriority::Urgent => self.urgent += 1,
        }
    }
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct HealSourceCounts {
    pub scanner: u64,
    pub admin: u64,
    pub auto_heal: u64,
    pub internal: u64,
    pub read_repair: u64,
    #[serde(default)]
    pub mrf: u64,
}

impl HealSourceCounts {
    fn increment(&mut self, source: HealRequestSource) {
        match source {
            HealRequestSource::Scanner => self.scanner += 1,
            HealRequestSource::Admin => self.admin += 1,
            HealRequestSource::AutoHeal => self.auto_heal += 1,
            HealRequestSource::Internal => self.internal += 1,
            HealRequestSource::ReadRepair => self.read_repair += 1,
            HealRequestSource::Mrf => self.mrf += 1,
        }
    }
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct HealOperationsSnapshot {
    pub queue_length: u64,
    pub active_tasks: u64,
    pub retrying_tasks: u64,
    pub queued_by_priority: HealPriorityCounts,
    pub active_by_priority: HealPriorityCounts,
    pub retrying_by_priority: HealPriorityCounts,
    pub queued_by_source: HealSourceCounts,
    pub active_by_source: HealSourceCounts,
    pub retrying_by_source: HealSourceCounts,
}

fn usize_to_u64_saturated(value: usize) -> u64 {
    u64::try_from(value).unwrap_or(u64::MAX)
}

impl PriorityHealQueue {
    fn new() -> Self {
        Self {
            heap: BinaryHeap::new(),
            sequence: 0,
            dedup_keys: HashMap::new(),
        }
    }

    fn len(&self) -> usize {
        self.heap.len()
    }

    fn pop_next(&mut self) -> Option<HealRequest> {
        self.heap.pop().map(|item| {
            let key = Self::make_dedup_key(&item.request);
            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
            item.request
        })
    }

    fn is_empty(&self) -> bool {
        self.heap.is_empty()
    }

    fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
        let key = Self::make_dedup_key(&request);

        // Check for duplicates unless the caller explicitly forces admission.
        if self.dedup_keys.contains_key(&key) && !request.force_start {
            return QueuePushOutcome::Merged;
        }
        // Track dedup keys for both normal and forced requests so queued forced work
        // also reserves the dedup key for later non-forced duplicates.
        *self.dedup_keys.entry(key).or_insert(0) += 1;
        self.sequence += 1;
        self.heap.push(PriorityQueueItem {
            priority: request.priority,
            sequence: self.sequence,
            request,
        });
        QueuePushOutcome::Accepted
    }

    fn can_displace_lower_priority(&self, priority: HealPriority) -> bool {
        self.heap.iter().any(|item| item.priority < priority)
    }

    fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option<HealRequest> {
        let mut retained = BinaryHeap::new();
        let mut displaced: Option<PriorityQueueItem> = None;

        while let Some(item) = self.heap.pop() {
            if item.priority < request.priority {
                let should_displace = displaced
                    .as_ref()
                    .map(|current| {
                        item.priority < current.priority
                            || (item.priority == current.priority && item.sequence > current.sequence)
                    })
                    .unwrap_or(true);
                if should_displace {
                    if let Some(current) = displaced.replace(item) {
                        retained.push(current);
                    }
                } else {
                    retained.push(item);
                }
            } else {
                retained.push(item);
            }
        }

        self.heap = retained;

        let displaced = displaced.map(|item| {
            let key = Self::make_dedup_key(&item.request);
            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
            item.request
        });

        if displaced.is_some() {
            // The enqueue side effect must run in ALL builds. Do NOT fold `self.push(request)`
            // into `debug_assert_eq!` — in release builds (`debug_assertions` off) the whole
            // macro, including its argument expression, is compiled out, which would silently
            // drop the new high-priority request after having already evicted a queued item.
            let outcome = self.push(request);
            debug_assert_eq!(outcome, QueuePushOutcome::Accepted);
        }

        displaced
    }

    /// Get statistics about queue contents by priority
    fn get_priority_stats(&self) -> HashMap<HealPriority, usize> {
        let mut stats = HashMap::new();
        for item in &self.heap {
            *stats.entry(item.priority).or_insert(0) += 1;
        }
        stats
    }

    fn operation_counts(&self) -> (HealPriorityCounts, HealSourceCounts) {
        let mut priority = HealPriorityCounts::default();
        let mut source = HealSourceCounts::default();
        for item in &self.heap {
            priority.increment(item.request.priority);
            source.increment(item.request.source);
        }
        (priority, source)
    }

    #[cfg(test)]
    fn pop(&mut self) -> Option<HealRequest> {
        self.heap.pop().map(|item| {
            let key = Self::make_dedup_key(&item.request);
            Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
            item.request
        })
    }

    #[cfg(test)]
    fn pop_runnable<F>(&mut self, can_run: F) -> Option<HealRequest>
    where
        F: Fn(&HealRequest) -> bool,
    {
        self.pop_runnable_with_skips(can_run, |_| None).0
    }

    fn pop_runnable_with_skips<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
    where
        F: Fn(&HealRequest) -> bool,
        G: Fn(&HealRequest) -> Option<String>,
    {
        let mut deferred = Vec::new();
        let mut selected = None;
        let mut skipped = Vec::new();

        while let Some(item) = self.heap.pop() {
            if can_run(&item.request) {
                selected = Some(item);
                break;
            }
            if let Some(label) = skip_label(&item.request) {
                skipped.push(label);
            }
            deferred.push(item);
        }

        for item in deferred {
            self.heap.push(item);
        }

        (
            selected.map(|item| {
                let key = Self::make_dedup_key(&item.request);
                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
                item.request
            }),
            skipped,
        )
    }

    /// Create a deduplication key from a heal request
    fn make_dedup_key(request: &HealRequest) -> String {
        Self::make_dedup_key_for_type(&request.heal_type)
    }

    fn make_dedup_key_for_type(heal_type: &HealType) -> String {
        match heal_type {
            HealType::Cluster => "cluster".to_string(),
            HealType::Object {
                bucket,
                object,
                version_id,
            } => {
                format!("object:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
            }
            HealType::Bucket { bucket } => {
                format!("bucket:{bucket}")
            }
            HealType::Prefix { bucket, prefix } => {
                format!("prefix:{bucket}/{prefix}")
            }
            HealType::ErasureSet { set_disk_id, .. } => {
                format!("erasure_set:{set_disk_id}")
            }
            HealType::Metadata { bucket, object } => {
                format!("metadata:{bucket}:{object}")
            }
            HealType::MRF { meta_path } => {
                format!("mrf:{meta_path}")
            }
            HealType::ECDecode {
                bucket,
                object,
                version_id,
            } => {
                format!("ecdecode:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
            }
        }
    }

    fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap<String, usize>, key: &str) {
        if let Some(count) = dedup_keys.get_mut(key) {
            if *count <= 1 {
                dedup_keys.remove(key);
            } else {
                *count -= 1;
            }
        }
    }

    /// Check if a request with the same key already exists in the queue
    #[allow(dead_code)]
    fn contains_key(&self, request: &HealRequest) -> bool {
        let key = Self::make_dedup_key(request);
        self.dedup_keys.contains_key(&key)
    }

    /// Check if an erasure set heal request for a specific set_disk_id exists
    fn contains_erasure_set(&self, set_disk_id: &str) -> bool {
        let key = format!("erasure_set:{set_disk_id}");
        self.dedup_keys.contains_key(&key)
    }

    /// Iterate queued requests (used by the admin overlap check).
    fn requests(&self) -> impl Iterator<Item = &HealRequest> {
        self.heap.iter().map(|item| &item.request)
    }

    fn contains_request_id(&self, request_id: &str) -> bool {
        self.heap.iter().any(|item| item.request.id == request_id)
    }

    fn contains_request_id_matching_path(&self, request_id: &str, heal_path: &str) -> bool {
        self.heap
            .iter()
            .any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path))
    }

    fn request_for_dedup_key(&self, key: &str) -> Option<&HealRequest> {
        self.heap
            .iter()
            .find_map(|item| (Self::make_dedup_key(&item.request) == key).then_some(&item.request))
    }

    fn contains_matching<F>(&self, mut matches: F) -> bool
    where
        F: FnMut(&HealRequest) -> bool,
    {
        self.heap.iter().any(|item| matches(&item.request))
    }

    fn remove_request_id(&mut self, request_id: &str) -> Option<HealRequest> {
        let mut retained = BinaryHeap::new();
        let mut removed = None;

        while let Some(item) = self.heap.pop() {
            if removed.is_none() && item.request.id == request_id {
                let key = Self::make_dedup_key(&item.request);
                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
                removed = Some(item.request);
            } else {
                retained.push(item);
            }
        }

        self.heap = retained;
        removed
    }

    fn remove_matching<F>(&mut self, mut should_remove: F) -> Vec<HealRequest>
    where
        F: FnMut(&HealRequest) -> bool,
    {
        let mut retained = BinaryHeap::new();
        let mut removed = Vec::new();

        while let Some(item) = self.heap.pop() {
            if should_remove(&item.request) {
                let key = Self::make_dedup_key(&item.request);
                Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
                removed.push(item.request);
            } else {
                retained.push(item);
            }
        }

        self.heap = retained;
        removed
    }
}

impl RetryingHeal {
    fn status(&self) -> HealTaskStatus {
        HealTaskStatus::Retrying {
            error: self.error.clone(),
            retry_attempt: self.request.retry_attempts,
        }
    }
}

fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool {
    let heal_path = heal_path.trim_matches('/');
    if heal_path.is_empty() || heal_path == LEGACY_ROOT_HEAL_PATH {
        return matches!(heal_type, HealType::Cluster);
    }

    match heal_type {
        HealType::Cluster => false,
        HealType::Object { bucket, object, .. }
        | HealType::Metadata { bucket, object }
        | HealType::ECDecode { bucket, object, .. } => heal_path_matches_bucket_child(heal_path, bucket, object),
        HealType::Bucket { bucket } => heal_path == bucket,
        HealType::Prefix { bucket, prefix } => heal_path_matches_bucket_child(heal_path, bucket, prefix),
        HealType::ErasureSet { set_disk_id, .. } => heal_path == set_disk_id,
        HealType::MRF { meta_path } => heal_path == meta_path.trim_matches('/'),
    }
}

fn heal_path_matches_bucket_child(heal_path: &str, bucket: &str, child: &str) -> bool {
    heal_path == bucket || heal_path == format!("{bucket}/{child}").trim_matches('/')
}

fn publish_active_heal_count(active_heals: &HashMap<String, Arc<HealTask>>) {
    crate::set_heal_active_tasks(active_heals.len());
}

fn publish_heal_queue_length(queue: &PriorityHealQueue) {
    crate::set_heal_queue_length(queue.len());
}

fn active_heals_contains_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> bool {
    active_heal_for_dedup_key(active_heals, key).is_some()
}

fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
    active_heals
        .iter()
        .find(|(_, task)| PriorityHealQueue::make_dedup_key_for_type(&task.heal_type) == key)
        .map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
}

fn retrying_heal_for_dedup_key(retrying_heals: &HashMap<String, RetryingHeal>, key: &str) -> Option<(String, HealType)> {
    retrying_heals
        .iter()
        .find(|(_, retrying)| PriorityHealQueue::make_dedup_key(&retrying.request) == key)
        .map(|(task_id, retrying)| (task_id.clone(), retrying.request.heal_type.clone()))
}

fn completed_status_is_retrying(status: &HealTaskStatus) -> bool {
    matches!(status, HealTaskStatus::Retrying { .. })
}

fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
    let Err(err) = result else {
        return None;
    };
    if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
        return None;
    }
    if task.has_batch_failure() {
        return None;
    }

    let error = err.to_string();
    if !err.is_recoverable_heal() {
        return None;
    }

    let request = task.retry_request();
    let delay = recoverable_heal_retry_delay(request.retry_attempts);
    Some((request, delay, error))
}

async fn retry_request_for_result_with_budget(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
    let (_, delay, error) = retry_request_for_result(task, result)?;
    let request = task.retry_request_with_remaining_timeout().await.ok()?;
    Some((request, delay, error))
}

fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration {
    let retry_attempt = retry_attempt.clamp(1, 5);
    let delay = Duration::from_secs(2_u64.saturating_pow(retry_attempt));
    delay.min(MAX_RECOVERABLE_HEAL_RETRY_DELAY)
}

/// Heal config
/// HS-06 admin overlap policy.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum HealOverlapPolicy {
    /// Default: overlapping admin starts merge into the existing task
    /// (today's dedup semantics).
    #[default]
    Merge,
    /// Return a typed already-running / overlapping-paths rejection like
    /// madmin's ErrHealAlreadyRunning / ErrHealOverlappingPaths.
    MinioError,
}

/// Path view of a heal type for overlap comparison: a bucket plus a
/// prefix/object path inside it (`None` bucket = cluster-wide, overlaps
/// everything).
fn heal_type_path_view(heal_type: &HealType) -> (Option<&str>, &str) {
    match heal_type {
        HealType::Cluster => (None, ""),
        HealType::Bucket { bucket } => (Some(bucket), ""),
        HealType::Prefix { bucket, prefix } => (Some(bucket), prefix),
        HealType::Object { bucket, object, .. }
        | HealType::Metadata { bucket, object }
        | HealType::ECDecode { bucket, object, .. } => (Some(bucket), object),
        // MRF/MetaPath heal keys on a meta path; treat the whole set of
        // buckets as one namespace so it only overlaps itself exactly.
        HealType::MRF { meta_path } => (Some("\u{0}mrf"), meta_path),
        // Erasure-set heal: the set id is the overlap dimension.
        HealType::ErasureSet { set_disk_id, .. } => (Some("\u{0}set"), set_disk_id),
    }
}

/// How two heal paths relate for the admin overlap check (HS-06).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum OverlapVerdict {
    /// Distinct targets: no conflict.
    Disjoint,
    /// Same target: an identical heal is already in flight.
    SameTarget,
    /// One target contains the other.
    Overlapping,
}

fn prefix_paths_overlap(a: &str, b: &str) -> OverlapVerdict {
    if a == b {
        return OverlapVerdict::SameTarget;
    }
    if a.is_empty() || b.is_empty() || a.starts_with(b) || b.starts_with(a) {
        return OverlapVerdict::Overlapping;
    }
    OverlapVerdict::Disjoint
}

fn heal_types_overlap(left: &HealType, right: &HealType) -> OverlapVerdict {
    let (left_bucket, left_path) = heal_type_path_view(left);
    let (right_bucket, right_path) = heal_type_path_view(right);
    match (left_bucket, right_bucket) {
        // Cluster-wide overlaps everything (but an exact cluster match is
        // SameTarget).
        (None, _) | (_, None) => {
            if matches!(left, HealType::Cluster) && matches!(right, HealType::Cluster) {
                OverlapVerdict::SameTarget
            } else {
                OverlapVerdict::Overlapping
            }
        }
        (Some(lb), Some(rb)) => {
            if lb != rb {
                return OverlapVerdict::Disjoint;
            }
            prefix_paths_overlap(left_path, right_path)
        }
    }
}

#[derive(Debug, Clone)]
pub struct HealConfig {
    /// Whether to enable auto heal
    pub enable_auto_heal: bool,
    /// Heal interval
    pub heal_interval: Duration,
    /// Maximum concurrent heal tasks
    pub max_concurrent_heals: usize,
    /// Maximum concurrent heal tasks allowed for a single erasure set
    pub max_concurrent_per_set: usize,
    /// Aggregate task execution timeout across recoverable retries
    pub task_timeout: Duration,
    /// Queue size
    pub queue_size: usize,
    /// Whether duplicate low-priority requests should merge into an existing queued request.
    pub low_priority_merge_enable: bool,
    /// Whether low-priority requests may be dropped when the queue is full.
    pub low_priority_drop_when_full: bool,
    /// Whether notify-driven scheduler wakeups are enabled.
    pub event_driven_scheduler_enable: bool,
    /// How admin heal starts behave on path overlap (HS-06): merge into the
    /// existing task (default) or return a typed already-running rejection.
    pub overlap_policy: HealOverlapPolicy,
    /// Whether per-set bulkhead scheduling is enabled.
    pub set_bulkhead_enable: bool,
    /// Whether erasure-set page parallelism is enabled.
    pub page_parallel_enable: bool,
    /// Whether foreground read pressure can delay best-effort heal task starts.
    pub mainline_throttle_enable: bool,
    /// Foreground read permit utilization percentage that delays best-effort heal starts.
    pub mainline_read_utilization_high_percent: usize,
    /// Foreground write utilization percentage that delays best-effort heal starts.
    pub mainline_write_utilization_high_percent: usize,
    /// Delay before rechecking foreground pressure after delaying heal starts.
    pub mainline_max_sleep: Duration,
}

impl Default for HealConfig {
    fn default() -> Self {
        let queue_size: usize =
            rustfs_utils::get_env_usize(rustfs_config::ENV_HEAL_QUEUE_SIZE, rustfs_config::DEFAULT_HEAL_QUEUE_SIZE);
        let heal_interval = Duration::from_secs(rustfs_utils::get_env_u64(
            rustfs_config::ENV_HEAL_INTERVAL_SECS,
            rustfs_config::DEFAULT_HEAL_INTERVAL_SECS,
        ));
        let enable_auto_heal =
            rustfs_utils::get_env_bool(rustfs_config::ENV_HEAL_AUTO_HEAL_ENABLE, rustfs_config::DEFAULT_HEAL_AUTO_HEAL_ENABLE);
        let task_timeout = Duration::from_secs(rustfs_utils::get_env_u64(
            rustfs_config::ENV_HEAL_TASK_TIMEOUT_SECS,
            rustfs_config::DEFAULT_HEAL_TASK_TIMEOUT_SECS,
        ));
        let max_concurrent_heals = rustfs_utils::get_env_usize(
            rustfs_config::ENV_HEAL_MAX_CONCURRENT_HEALS,
            rustfs_config::DEFAULT_HEAL_MAX_CONCURRENT_HEALS,
        );
        let max_concurrent_per_set = rustfs_utils::get_env_usize(
            rustfs_config::ENV_HEAL_MAX_CONCURRENT_PER_SET,
            rustfs_config::DEFAULT_HEAL_MAX_CONCURRENT_PER_SET,
        );
        let low_priority_merge_enable = rustfs_utils::get_env_bool(
            rustfs_config::ENV_HEAL_LOW_PRIORITY_MERGE_ENABLE,
            rustfs_config::DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE,
        );
        let low_priority_drop_when_full = rustfs_utils::get_env_bool(
            rustfs_config::ENV_HEAL_LOW_PRIORITY_DROP_WHEN_FULL,
            rustfs_config::DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL,
        );
        let event_driven_scheduler_enable = rustfs_utils::get_env_bool(
            rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
            rustfs_config::DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE,
        );
        let overlap_policy =
            match rustfs_utils::get_env_str(rustfs_config::ENV_HEAL_OVERLAP_POLICY, rustfs_config::DEFAULT_HEAL_OVERLAP_POLICY)
                .to_lowercase()
                .as_str()
            {
                "minio_error" => HealOverlapPolicy::MinioError,
                _ => HealOverlapPolicy::Merge,
            };
        let set_bulkhead_enable = rustfs_utils::get_env_bool(
            rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE,
            rustfs_config::DEFAULT_HEAL_SET_BULKHEAD_ENABLE,
        );
        let page_parallel_enable = rustfs_utils::get_env_bool(
            rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE,
            rustfs_config::DEFAULT_HEAL_PAGE_PARALLEL_ENABLE,
        );
        let mainline_throttle_enable = rustfs_utils::get_env_bool(
            rustfs_config::ENV_HEAL_MAINLINE_THROTTLE_ENABLE,
            rustfs_config::DEFAULT_HEAL_MAINLINE_THROTTLE_ENABLE,
        );
        let mainline_read_utilization_high_percent = rustfs_utils::get_env_usize(
            rustfs_config::ENV_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT,
            rustfs_config::DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT,
        )
        .min(100);
        let mainline_write_utilization_high_percent = rustfs_utils::get_env_usize(
            rustfs_config::ENV_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT,
            rustfs_config::DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT,
        )
        .min(100);
        let mainline_max_sleep = Duration::from_millis(rustfs_utils::get_env_u64(
            rustfs_config::ENV_HEAL_MAINLINE_MAX_SLEEP_MS,
            rustfs_config::DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS,
        ));
        Self {
            enable_auto_heal,
            heal_interval,        // 10 seconds
            max_concurrent_heals, // max 4,
            max_concurrent_per_set: std::cmp::min(max_concurrent_heals.max(1), max_concurrent_per_set.max(1)),
            task_timeout, // 5 minutes
            queue_size,
            low_priority_merge_enable,
            low_priority_drop_when_full,
            event_driven_scheduler_enable,
            overlap_policy,
            set_bulkhead_enable,
            page_parallel_enable,
            mainline_throttle_enable,
            mainline_read_utilization_high_percent,
            mainline_write_utilization_high_percent,
            mainline_max_sleep,
        }
    }
}

/// Heal state
#[derive(Debug, Default)]
pub struct HealState {
    /// Whether running
    pub is_running: bool,
    /// Current heal cycle
    pub current_cycle: u64,
    /// Last heal time
    pub last_heal_time: Option<SystemTime>,
    /// Total healed objects
    pub total_healed_objects: u64,
    /// Total heal failures
    pub total_heal_failures: u64,
    /// Current active heal tasks
    pub active_heal_count: usize,
}

/// Heal manager
pub struct HealManager {
    /// Heal config
    config: Arc<RwLock<HealConfig>>,
    /// Heal state
    state: Arc<RwLock<HealState>>,
    /// Active heal tasks
    active_heals: Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
    /// Heal queue (priority-based)
    heal_queue: Arc<Mutex<PriorityHealQueue>>,
    /// Recently completed heal statuses retained for status queries.
    completed_heals: Arc<Mutex<HashMap<String, CompletedHealStatus>>>,
    /// Client tokens merged into an existing task id.
    task_aliases: Arc<Mutex<HashMap<String, HealTaskAlias>>>,
    /// Heal tasks waiting for a retry backoff to expire.
    retrying_heals: Arc<Mutex<HashMap<String, RetryingHeal>>>,
    /// Surviving disks that hold durable replacement intents, keyed by task ID.
    /// This is rebuilt from durable state after restart and never crosses the
    /// public request boundary.
    replacement_recovery_anchors: Arc<std::sync::Mutex<HashMap<String, String>>>,
    /// Set IDs whose durable replacement metadata is corrupt or conflicting.
    replacement_recovery_blocked_sets: Arc<std::sync::Mutex<HashSet<String>>>,
    /// Storage layer interface
    storage: Arc<dyn HealStorageAPI>,
    /// Cancel token
    cancel_token: CancellationToken,
    /// Statistics
    statistics: Arc<RwLock<HealStatistics>>,
    /// Scheduler wake-up notifier for event-driven dispatch
    notify: Arc<Notify>,
    /// Optional runtime workload snapshot provider used to protect foreground data-plane work.
    workload_provider: Option<WorkloadSnapshotProviderRef>,
}

struct HealQueueContext<'a> {
    heal_queue: &'a Arc<Mutex<PriorityHealQueue>>,
    active_heals: &'a Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
    completed_heals: &'a Arc<Mutex<HashMap<String, CompletedHealStatus>>>,
    retrying_heals: &'a Arc<Mutex<HashMap<String, RetryingHeal>>>,
    replacement_recovery_anchors: &'a Arc<std::sync::Mutex<HashMap<String, String>>>,
    config: &'a Arc<RwLock<HealConfig>>,
    statistics: &'a Arc<RwLock<HealStatistics>>,
    storage: &'a Arc<dyn HealStorageAPI>,
    notify: &'a Arc<Notify>,
    cancel_token: &'a CancellationToken,
    workload_provider: &'a Option<WorkloadSnapshotProviderRef>,
}

impl HealManager {
    fn classify_full_admission(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
        let best_effort_source = matches!(
            request.source,
            HealRequestSource::Scanner | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
        );
        if best_effort_source || (request.priority == HealPriority::Low && config.low_priority_drop_when_full) {
            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
        } else {
            HealAdmissionResult::Full
        }
    }

    fn queue_usage_pct(queue_len: usize, queue_capacity: usize) -> usize {
        queue_len.saturating_mul(100).checked_div(queue_capacity).unwrap_or(0)
    }

    fn classify_pressure_admission(
        request: &HealRequest,
        queue_len: usize,
        queue_capacity: usize,
    ) -> Option<HealAdmissionResult> {
        if request.force_start || queue_capacity == 0 {
            return None;
        }

        let queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity);
        if queue_usage_pct < 80 {
            return None;
        }

        match request.source {
            HealRequestSource::ReadRepair => Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)),
            HealRequestSource::Scanner if request.priority == HealPriority::Low => {
                Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
            }
            HealRequestSource::AutoHeal if queue_usage_pct >= 95 => {
                Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
            }
            _ => None,
        }
    }

    fn duplicate_admission_for_request(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
        if request.priority == HealPriority::Low && !config.low_priority_merge_enable {
            HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
        } else {
            HealAdmissionResult::Merged
        }
    }

    fn can_displace_queued_work(request: &HealRequest) -> bool {
        matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal)
    }

    fn request_bypasses_mainline_throttle(request: &HealRequest) -> bool {
        request.force_start
            || matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal)
            || matches!(request.priority, HealPriority::High | HealPriority::Urgent)
    }

    fn mainline_throttle_active(
        config: &HealConfig,
        provider: &Option<WorkloadSnapshotProviderRef>,
    ) -> Option<ForegroundPressure> {
        if !config.mainline_throttle_enable
            || (config.mainline_read_utilization_high_percent == 0 && config.mainline_write_utilization_high_percent == 0)
        {
            return None;
        }

        let provider = provider.as_ref()?;
        let snapshot = provider.workload_admission_snapshot();
        [
            (WorkloadClass::ForegroundRead, config.mainline_read_utilization_high_percent),
            (WorkloadClass::ForegroundWrite, config.mainline_write_utilization_high_percent),
        ]
        .into_iter()
        .filter_map(|(class, threshold_pct)| {
            if threshold_pct == 0 {
                return None;
            }

            let entry = snapshot.get(class)?;
            let usage_pct = if matches!(entry.state, AdmissionState::Saturated) {
                100
            } else {
                let limit = entry.limit?;
                if limit == 0 {
                    return None;
                }
                entry
                    .active
                    .unwrap_or(0)
                    .saturating_mul(100)
                    .checked_div(limit)
                    .unwrap_or(100)
            };

            (usage_pct >= threshold_pct).then_some(ForegroundPressure {
                class,
                usage_pct,
                threshold_pct,
            })
        })
        .max_by_key(|pressure| pressure.usage_pct)
    }

    fn schedule_mainline_throttle_recheck(notify: Arc<Notify>, delay: Duration) {
        if delay.is_zero() {
            notify.notify_one();
            return;
        }

        tokio::spawn(async move {
            sleep(delay).await;
            notify.notify_one();
        });
    }

    fn record_mainline_throttle_delay(pressure: ForegroundPressure, config: &HealConfig) {
        counter!(
            "rustfs_heal_mainline_throttle_total",
            "source" => "background",
            "result" => "delayed",
            "reason" => pressure.reason()
        )
        .increment(1);
        debug!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_MAINLINE_THROTTLE,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_MANAGER,
            state = "delayed",
            reason = pressure.reason(),
            workload_class = pressure.class.as_str(),
            foreground_usage_pct = pressure.usage_pct,
            threshold_pct = pressure.threshold_pct,
            recheck_delay_ms = config.mainline_max_sleep.as_millis(),
            "Heal scheduler delayed background work under foreground pressure"
        );
    }

    fn record_admission_metric(source: HealRequestSource, admission: HealAdmissionResult, context: &'static str) {
        counter!(
            "rustfs_heal_admission_total",
            "source" => source.as_str().to_string(),
            "result" => admission.result_label().to_string(),
            "reason" => admission.reason_label().to_string(),
            "context" => context.to_string()
        )
        .increment(1);
    }

    fn admit_request_to_queue(
        queue: &mut PriorityHealQueue,
        request: HealRequest,
        config: &HealConfig,
        context: &'static str,
    ) -> HealAdmissionResult {
        let queue_len = queue.len();
        publish_heal_queue_length(queue);
        let queue_capacity = config.queue_size;
        let per_object_request = request.heal_type.is_per_object();

        if queue_len >= queue_capacity && !request.force_start {
            if Self::can_displace_queued_work(&request) && queue.can_displace_lower_priority(request.priority) {
                let request_id = request.id.clone();
                let priority = request.priority;
                let source = request.source;
                if let Some(displaced) = queue.push_displacing_lower_priority(request) {
                    publish_heal_queue_length(queue);
                    Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
                    demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request_id,
                        priority = ?priority,
                        source = source.as_str(),
                        context,
                        displaced_request_id = %displaced.id,
                        displaced_priority = ?displaced.priority,
                        queue_len,
                        queue_capacity,
                        result = "accepted_by_displacement",
                        "Heal queue request accepted by displacement"
                    });
                    return HealAdmissionResult::Accepted;
                }

                demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                    event = EVENT_HEAL_QUEUE_ADMISSION,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    request_id = %request_id,
                    priority = ?priority,
                    source = source.as_str(),
                    context,
                    queue_len,
                    queue_capacity,
                    result = "full_no_displacement_candidate",
                    "Heal queue request rejected without displacement"
                });
                Self::record_admission_metric(source, HealAdmissionResult::Full, context);
                return HealAdmissionResult::Full;
            }

            let admission = Self::classify_full_admission(&request, config);
            Self::record_admission_metric(request.source, admission, context);
            match admission {
                HealAdmissionResult::Dropped(reason) => {
                    demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request.id,
                        priority = ?request.priority,
                        source = request.source.as_str(),
                        context,
                        queue_len,
                        queue_capacity,
                        reason = reason.as_str(),
                        result = "dropped_full",
                        "Heal queue request dropped"
                    });
                }
                HealAdmissionResult::Full => {
                    demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request.id,
                        priority = ?request.priority,
                        source = request.source.as_str(),
                        context,
                        queue_len,
                        queue_capacity,
                        result = "rejected_full",
                        "Heal queue request rejected"
                    });
                }
                HealAdmissionResult::Accepted | HealAdmissionResult::Merged => {}
            }
            return admission;
        }

        if let Some(admission) = Self::classify_pressure_admission(&request, queue_len, queue_capacity) {
            Self::record_admission_metric(request.source, admission, context);
            if let HealAdmissionResult::Dropped(reason) = admission {
                debug!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_QUEUE_ADMISSION,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    request_id = %request.id,
                    priority = ?request.priority,
                    source = request.source.as_str(),
                    context,
                    queue_len,
                    queue_capacity,
                    queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity),
                    reason = reason.as_str(),
                    result = "dropped_pressure",
                    "Heal queue request dropped under pressure"
                );
            }
            return admission;
        }

        if queue_capacity > 0 {
            let capacity_threshold = queue_capacity.saturating_mul(80) / 100;
            if queue_len >= capacity_threshold {
                debug!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_QUEUE_ADMISSION,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    queue_len,
                    queue_capacity,
                    queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity),
                    context,
                    result = "queue_pressure_high",
                    "Heal queue pressure high"
                );
            }
        }

        let request_id = request.id.clone();
        let priority = request.priority;
        let source = request.source;

        match queue.push(request) {
            QueuePushOutcome::Accepted => {
                publish_heal_queue_length(queue);
                Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
                if matches!(priority, HealPriority::High | HealPriority::Urgent) {
                    let stats = queue.get_priority_stats();
                    debug!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request_id,
                        priority = ?priority,
                        source = source.as_str(),
                        context,
                        queue_len = queue_len + 1,
                        urgent = *stats.get(&HealPriority::Urgent).unwrap_or(&0),
                        high = *stats.get(&HealPriority::High).unwrap_or(&0),
                        normal = *stats.get(&HealPriority::Normal).unwrap_or(&0),
                        low = *stats.get(&HealPriority::Low).unwrap_or(&0),
                        result = "accepted",
                        "Heal queue snapshot recorded"
                    );
                }
                debug!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_QUEUE_ADMISSION,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    request_id = %request_id,
                    priority = ?priority,
                    source = source.as_str(),
                    context,
                    queue_len = queue_len + 1,
                    result = "accepted",
                    "Heal queue request accepted"
                );
                HealAdmissionResult::Accepted
            }
            QueuePushOutcome::Merged => {
                Self::record_admission_metric(source, HealAdmissionResult::Merged, context);
                debug!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_QUEUE_ADMISSION,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    request_id = %request_id,
                    priority = ?priority,
                    source = source.as_str(),
                    context,
                    queue_len,
                    result = "merged_duplicate",
                    "Heal queue request merged"
                );
                HealAdmissionResult::Merged
            }
        }
    }

    async fn insert_task_alias(&self, alias_id: &str, task_id: &str) {
        if alias_id == task_id {
            return;
        }

        self.task_aliases.lock().await.insert(
            alias_id.to_string(),
            HealTaskAlias {
                task_id: task_id.to_string(),
            },
        );
    }

    async fn canonical_task_id(&self, task_id: &str) -> String {
        self.task_aliases
            .lock()
            .await
            .get(task_id)
            .map(|alias| alias.task_id.clone())
            .unwrap_or_else(|| task_id.to_string())
    }

    async fn remove_aliases_for_task(&self, task_id: &str) {
        self.task_aliases
            .lock()
            .await
            .retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
    }

    fn block_replacement_recovery_set(&self, set_disk_id: &str) {
        self.replacement_recovery_blocked_sets
            .lock()
            .expect("replacement recovery blocked set lock poisoned")
            .insert(set_disk_id.to_string());
    }

    fn replacement_recovery_set_is_blocked(&self, set_disk_id: &str) -> bool {
        self.replacement_recovery_blocked_sets
            .lock()
            .expect("replacement recovery blocked set lock poisoned")
            .contains(set_disk_id)
    }

    async fn validate_replacement_recovery_records(disk: &crate::heal::DiskStore) -> Result<()> {
        ResumeUtils::migrate_legacy_replacement_records(disk).await?;
        for task_id in ResumeUtils::get_replacement_intent_tasks(disk).await? {
            ResumeManager::load_replacement_intent(disk.clone(), &task_id).await?;
        }
        Ok(())
    }

    /// Create new HealManager
    pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
        Self::new_with_workload_provider(storage, config, None)
    }

    /// Create new HealManager with an optional workload admission snapshot provider.
    pub fn new_with_workload_provider(
        storage: Arc<dyn HealStorageAPI>,
        config: Option<HealConfig>,
        workload_provider: Option<WorkloadSnapshotProviderRef>,
    ) -> Self {
        let config = config.unwrap_or_default();
        Self {
            config: Arc::new(RwLock::new(config)),
            state: Arc::new(RwLock::new(HealState::default())),
            active_heals: Arc::new(Mutex::new(HashMap::new())),
            heal_queue: Arc::new(Mutex::new(PriorityHealQueue::new())),
            completed_heals: Arc::new(Mutex::new(HashMap::new())),
            task_aliases: Arc::new(Mutex::new(HashMap::new())),
            retrying_heals: Arc::new(Mutex::new(HashMap::new())),
            replacement_recovery_anchors: Arc::new(std::sync::Mutex::new(HashMap::new())),
            replacement_recovery_blocked_sets: Arc::new(std::sync::Mutex::new(HashSet::new())),
            storage,
            cancel_token: CancellationToken::new(),
            statistics: Arc::new(RwLock::new(HealStatistics::new())),
            notify: Arc::new(Notify::new()),
            workload_provider,
        }
    }

    /// Start HealManager
    pub async fn start(&self) -> Result<()> {
        let mut state = self.state.write().await;
        if state.is_running {
            warn!(
                target: "rustfs::heal::manager",
                event = EVENT_HEAL_MANAGER_STATE,
                component = LOG_COMPONENT_HEAL,
                subsystem = LOG_SUBSYSTEM_MANAGER,
                state = "already_running",
                "Heal manager already running"
            );
            return Ok(());
        }
        state.is_running = true;
        drop(state);

        info!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_MANAGER_STATE,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_MANAGER,
            state = "starting",
            "Heal manager starting"
        );

        // start scheduler
        self.start_scheduler().await?;

        // Recover durable replacement intents before the scanner can admit a
        // competing task for the same set.
        self.process_unclean_shutdown().await;

        // start auto disk scanner to heal unformatted disks
        if self.config.read().await.enable_auto_heal {
            self.start_auto_disk_scanner().await?;
        } else {
            info!(
                target: "rustfs::heal::manager",
                event = EVENT_HEAL_AUTO_SCAN_STATE,
                component = LOG_COMPONENT_HEAL,
                subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                state = "disabled",
                "Heal auto disk scanner disabled"
            );
        }

        info!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_MANAGER_STATE,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_MANAGER,
            state = "running",
            "Heal manager started"
        );
        Ok(())
    }

    /// Detect whether the previous run ended without a clean shutdown and, if so,
    /// enqueue a full erasure-set heal for every local set. Also (re)writes the
    /// marker for the current run; [`super::clear_unclean_shutdown_markers`]
    /// removes it again during graceful shutdown. Best-effort: failures only log.
    async fn process_unclean_shutdown(&self) {
        let mut unclean = false;
        let mut set_disk_ids = HashSet::new();
        let mut replacement_intents = HashMap::<String, (String, Vec<String>, Vec<String>, String)>::new();
        let mut replacement_restarts = HashMap::<String, (String, Vec<String>)>::new();
        let mut conflicted_replacement_sets = HashSet::new();

        {
            let local_disks = {
                let local_disk_map = local_disk_map_read().await;
                local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
            };
            for disk in &local_disks {
                let endpoint = disk.endpoint();
                match disk
                    .read_all(super::RUSTFS_META_BUCKET, super::UNCLEAN_SHUTDOWN_MARKER_PATH)
                    .await
                {
                    Ok(_) => unclean = true,
                    Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
                    Err(err) => {
                        debug!(
                            target: "rustfs::heal::manager",
                            event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                            component = LOG_COMPONENT_HEAL,
                            subsystem = LOG_SUBSYSTEM_MANAGER,
                            endpoint = %endpoint,
                            error = ?err,
                            "Unclean-shutdown marker check failed"
                        );
                    }
                }

                let marker = SystemTime::now()
                    .duration_since(std::time::UNIX_EPOCH)
                    .map(|d| d.as_secs().to_string())
                    .unwrap_or_default();
                if let Err(err) = disk
                    .write_all(super::RUSTFS_META_BUCKET, super::UNCLEAN_SHUTDOWN_MARKER_PATH, marker.into())
                    .await
                {
                    warn!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        endpoint = %endpoint,
                        error = ?err,
                        "Unclean-shutdown marker write failed"
                    );
                }

                let disk_set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
                if let Some(set_disk_id) = &disk_set_disk_id {
                    set_disk_ids.insert(set_disk_id.clone());
                }

                // Legacy flat records are inspected only while starting. The
                // periodic scanner lists the dedicated replacement directory.
                if let Err(error) = ResumeUtils::migrate_legacy_replacement_records(disk).await {
                    if let Some(set_disk_id) = &disk_set_disk_id {
                        self.block_replacement_recovery_set(set_disk_id);
                    }
                    warn!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        endpoint = %endpoint,
                        error = %error,
                        "Legacy replacement recovery migration failed"
                    );
                }
                let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
                    Ok(task_ids) => task_ids,
                    Err(error) => {
                        if let Some(set_disk_id) = &disk_set_disk_id {
                            self.block_replacement_recovery_set(set_disk_id);
                        }
                        warn!(
                            target: "rustfs::heal::manager",
                            event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                            component = LOG_COMPONENT_HEAL,
                            subsystem = LOG_SUBSYSTEM_MANAGER,
                            endpoint = %endpoint,
                            error = %error,
                            "Replacement recovery discovery failed"
                        );
                        continue;
                    }
                };
                for task_id in replacement_task_ids {
                    let manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
                        Ok(manager) => manager,
                        Err(error) => {
                            if let Some(set_disk_id) = &disk_set_disk_id {
                                self.block_replacement_recovery_set(set_disk_id);
                            }
                            warn!(
                                target: "rustfs::heal::manager",
                                event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                                component = LOG_COMPONENT_HEAL,
                                subsystem = LOG_SUBSYSTEM_MANAGER,
                                endpoint = %endpoint,
                                task_id,
                                error = %error,
                                "Replacement recovery intent load failed"
                            );
                            continue;
                        }
                    };
                    let state = manager.get_state().await;
                    let active_replacement = !state.completed
                        && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding);
                    let verified_replacement = state.completed
                        && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending);
                    if (active_replacement || verified_replacement)
                        && state.replacement_generation.as_deref() == Some(task_id.as_str())
                        && !state.replacement_targets.is_empty()
                    {
                        if matches!(state.replacement_phase, ReplacementPhase::CleanupPending) {
                            replacement_intents.entry(task_id).or_insert((
                                state.set_disk_id,
                                state.replacement_targets,
                                state.replacement_buckets,
                                endpoint.to_string(),
                            ));
                            continue;
                        }
                        match self.storage.replacement_target_identities(&state.replacement_targets).await {
                            Ok(identities) if identities == state.replacement_target_identities => {
                                let resume_endpoint = endpoint.to_string();
                                match replacement_intents.entry(task_id) {
                                    std::collections::hash_map::Entry::Vacant(entry) => {
                                        entry.insert((
                                            state.set_disk_id,
                                            state.replacement_targets,
                                            state.replacement_buckets,
                                            resume_endpoint,
                                        ));
                                    }
                                    std::collections::hash_map::Entry::Occupied(entry) => {
                                        let (existing_set_disk_id, existing_targets, existing_buckets, existing_anchor) =
                                            entry.get();
                                        if existing_set_disk_id != &state.set_disk_id
                                            || existing_targets != &state.replacement_targets
                                            || existing_buckets != &state.replacement_buckets
                                            || existing_anchor != &resume_endpoint
                                        {
                                            conflicted_replacement_sets.insert(state.set_disk_id.clone());
                                            self.block_replacement_recovery_set(&state.set_disk_id);
                                        }
                                    }
                                }
                            }
                            Ok(_) => {
                                if manager.abandon_replacement_intent().await.is_ok() {
                                    replacement_restarts
                                        .entry(task_id)
                                        .or_insert((state.set_disk_id, state.replacement_targets));
                                }
                            }
                            Err(_) => {}
                        }
                    }
                }
            }
        }

        if !unclean && replacement_intents.is_empty() && replacement_restarts.is_empty() {
            return;
        }

        let mut recovery_by_set = HashMap::<String, Vec<(Option<String>, Vec<String>, Vec<String>, Option<String>)>>::new();
        for (task_id, (set_disk_id, heal_endpoints, buckets, resume_endpoint)) in replacement_intents {
            recovery_by_set
                .entry(set_disk_id)
                .or_default()
                .push((Some(task_id), heal_endpoints, buckets, Some(resume_endpoint)));
        }
        for (_abandoned_task_id, (set_disk_id, heal_endpoints)) in replacement_restarts {
            recovery_by_set
                .entry(set_disk_id)
                .or_default()
                .push((None, heal_endpoints, Vec::new(), None));
        }

        for (set_disk_id, mut recoveries) in recovery_by_set {
            let Ok((pool_index, set_index)) = crate::heal::utils::parse_set_disk_id(&set_disk_id) else {
                continue;
            };
            if self.replacement_recovery_set_is_blocked(&set_disk_id) {
                debug!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    set_disk_id,
                    recovery_count = recoveries.len(),
                    "Replacement recovery deferred because durable recovery validation is blocked"
                );
                continue;
            }
            if conflicted_replacement_sets.contains(&set_disk_id) || recoveries.len() != 1 {
                self.block_replacement_recovery_set(&set_disk_id);
                debug!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    set_disk_id,
                    recovery_count = recoveries.len(),
                    "Replacement recovery deferred because multiple durable generations exist"
                );
                continue;
            }
            let reuse_single_generation = recoveries.len() == 1 && recoveries[0].0.is_some();
            let mut heal_endpoints = recoveries
                .iter_mut()
                .flat_map(|(_, targets, _, _)| std::mem::take(targets))
                .collect::<Vec<_>>();
            heal_endpoints.sort_unstable();
            heal_endpoints.dedup();
            let buckets = if reuse_single_generation {
                std::mem::take(&mut recoveries[0].2)
            } else {
                Vec::new()
            };
            let mut req = HealRequest::new(
                HealType::ErasureSet {
                    buckets,
                    set_disk_id: set_disk_id.clone(),
                },
                HealOptions {
                    pool_index: Some(pool_index),
                    set_index: Some(set_index),
                    timeout: None,
                    ..HealOptions::default()
                },
                HealPriority::Low,
            );
            if reuse_single_generation && let Some(task_id) = recoveries[0].0.take() {
                req.id = task_id;
            }
            let recovery_anchor = reuse_single_generation.then(|| recoveries[0].3.take()).flatten();
            req.source = HealRequestSource::AutoHeal;
            req.heal_endpoints = heal_endpoints;
            let request_id = req.id.clone();
            if let Some(anchor) = &recovery_anchor {
                self.replacement_recovery_anchors
                    .lock()
                    .unwrap_or_else(|poisoned| poisoned.into_inner())
                    .insert(request_id.clone(), anchor.clone());
            }
            match self.submit_heal_request(req).await {
                Ok(HealAdmissionResult::Accepted) => {}
                Ok(_) => {
                    self.replacement_recovery_anchors
                        .lock()
                        .unwrap_or_else(|poisoned| poisoned.into_inner())
                        .remove(&request_id);
                }
                Err(err) => {
                    self.replacement_recovery_anchors
                        .lock()
                        .unwrap_or_else(|poisoned| poisoned.into_inner())
                        .remove(&request_id);
                    warn!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        set_disk_id,
                        error = %err,
                        "Replacement recovery enqueue failed"
                    );
                }
            }
        }

        if !unclean || set_disk_ids.is_empty() {
            return;
        }

        info!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_MANAGER,
            set_count = set_disk_ids.len(),
            "Unclean shutdown detected; scheduling erasure-set heal for local sets"
        );

        let buckets = match self.storage.list_buckets().await {
            Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>(),
            Err(err) => {
                error!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    error = %err,
                    "Unclean-shutdown heal skipped: bucket listing failed"
                );
                return;
            }
        };

        for set_disk_id in set_disk_ids {
            let mut req = HealRequest::new(
                HealType::ErasureSet {
                    buckets: buckets.clone(),
                    set_disk_id: set_disk_id.clone(),
                },
                HealOptions {
                    timeout: None,
                    ..HealOptions::default()
                },
                HealPriority::Low,
            );
            req.source = HealRequestSource::AutoHeal;
            if let Err(err) = self.submit_heal_request(req).await {
                warn!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    set_disk_id,
                    error = %err,
                    "Unclean-shutdown heal enqueue failed"
                );
            }
        }
    }

    /// Stop HealManager
    pub async fn stop(&self) -> Result<()> {
        info!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_MANAGER_STATE,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_MANAGER,
            state = "stopping",
            "Heal manager stopping"
        );

        // cancel all tasks
        self.cancel_token.cancel();

        // wait for all tasks to complete
        let mut active_heals = self.active_heals.lock().await;
        for task in active_heals.values() {
            if let Err(e) = task.cancel().await {
                warn!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_MANAGER_STATE,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    state = "task_cancel_failed",
                    task_id = %task.id,
                    error = %e,
                    "Heal active task cancellation failed"
                );
            }
        }
        active_heals.clear();
        publish_active_heal_count(&active_heals);
        self.completed_heals.lock().await.clear();
        self.task_aliases.lock().await.clear();
        self.retrying_heals.lock().await.clear();
        crate::set_heal_queue_length(0);

        // update state
        let mut state = self.state.write().await;
        state.is_running = false;

        info!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_MANAGER_STATE,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_MANAGER,
            state = "stopped",
            "Heal manager stopped"
        );
        Ok(())
    }

    /// Submit heal request
    pub async fn submit_heal_request_with_receipt(&self, request: HealRequest) -> Result<HealAdmissionReceipt> {
        self.submit_heal_request_with_receipt_and_alias(request, false).await
    }

    pub(crate) async fn submit_heal_request_with_receipt_and_alias(
        &self,
        request: HealRequest,
        preserve_alias: bool,
    ) -> Result<HealAdmissionReceipt> {
        // HS-06 forceStart semantics (admin only): MinIO stops the old task
        // first and then starts the new one. Cancel any active admin task
        // overlapping this request's path before entering admission, so the
        // fresh task is never merged into the one being replaced.
        if request.source == HealRequestSource::Admin && request.force_start {
            let overlapping: Vec<String> = {
                let active_heals = self.active_heals.lock().await;
                active_heals
                    .iter()
                    .filter(|(task_id, task)| {
                        task.source == HealRequestSource::Admin
                            && heal_types_overlap(&request.heal_type, &task.heal_type) != OverlapVerdict::Disjoint
                            && *task_id != &request.id
                    })
                    .map(|(task_id, _)| task_id.clone())
                    .collect()
            };
            for task_id in overlapping {
                match self.cancel_task(&task_id).await {
                    Ok(_) => info!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request.id,
                        cancelled_task_id = %task_id,
                        result = "force_start_cancelled_overlap",
                        "Admin forceStart cancelled an overlapping heal task"
                    ),
                    Err(err) => warn!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request.id,
                        cancelled_task_id = %task_id,
                        error = %err,
                        result = "force_start_cancel_failed",
                        "Admin forceStart failed to cancel an overlapping heal task"
                    ),
                }
            }
        }

        let config = self.config.read().await;
        let dedup_key = PriorityHealQueue::make_dedup_key(&request);

        // Match the scheduler's active -> queue order and keep retry ownership
        // in the same atomic view. Otherwise queue -> active and
        // active -> retrying transitions can slip between duplicate checks.
        let active_heals = self.active_heals.lock().await;
        #[cfg(test)]
        pause_duplicate_admission_after_active_lock(&request.id).await;
        let mut queue = self.heal_queue.lock().await;
        let retrying_heals = self.retrying_heals.lock().await;
        let duplicate = (!request.force_start).then(|| {
            active_heal_for_dedup_key(&active_heals, &dedup_key)
                .map(|(task_id, _)| (task_id, "active"))
                .or_else(|| {
                    queue
                        .request_for_dedup_key(&dedup_key)
                        .map(|queued| (queued.id.clone(), "queued"))
                })
                .or_else(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key).map(|(task_id, _)| (task_id, "retrying")))
        });
        if let Some((merged_task_id, duplicate_state)) = duplicate.flatten() {
            // HS-06: under the minio_error overlap policy an exact duplicate
            // admin start reports the typed AlreadyRunning rejection instead
            // of the silent merge (MinIO's ErrHealAlreadyRunning).
            let admission =
                if request.source == HealRequestSource::Admin && config.overlap_policy == HealOverlapPolicy::MinioError {
                    HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
                } else {
                    Self::duplicate_admission_for_request(&request, &config)
                };
            drop(retrying_heals);
            drop(queue);
            drop(active_heals);
            Self::record_admission_metric(request.source, admission, "duplicate");

            match admission {
                HealAdmissionResult::Merged => {
                    if preserve_alias {
                        self.insert_task_alias(&request.id, &merged_task_id).await;
                    }
                    debug!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request.id,
                        merged_task_id = %merged_task_id,
                        priority = ?request.priority,
                        duplicate_state,
                        result = "merged_duplicate",
                        "Heal queue admission decided"
                    );
                }
                HealAdmissionResult::Dropped(reason) => {
                    demote_to_debug_when!(request.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", {
                        event = EVENT_HEAL_QUEUE_ADMISSION,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        request_id = %request.id,
                        priority = ?request.priority,
                        reason = reason.as_str(),
                        duplicate_state,
                        result = "dropped_duplicate",
                        "Heal queue admission decided"
                    });
                }
                HealAdmissionResult::Accepted | HealAdmissionResult::Full => {}
            }

            return Ok(HealAdmissionReceipt {
                result: admission,
                task_id: merged_task_id,
            });
        }

        // HS-06 typed overlap rejection (admin only, minio_error policy):
        // paths containing or contained by an active/queued task reject with
        // AlreadyRunning / OverlappingPaths instead of merging. Exact
        // duplicates already merged above; scanner/autoheal/read-repair
        // sources never take this path.
        if request.source == HealRequestSource::Admin && config.overlap_policy == HealOverlapPolicy::MinioError {
            let mut rejection = None;
            for (task_id, task) in active_heals.iter() {
                match heal_types_overlap(&request.heal_type, &task.heal_type) {
                    OverlapVerdict::SameTarget => {
                        rejection = Some((HealAdmissionDropReason::AlreadyRunning, task_id.clone()));
                        break;
                    }
                    OverlapVerdict::Overlapping => {
                        rejection = Some((HealAdmissionDropReason::OverlappingPaths, task_id.clone()));
                    }
                    OverlapVerdict::Disjoint => {}
                }
            }
            if rejection.is_none() {
                for queued in queue.requests() {
                    match heal_types_overlap(&request.heal_type, &queued.heal_type) {
                        OverlapVerdict::SameTarget => {
                            rejection = Some((HealAdmissionDropReason::AlreadyRunning, queued.id.clone()));
                            break;
                        }
                        OverlapVerdict::Overlapping => {
                            rejection = Some((HealAdmissionDropReason::OverlappingPaths, queued.id.clone()));
                        }
                        OverlapVerdict::Disjoint => {}
                    }
                }
            }
            if let Some((reason, overlap_task_id)) = rejection {
                drop(retrying_heals);
                drop(queue);
                drop(active_heals);
                Self::record_admission_metric(request.source, HealAdmissionResult::Dropped(reason), "overlap_rejected");
                warn!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_QUEUE_ADMISSION,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    request_id = %request.id,
                    overlap_task_id = %overlap_task_id,
                    reason = reason.as_str(),
                    result = "overlap_rejected",
                    "Admin heal start rejected by overlap policy"
                );
                return Ok(HealAdmissionReceipt {
                    result: HealAdmissionResult::Dropped(reason),
                    task_id: overlap_task_id,
                });
            }
        }

        let mut task_id = request.id.clone();
        let admission = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
        if admission == HealAdmissionResult::Merged
            && let Some(queued) = queue.request_for_dedup_key(&dedup_key)
        {
            task_id.clone_from(&queued.id);
        }
        let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
        drop(retrying_heals);
        drop(queue);
        drop(active_heals);

        if should_notify {
            self.notify.notify_one();
        }

        Ok(HealAdmissionReceipt {
            result: admission,
            task_id,
        })
    }

    /// Submit heal request.
    pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
        Ok(self.submit_heal_request_with_receipt_and_alias(request, true).await?.result)
    }

    /// Get task status
    pub async fn get_task_status(&self, task_id: &str) -> Result<HealTaskStatus> {
        let canonical_task_id = self.canonical_task_id(task_id).await;
        {
            let active_heals = self.active_heals.lock().await;
            if let Some(task) = active_heals.get(&canonical_task_id) {
                return Ok(task.get_status().await);
            }
        }

        {
            let retrying_heals = self.retrying_heals.lock().await;
            if let Some(retrying) = retrying_heals.get(&canonical_task_id) {
                return Ok(retrying.status());
            }
        }

        {
            let mut completed_heals = self.completed_heals.lock().await;
            prune_completed_heal_statuses(&mut completed_heals);
            if let Some(completed) = completed_heals.get(&canonical_task_id)
                && completed_status_is_retrying(&completed.status)
            {
                return Ok(completed.status.clone());
            }
        }

        let queue = self.heal_queue.lock().await;
        if queue.contains_request_id(&canonical_task_id) {
            return Ok(HealTaskStatus::Pending);
        }
        drop(queue);

        let mut completed_heals = self.completed_heals.lock().await;
        prune_completed_heal_statuses(&mut completed_heals);
        if let Some(completed) = completed_heals.get(&canonical_task_id) {
            return Ok(completed.status.clone());
        }

        Err(Error::TaskNotFound {
            task_id: task_id.to_string(),
        })
    }

    pub async fn get_task_report(&self, task_id: &str) -> Result<HealTaskReport> {
        self.get_task_report_since(task_id, None).await
    }

    /// Incremental variant of [`Self::get_task_report`] (HS-06): `since` is
    /// the client's last seen sequence number; `None` keeps the legacy
    /// full-snapshot semantics.
    pub async fn get_task_report_since(&self, task_id: &str, since: Option<u64>) -> Result<HealTaskReport> {
        let canonical_task_id = self.canonical_task_id(task_id).await;
        {
            let active_heals = self.active_heals.lock().await;
            if let Some(task) = active_heals.get(&canonical_task_id) {
                return Ok(active_task_report(task, since).await);
            }
        }

        {
            let retrying_heals = self.retrying_heals.lock().await;
            if let Some(retrying) = retrying_heals.get(&canonical_task_id) {
                return Ok(empty_task_report(retrying.status()));
            }
        }

        {
            let mut completed_heals = self.completed_heals.lock().await;
            prune_completed_heal_statuses(&mut completed_heals);
            if let Some(completed) = completed_heals.get(&canonical_task_id)
                && completed_status_is_retrying(&completed.status)
            {
                return Ok(completed_task_report(completed, since));
            }
        }

        {
            let queue = self.heal_queue.lock().await;
            if queue.contains_request_id(&canonical_task_id) {
                return Ok(empty_task_report(HealTaskStatus::Pending));
            }
        }

        let mut completed_heals = self.completed_heals.lock().await;
        prune_completed_heal_statuses(&mut completed_heals);
        if let Some(completed) = completed_heals.get(&canonical_task_id) {
            return Ok(completed_task_report(completed, since));
        }

        Err(Error::TaskNotFound {
            task_id: task_id.to_string(),
        })
    }

    pub async fn get_task_report_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskReport> {
        self.get_task_report_for_path_since(heal_path, task_id, None).await
    }

    /// Incremental variant of [`Self::get_task_report_for_path`] (HS-06).
    pub async fn get_task_report_for_path_since(
        &self,
        heal_path: &str,
        task_id: &str,
        since: Option<u64>,
    ) -> Result<HealTaskReport> {
        let canonical_task_id = self.canonical_task_id(task_id).await;
        {
            let active_heals = self.active_heals.lock().await;
            if let Some(task) = active_heals.get(&canonical_task_id)
                && heal_type_matches_path(&task.heal_type, heal_path)
            {
                return Ok(active_task_report(task, since).await);
            }
        }

        {
            let retrying_heals = self.retrying_heals.lock().await;
            if let Some(retrying) = retrying_heals.get(&canonical_task_id)
                && heal_type_matches_path(&retrying.request.heal_type, heal_path)
            {
                return Ok(empty_task_report(retrying.status()));
            }
        }

        {
            let mut completed_heals = self.completed_heals.lock().await;
            prune_completed_heal_statuses(&mut completed_heals);
            if let Some(completed) = completed_heals.get(&canonical_task_id)
                && heal_type_matches_path(&completed.heal_type, heal_path)
                && completed_status_is_retrying(&completed.status)
            {
                return Ok(completed_task_report(completed, since));
            }
        }

        {
            let queue = self.heal_queue.lock().await;
            if queue.contains_request_id_matching_path(&canonical_task_id, heal_path) {
                return Ok(empty_task_report(HealTaskStatus::Pending));
            }
        }

        {
            let mut completed_heals = self.completed_heals.lock().await;
            prune_completed_heal_statuses(&mut completed_heals);
            if let Some(completed) = completed_heals.get(&canonical_task_id)
                && heal_type_matches_path(&completed.heal_type, heal_path)
            {
                return Ok(completed_task_report(completed, since));
            }
        }

        if self.path_has_task(heal_path).await {
            return Err(Error::InvalidClientToken);
        }

        Err(Error::TaskNotFound {
            task_id: task_id.to_string(),
        })
    }

    /// Get task status for a path-bound client token.
    ///
    /// If the token is unknown but no task remains for the path, the caller can
    /// treat it as an already-finished sequence. If the path still has a live or
    /// recently completed task, a different token is invalid for that path.
    pub async fn get_task_status_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskStatus> {
        let canonical_task_id = self.canonical_task_id(task_id).await;
        {
            let active_heals = self.active_heals.lock().await;
            if let Some(task) = active_heals.get(&canonical_task_id)
                && heal_type_matches_path(&task.heal_type, heal_path)
            {
                return Ok(task.get_status().await);
            }
        }

        {
            let retrying_heals = self.retrying_heals.lock().await;
            if let Some(retrying) = retrying_heals.get(&canonical_task_id)
                && heal_type_matches_path(&retrying.request.heal_type, heal_path)
            {
                return Ok(retrying.status());
            }
        }

        {
            let mut completed_heals = self.completed_heals.lock().await;
            prune_completed_heal_statuses(&mut completed_heals);
            if let Some(completed) = completed_heals.get(&canonical_task_id)
                && heal_type_matches_path(&completed.heal_type, heal_path)
                && completed_status_is_retrying(&completed.status)
            {
                return Ok(completed.status.clone());
            }
        }

        {
            let queue = self.heal_queue.lock().await;
            if queue.contains_request_id_matching_path(&canonical_task_id, heal_path) {
                return Ok(HealTaskStatus::Pending);
            }
        }

        {
            let mut completed_heals = self.completed_heals.lock().await;
            prune_completed_heal_statuses(&mut completed_heals);
            if let Some(completed) = completed_heals.get(&canonical_task_id)
                && heal_type_matches_path(&completed.heal_type, heal_path)
            {
                return Ok(completed.status.clone());
            }
        }

        if self.path_has_task(heal_path).await {
            return Err(Error::InvalidClientToken);
        }

        Err(Error::TaskNotFound {
            task_id: task_id.to_string(),
        })
    }

    async fn path_has_task(&self, heal_path: &str) -> bool {
        {
            let active_heals = self.active_heals.lock().await;
            if active_heals
                .values()
                .any(|task| heal_type_matches_path(&task.heal_type, heal_path))
            {
                return true;
            }
        }

        {
            let queue = self.heal_queue.lock().await;
            if queue.contains_matching(|request| heal_type_matches_path(&request.heal_type, heal_path)) {
                return true;
            }
        }

        {
            let retrying_heals = self.retrying_heals.lock().await;
            if retrying_heals
                .values()
                .any(|retrying| heal_type_matches_path(&retrying.request.heal_type, heal_path))
            {
                return true;
            }
        }

        let mut completed_heals = self.completed_heals.lock().await;
        prune_completed_heal_statuses(&mut completed_heals);
        completed_heals
            .values()
            .any(|completed| heal_type_matches_path(&completed.heal_type, heal_path))
    }

    /// Get task progress
    pub async fn get_active_tasks_count(&self) -> usize {
        let active_heals = self.active_heals.lock().await;
        publish_active_heal_count(&active_heals);
        active_heals.len()
    }

    pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
        let canonical_task_id = self.canonical_task_id(task_id).await;
        let active_heals = self.active_heals.lock().await;
        if let Some(task) = active_heals.get(&canonical_task_id) {
            Ok(task.get_progress().await)
        } else {
            Err(Error::TaskNotFound {
                task_id: task_id.to_string(),
            })
        }
    }

    /// Cancel task
    pub async fn cancel_task(&self, task_id: &str) -> Result<()> {
        let canonical_task_id = self.canonical_task_id(task_id).await;
        {
            let mut active_heals = self.active_heals.lock().await;
            if let Some(task) = active_heals.get(&canonical_task_id) {
                task.cancel().await?;
                active_heals.remove(&canonical_task_id);
                publish_active_heal_count(&active_heals);
                info!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_MANAGER_STATE,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    task_id = %canonical_task_id,
                    state = "cancelled_active_task",
                    "Heal manager cancelled active task"
                );
                drop(active_heals);
                self.remove_aliases_for_task(&canonical_task_id).await;
                return Ok(());
            }
        }

        {
            let mut retrying_heals = self.retrying_heals.lock().await;
            if let Some(retrying) = retrying_heals.remove(&canonical_task_id) {
                retrying.cancel_token.cancel();
                drop(retrying_heals);
                self.completed_heals.lock().await.remove(&canonical_task_id);
                self.remove_aliases_for_task(&canonical_task_id).await;
                info!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_MANAGER_STATE,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    task_id = %canonical_task_id,
                    state = "cancelled_retrying_task",
                    "Heal manager cancelled retrying task"
                );
                return Ok(());
            }
        }

        let mut queue = self.heal_queue.lock().await;
        if queue.remove_request_id(&canonical_task_id).is_some() {
            publish_heal_queue_length(&queue);
            info!(
                target: "rustfs::heal::manager",
                event = EVENT_HEAL_MANAGER_STATE,
                component = LOG_COMPONENT_HEAL,
                subsystem = LOG_SUBSYSTEM_MANAGER,
                task_id = %canonical_task_id,
                state = "cancelled_queued_task",
                "Heal manager cancelled queued task"
            );
            drop(queue);
            self.remove_aliases_for_task(&canonical_task_id).await;
            return Ok(());
        }

        Err(Error::TaskNotFound {
            task_id: task_id.to_string(),
        })
    }

    /// Cancel all queued or active tasks matching a heal path.
    pub async fn cancel_tasks_for_path(&self, heal_path: &str) -> Result<usize> {
        let mut cancelled = 0usize;

        {
            let mut active_heals = self.active_heals.lock().await;
            let task_ids = active_heals
                .iter()
                .filter_map(|(task_id, task)| heal_type_matches_path(&task.heal_type, heal_path).then_some(task_id.clone()))
                .collect::<Vec<_>>();

            for task_id in &task_ids {
                if let Some(task) = active_heals.get(task_id) {
                    task.cancel().await?;
                }
                active_heals.remove(task_id);
                cancelled += 1;
            }

            if cancelled > 0 {
                publish_active_heal_count(&active_heals);
            }
            drop(active_heals);
            for task_id in &task_ids {
                self.remove_aliases_for_task(task_id).await;
            }
        }

        {
            let mut retrying_heals = self.retrying_heals.lock().await;
            let task_ids = retrying_heals
                .iter()
                .filter_map(|(task_id, retrying)| {
                    heal_type_matches_path(&retrying.request.heal_type, heal_path).then_some(task_id.clone())
                })
                .collect::<Vec<_>>();

            for task_id in &task_ids {
                if let Some(retrying) = retrying_heals.remove(task_id) {
                    retrying.cancel_token.cancel();
                    cancelled += 1;
                }
            }
            drop(retrying_heals);

            if !task_ids.is_empty() {
                let mut completed_heals = self.completed_heals.lock().await;
                for task_id in &task_ids {
                    completed_heals.remove(task_id);
                }
                drop(completed_heals);

                for task_id in &task_ids {
                    self.remove_aliases_for_task(task_id).await;
                }
            }
        }

        let mut queue = self.heal_queue.lock().await;
        let queued_cancelled = queue.remove_matching(|request| heal_type_matches_path(&request.heal_type, heal_path));
        if !queued_cancelled.is_empty() {
            publish_heal_queue_length(&queue);
            cancelled += queued_cancelled.len();
        }
        drop(queue);
        for request in &queued_cancelled {
            self.remove_aliases_for_task(&request.id).await;
        }

        if cancelled == 0 {
            return Err(Error::TaskNotFound {
                task_id: heal_path.to_string(),
            });
        }

        info!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_MANAGER_STATE,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_MANAGER,
            heal_path,
            cancelled,
            state = "cancelled_path_tasks",
            "Heal manager cancelled tasks for path"
        );
        Ok(cancelled)
    }

    /// Get statistics
    pub async fn get_statistics(&self) -> HealStatistics {
        self.statistics.read().await.clone()
    }

    /// Get active task count
    pub async fn get_active_task_count(&self) -> usize {
        let active_heals = self.active_heals.lock().await;
        publish_active_heal_count(&active_heals);
        active_heals.len()
    }

    /// Get queue length
    pub async fn get_queue_length(&self) -> usize {
        let queue = self.heal_queue.lock().await;
        publish_heal_queue_length(&queue);
        queue.len()
    }

    pub async fn operations_snapshot(&self) -> HealOperationsSnapshot {
        // Match the scheduler's active -> queue order. Retry transitions also
        // acquire active before retrying, so the three sets form one snapshot.
        let active_heals = self.active_heals.lock().await;
        let queue = self.heal_queue.lock().await;
        let retrying_heals = self.retrying_heals.lock().await;
        let (queued_by_priority, queued_by_source) = queue.operation_counts();
        let mut active_by_priority = HealPriorityCounts::default();
        let mut active_by_source = HealSourceCounts::default();
        for task in active_heals.values() {
            active_by_priority.increment(task.priority);
            active_by_source.increment(task.source);
        }
        let mut retrying_by_priority = HealPriorityCounts::default();
        let mut retrying_by_source = HealSourceCounts::default();
        for retrying in retrying_heals.values() {
            retrying_by_priority.increment(retrying.request.priority);
            retrying_by_source.increment(retrying.request.source);
        }
        publish_active_heal_count(&active_heals);
        publish_heal_queue_length(&queue);

        HealOperationsSnapshot {
            queue_length: usize_to_u64_saturated(queue.len()),
            active_tasks: usize_to_u64_saturated(active_heals.len()),
            retrying_tasks: usize_to_u64_saturated(retrying_heals.len()),
            queued_by_priority,
            active_by_priority,
            retrying_by_priority,
            queued_by_source,
            active_by_source,
            retrying_by_source,
        }
    }

    pub async fn active_progress_snapshot(&self) -> Option<HealProgress> {
        let active_tasks = {
            let active_heals = self.active_heals.lock().await;
            active_heals.values().cloned().collect::<Vec<_>>()
        };
        if active_tasks.is_empty() {
            return None;
        }

        let mut snapshot = HealProgress::default();
        for task in active_tasks {
            let progress = task.get_progress().await;
            snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
            snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
            snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed);
            snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions);
            snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired);
            snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count);
            snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size);
            snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed);
            snapshot.start_time = match (snapshot.start_time, progress.start_time) {
                (Some(current), Some(next)) => Some(current.min(next)),
                (None, next) => next,
                (current, None) => current,
            };
            snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
                (Some(current), Some(next)) => Some(current.max(next)),
                (None, next) => next,
                (current, None) => current,
            };
            if progress.current_object.is_some() {
                snapshot.current_object = progress.current_object;
            }
        }
        snapshot.refresh_progress_percentage();
        snapshot.refresh_estimated_completion_time();
        Some(snapshot)
    }

    /// Start scheduler
    async fn start_scheduler(&self) -> Result<()> {
        let config = self.config.clone();
        let heal_queue = self.heal_queue.clone();
        let active_heals = self.active_heals.clone();
        let completed_heals = self.completed_heals.clone();
        let retrying_heals = self.retrying_heals.clone();
        let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
        let cancel_token = self.cancel_token.clone();
        let statistics = self.statistics.clone();
        let storage = self.storage.clone();
        let notify = self.notify.clone();
        let workload_provider = self.workload_provider.clone();

        tokio::spawn(async move {
            let mut interval = interval(config.read().await.heal_interval);

            loop {
                let event_driven_scheduler_enable = config.read().await.event_driven_scheduler_enable;
                tokio::select! {
                    _ = cancel_token.cancelled() => {
                        info!(
                            target: "rustfs::heal::manager",
                            event = EVENT_HEAL_SCHEDULER_STATE,
                            component = LOG_COMPONENT_HEAL,
                            subsystem = LOG_SUBSYSTEM_MANAGER,
                            state = "shutdown",
                            "Heal scheduler stopped"
                        );
                        break;
                    }
                    _ = notify.notified(), if event_driven_scheduler_enable => {
                        Self::process_heal_queue(HealQueueContext {
                            heal_queue: &heal_queue,
                            active_heals: &active_heals,
                            completed_heals: &completed_heals,
                            retrying_heals: &retrying_heals,
                            replacement_recovery_anchors: &replacement_recovery_anchors,
                            config: &config,
                            statistics: &statistics,
                            storage: &storage,
                            notify: &notify,
                            cancel_token: &cancel_token,
                            workload_provider: &workload_provider,
                        })
                        .await;
                    }
                    _ = interval.tick() => {
                        Self::process_heal_queue(HealQueueContext {
                            heal_queue: &heal_queue,
                            active_heals: &active_heals,
                            completed_heals: &completed_heals,
                            retrying_heals: &retrying_heals,
                            replacement_recovery_anchors: &replacement_recovery_anchors,
                            config: &config,
                            statistics: &statistics,
                            storage: &storage,
                            notify: &notify,
                            cancel_token: &cancel_token,
                            workload_provider: &workload_provider,
                        })
                        .await;
                    }
                }
            }
        });

        Ok(())
    }

    /// Start background task to auto scan local disks and enqueue erasure set heal requests
    async fn start_auto_disk_scanner(&self) -> Result<()> {
        let config = self.config.clone();
        let heal_queue = self.heal_queue.clone();
        let active_heals = self.active_heals.clone();
        let storage = self.storage.clone();
        let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
        let replacement_recovery_blocked_sets = self.replacement_recovery_blocked_sets.clone();
        let cancel_token = self.cancel_token.clone();
        let notify = self.notify.clone();
        let mut duration = {
            let config = config.read().await;
            config.heal_interval
        };
        if duration < Duration::from_secs(10) {
            duration = Duration::from_secs(10);
        }
        info!(
            target: "rustfs::heal::manager",
            event = EVENT_HEAL_AUTO_SCAN_STATE,
            component = LOG_COMPONENT_HEAL,
            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
            state = "started",
            interval = ?duration,
            "Heal auto disk scanner started"
        );

        tokio::spawn(async move {
            let mut interval = interval(duration);

            loop {
                let mut candidate_count = 0usize;
                let mut skipped_duplicate_count = 0usize;
                let mut skipped_invalid_count = 0usize;
                let mut enqueued_count = 0usize;
                let mut not_enqueued_count = 0usize;
                let mut dropped_count = 0usize;
                let mut full_count = 0usize;
                tokio::select! {
                    _ = cancel_token.cancelled() => {
                        info!(
                            target: "rustfs::heal::manager",
                            event = EVENT_HEAL_AUTO_SCAN_STATE,
                            component = LOG_COMPONENT_HEAL,
                            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                            state = "shutdown",
                            "Heal auto disk scanner stopped"
                        );
                        break;
                    }
                    _ = interval.tick() => {
                        // Build list of endpoints that need healing
                        let mut endpoints = HashMap::<String, Vec<Endpoint>>::new();
                        let mut durable_recoveries = HashMap::<String, (String, Vec<Endpoint>, Vec<String>, String)>::new();
                        let mut conflicted_recovery_sets = HashSet::<String>::new();
                        let mut deferred_replacement_endpoints = HashSet::<String>::new();
                        let local_disks = {
                            let local_disk_map = local_disk_map_read().await;
                            local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
                        };
                        let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::<Vec<_>>();
                        let blocked_sets = replacement_recovery_blocked_sets
                            .lock()
                            .expect("replacement recovery blocked set lock poisoned")
                            .clone();
                        if !blocked_sets.is_empty() {
                            let mut retry_succeeded = HashSet::new();
                            let mut retry_failed = HashSet::new();
                            for disk in &local_disks {
                                let endpoint = disk.endpoint();
                                let Some(set_disk_id) =
                                    crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx)
                                else {
                                    continue;
                                };
                                if !blocked_sets.contains(&set_disk_id) {
                                    continue;
                                }
                                match Self::validate_replacement_recovery_records(disk).await {
                                    Ok(()) => {
                                        retry_succeeded.insert(set_disk_id);
                                    }
                                    Err(error) => {
                                        retry_failed.insert(set_disk_id.clone());
                                        conflicted_recovery_sets.insert(set_disk_id);
                                        warn!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                            endpoint = %endpoint,
                                            error = %error,
                                            "Replacement recovery retry failed"
                                        );
                                    }
                                }
                            }
                            let mut blocked = replacement_recovery_blocked_sets
                                .lock()
                                .expect("replacement recovery blocked set lock poisoned");
                            unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed);
                        }
                        for disk in &local_disks {
                            let endpoint = disk.endpoint();
                                let runtime_state = disk.runtime_state();
                                let set_disk_id =
                                    crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
                            if set_disk_id.as_ref().is_some_and(|set_disk_id| {
                                replacement_recovery_blocked_sets
                                    .lock()
                                    .expect("replacement recovery blocked set lock poisoned")
                                    .contains(set_disk_id)
                            }) {
                                skipped_invalid_count += 1;
                                debug!(
                                    target: "rustfs::heal::manager",
                                    event = EVENT_HEAL_AUTO_SCAN_DISK,
                                    component = LOG_COMPONENT_HEAL,
                                    subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                    endpoint = %endpoint,
                                    set_disk_id = set_disk_id.as_deref().unwrap_or_default(),
                                    disk_state = "replacement_recovery_blocked",
                                    "Heal auto-scan replacement deferred because durable recovery is blocked"
                                );
                                continue;
                            }

                            // detect unformatted disk via get_disk_id()
                            match disk.get_disk_id().await {
                                Err(DiskError::UnformattedDisk) => {
                                    if !super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
                                        .await
                                    {
                                        deferred_replacement_endpoints.insert(endpoint.to_string());
                                        skipped_invalid_count += 1;
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_AUTO_SCAN_DISK,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                            endpoint = %endpoint,
                                            disk_state = "replacement_path_unavailable",
                                            "Heal auto-scan replacement deferred"
                                        );
                                        continue;
                                    }
                                    let Some(set_disk_id) = set_disk_id else {
                                        skipped_invalid_count += 1;
                                        continue;
                                    };
                                    candidate_count += 1;
                                    debug!(
                                        target: "rustfs::heal::manager",
                                        event = EVENT_HEAL_AUTO_SCAN_DISK,
                                        component = LOG_COMPONENT_HEAL,
                                        subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                        endpoint = %endpoint,
                                        disk_state = "unformatted",
                                        "Heal auto-scan candidate detected"
                                    );
                                    endpoints.entry(set_disk_id).or_default().push(endpoint);
                                }
                                Err(e) => {
                                    warn!(
                                        target: "rustfs::heal::manager",
                                        event = EVENT_HEAL_AUTO_SCAN_DISK,
                                        component = LOG_COMPONENT_HEAL,
                                        subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                        endpoint = %endpoint,
                                        disk_state = "check_failed",
                                        error = ?e,
                                        "Heal auto-scan disk inspection failed"
                                    );
                                }
                                Ok(_) => {
                                    if runtime_state.as_str() == "returning" && let Some(set_disk_id) = set_disk_id {
                                        candidate_count += 1;
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_AUTO_SCAN_DISK,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                            endpoint = %endpoint,
                                            set_disk_id,
                                            disk_state = "returning",
                                            "Heal auto-scan returning disk candidate detected"
                                        );
                                        endpoints.entry(set_disk_id).or_default().push(endpoint);
                                    }
                                }
                            }
                        }

                        // Once formatting succeeds a replacement is no longer
                        // discoverable as UnformattedDisk. Re-admit exactly one
                        // incomplete durable generation per set after bounded
                        // scheduler retries are exhausted, or re-admit its
                        // verified terminal cleanup. Multiple generations are a
                        // durable conflict: leave every marker/state intact and
                        // require reconciliation rather than choosing one.
                        for disk in &local_disks {
                            let endpoint = disk.endpoint();
                            let disk_set_disk_id =
                                crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
                            let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
                                Ok(task_ids) => task_ids,
                                Err(error) => {
                                    let endpoint_string = endpoint.to_string();
                                    if replacement_discovery_error_is_expected_for_deferred_endpoint(
                                        &error,
                                        &endpoint_string,
                                        &deferred_replacement_endpoints,
                                    ) {
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                            endpoint = %endpoint,
                                            disk_state = "replacement_path_unavailable",
                                            result = "recovery_records_unavailable",
                                            "Replacement recovery discovery skipped for deferred replacement"
                                        );
                                        continue;
                                    }
                                    if let Some(set_disk_id) = &disk_set_disk_id {
                                        conflicted_recovery_sets.insert(set_disk_id.clone());
                                    }
                                    warn!(
                                        target: "rustfs::heal::manager",
                                        event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                        component = LOG_COMPONENT_HEAL,
                                        subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                        endpoint = %endpoint,
                                        error = %error,
                                        "Replacement recovery discovery failed"
                                    );
                                    continue;
                                }
                            };
                            for task_id in replacement_task_ids {
                                let resume_manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
                                    Ok(resume_manager) => resume_manager,
                                    Err(error) => {
                                        if let Some(set_disk_id) = &disk_set_disk_id {
                                            conflicted_recovery_sets.insert(set_disk_id.clone());
                                        }
                                        warn!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                            endpoint = %endpoint,
                                            task_id,
                                            error = %error,
                                            "Replacement recovery intent load failed"
                                        );
                                        continue;
                                    }
                                };
                                let state = resume_manager.get_state().await;
                                if !durable_replacement_recovery_is_due(&state, &task_id) {
                                    continue;
                                }
                                if !matches!(state.replacement_phase, ReplacementPhase::CleanupPending) {
                                    let Ok(identities) = storage.replacement_target_identities(&state.replacement_targets).await else {
                                        continue;
                                    };
                                    if identities != state.replacement_target_identities {
                                        continue;
                                    }
                                }
                                let targets = state
                                    .replacement_targets
                                    .iter()
                                    .filter_map(|target| {
                                        local_endpoints
                                            .iter()
                                            .find(|endpoint| endpoint.to_string() == *target)
                                            .cloned()
                                    })
                                    .collect::<Vec<_>>();
                                if targets.len() != state.replacement_targets.len() {
                                    continue;
                                }
                                let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32(
                                    targets[0].pool_idx,
                                    targets[0].set_idx,
                                ) else {
                                    continue;
                                };
                                if targets.iter().any(|target| {
                                    crate::heal::utils::format_set_disk_id_from_i32(target.pool_idx, target.set_idx)
                                        .as_deref()
                                        != Some(set_disk_id.as_str())
                                }) {
                                    continue;
                                }
                                let resume_endpoint = disk.endpoint().to_string();
                                match durable_recoveries.get(&set_disk_id) {
                                    Some((existing_task_id, _, _, existing_anchor))
                                        if existing_task_id != &task_id || existing_anchor != &resume_endpoint => {
                                        replacement_recovery_blocked_sets
                                            .lock()
                                            .expect("replacement recovery blocked set lock poisoned")
                                            .insert(set_disk_id.clone());
                                        conflicted_recovery_sets.insert(set_disk_id);
                                    }
                                    Some(_) => {}
                                    None => {
                                        durable_recoveries.insert(
                                            set_disk_id,
                                            (task_id, targets, state.replacement_buckets, resume_endpoint),
                                        );
                                    }
                                }
                            }
                        }

                        for set_disk_id in &conflicted_recovery_sets {
                            durable_recoveries.remove(set_disk_id);
                            endpoints.remove(set_disk_id);
                            warn!(
                                target: "rustfs::heal::manager",
                                event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                component = LOG_COMPONENT_HEAL,
                                subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                set_disk_id,
                                result = "durable_generation_conflict",
                                "Replacement recovery deferred because multiple durable generations exist"
                            );
                        }

                        for (set_disk_id, (_, targets, _, _)) in &durable_recoveries {
                            let expected = targets.iter().map(ToString::to_string).collect::<HashSet<_>>();
                            let observed = endpoints
                                .get(set_disk_id)
                                .map(|endpoints| endpoints.iter().map(ToString::to_string).collect::<HashSet<_>>())
                                .unwrap_or_default();
                            if !observed.is_subset(&expected) {
                                replacement_recovery_blocked_sets
                                    .lock()
                                    .expect("replacement recovery blocked set lock poisoned")
                                    .insert(set_disk_id.clone());
                                conflicted_recovery_sets.insert(set_disk_id.clone());
                                continue;
                            }
                            endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone());
                        }
                        for set_disk_id in &conflicted_recovery_sets {
                            durable_recoveries.remove(set_disk_id);
                            endpoints.remove(set_disk_id);
                        }

                        for target_endpoints in endpoints.values_mut() {
                            target_endpoints.sort_by_key(ToString::to_string);
                            target_endpoints.dedup_by(|left, right| left.to_string() == right.to_string());
                        }

                        if endpoints.is_empty() {
                            debug!(
                                target: "rustfs::heal::manager",
                                event = EVENT_HEAL_AUTO_SCAN_STATE,
                                component = LOG_COMPONENT_HEAL,
                                subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                state = "idle",
                                "Heal auto disk scanner idle"
                            );
                            continue;
                        }

                        // Admit one set task with every ready replacement target. Queue deduplication is
                        // set-scoped, so admitting endpoints independently would silently drop later targets.
                        for (set_disk_id, endpoints) in endpoints {
                            if replacement_recovery_blocked_sets
                                .lock()
                                .expect("replacement recovery blocked set lock poisoned")
                                .contains(&set_disk_id)
                            {
                                skipped_invalid_count += 1;
                                debug!(
                                    target: "rustfs::heal::manager",
                                    event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                    component = LOG_COMPONENT_HEAL,
                                    subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                    set_disk_id,
                                    result = "replacement_recovery_blocked",
                                    "Heal auto-scan replacement admission deferred because durable recovery is blocked"
                                );
                                continue;
                            }
                            // skip if already queued or healing
                            // Use consistent lock order: queue first, then active_heals to avoid deadlock
                            let mut skip = false;
                            {
                                let queue = heal_queue.lock().await;
                                if queue.contains_erasure_set(&set_disk_id) {
                                    skip = true;
                                }
                            }
                            if !skip {
                                let active = active_heals.lock().await;
                                if active.values().any(|task| {
                                    matches!(
                                        &task.heal_type,
                                        crate::heal::task::HealType::ErasureSet { set_disk_id: active_id, .. }
                                        if active_id == &set_disk_id
                                    )
                                }) {
                                    skip = true;
                                }
                            }

                            if skip {
                                skipped_duplicate_count += 1;
                                debug!(
                                    target: "rustfs::heal::manager",
                                    event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                    component = LOG_COMPONENT_HEAL,
                                    subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                    endpoint_count = endpoints.len(),
                                    set_disk_id,
                                    result = "skipped_duplicate",
                                    "Heal auto-scan duplicate skipped"
                                );
                                continue;
                            }

                            // enqueue erasure set heal request for all ready replacements in this set
                            let recovery = durable_recoveries.remove(&set_disk_id);
                            let mut req = HealRequest::new(
                                HealType::ErasureSet {
                                    buckets: recovery
                                        .as_ref()
                                        .map(|(_, _, buckets, _)| buckets.clone())
                                        .unwrap_or_default(),
                                    set_disk_id: set_disk_id.clone(),
                                },
                                HealOptions {
                                    pool_index: endpoints
                                        .first()
                                        .and_then(|endpoint| usize::try_from(endpoint.pool_idx).ok()),
                                    set_index: endpoints
                                        .first()
                                        .and_then(|endpoint| usize::try_from(endpoint.set_idx).ok()),
                                    timeout: None,
                                    ..HealOptions::default()
                                },
                                HealPriority::Low,
                            );
                            let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone());
                            if let Some((task_id, _, _, _)) = recovery {
                                req.id = task_id;
                            }
                            req.source = HealRequestSource::AutoHeal;
                            req.heal_endpoints = endpoints.iter().map(ToString::to_string).collect();
                            let request_id = req.id.clone();
                            let endpoint_count = req.heal_endpoints.len();
                            let config = config.read().await;
                            let mut queue = heal_queue.lock().await;
                            let admission = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan");
                            let should_notify =
                                matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
                            if matches!(admission, HealAdmissionResult::Accepted)
                                && let Some(anchor) = recovery_anchor
                            {
                                replacement_recovery_anchors
                                    .lock()
                                    .unwrap_or_else(|poisoned| poisoned.into_inner())
                                    .insert(request_id, anchor);
                            }
                            drop(queue);
                            drop(config);
                            if matches!(admission, HealAdmissionResult::Accepted) {
                                if should_notify {
                                    notify.notify_one();
                                }
                                enqueued_count += 1;
                                debug!(
                                    target: "rustfs::heal::manager",
                                    event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                    component = LOG_COMPONENT_HEAL,
                                    subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                    endpoint_count,
                                    set_disk_id,
                                    bucket_count = 0,
                                    result = "enqueued",
                                    "Heal auto-scan task enqueued"
                                );
                            } else {
                                if matches!(admission, HealAdmissionResult::Merged) {
                                    skipped_duplicate_count += 1;
                                } else {
                                    not_enqueued_count += 1;
                                }
                                if matches!(admission, HealAdmissionResult::Full) {
                                    full_count += 1;
                                }
                                if matches!(admission, HealAdmissionResult::Dropped(_)) {
                                    dropped_count += 1;
                                }
                                debug!(
                                    target: "rustfs::heal::manager",
                                    event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
                                    component = LOG_COMPONENT_HEAL,
                                    subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                                    endpoint_count,
                                    set_disk_id,
                                    bucket_count = 0,
                                    admission = admission.result_label(),
                                    reason = admission.reason_label(),
                                    result = "not_enqueued",
                                    "Heal auto-scan task not enqueued"
                                );
                            }
                        }
                        info!(
                            target: "rustfs::heal::manager",
                            event = EVENT_HEAL_AUTO_SCAN_STATE,
                            component = LOG_COMPONENT_HEAL,
                            subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
                            state = "cycle_completed",
                            candidate_count,
                            enqueued_count,
                            not_enqueued_count,
                            dropped_count,
                            full_count,
                            skipped_duplicate_count,
                            skipped_invalid_count,
                            "Heal auto-scan cycle completed"
                        );
                    }
                }
            }
        });
        Ok(())
    }

    /// Process heal queue
    /// Processes multiple tasks per cycle when capacity allows and queue has high-priority items
    async fn process_heal_queue(context: HealQueueContext<'_>) {
        let HealQueueContext {
            heal_queue,
            active_heals,
            completed_heals,
            retrying_heals,
            replacement_recovery_anchors,
            config,
            statistics,
            storage,
            notify,
            cancel_token,
            workload_provider,
        } = context;

        let config = config.read().await;
        let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider);
        let mut active_heals_guard = active_heals.lock().await;
        publish_active_heal_count(&active_heals_guard);

        // Check if new heal tasks can be started
        let active_count = active_heals_guard.len();
        if active_count >= config.max_concurrent_heals {
            return;
        }

        // Calculate how many tasks we can start this cycle
        let available_slots = config.max_concurrent_heals - active_count;

        let mut queue = heal_queue.lock().await;
        let queue_len = queue.len();
        publish_heal_queue_length(&queue);

        if queue_len == 0 {
            return;
        }

        let mut running_per_set = running_heal_set_counts(&active_heals_guard);
        let mut tasks_started = 0usize;
        let mut delayed_by_mainline_throttle = false;

        for _ in 0..available_slots {
            let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() {
                let max_concurrent_per_set = config.max_concurrent_per_set;
                let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
                    |request| {
                        let set_allowed = !config.set_bulkhead_enable
                            || can_schedule_request(request, &running_per_set, max_concurrent_per_set);
                        let mainline_allowed = mainline_pressure.is_none() || Self::request_bypasses_mainline_throttle(request);
                        set_allowed && mainline_allowed
                    },
                    |request| heal_request_set_key(request).map(|_| heal_request_set_metric_label(request)),
                );
                for skipped_set in skipped_sets {
                    record_scheduler_skip(&skipped_set);
                }
                selected_request
            } else {
                queue.pop_next()
            };

            if let Some(mut request) = selected_request {
                request.options.timeout.get_or_insert(config.task_timeout);
                let task_priority = request.priority;
                let task_type_label = heal_request_type_label(&request).to_string();
                let task_set_label = heal_request_set_metric_label(&request);
                if config.set_bulkhead_enable
                    && let Some(set_key) = heal_request_set_key(&request)
                {
                    *running_per_set.entry(set_key).or_insert(0) += 1;
                }
                let replacement_resume_endpoint = replacement_recovery_anchors
                    .lock()
                    .unwrap_or_else(|poisoned| poisoned.into_inner())
                    .get(&request.id)
                    .cloned();
                let task = Arc::new(HealTask::from_replacement_recovery_request(
                    request,
                    storage.clone(),
                    replacement_resume_endpoint,
                ));
                let task_id = task.id.clone();
                active_heals_guard.insert(task_id.clone(), task.clone());
                publish_active_heal_count(&active_heals_guard);
                update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
                let active_heals_clone = active_heals.clone();
                let heal_queue_clone = heal_queue.clone();
                let completed_heals_clone = completed_heals.clone();
                let retrying_heals_clone = retrying_heals.clone();
                let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone();
                let statistics_clone = statistics.clone();
                let notify_clone = notify.clone();
                let manager_cancel_token = cancel_token.clone();
                let task_type_label_for_spawn = task_type_label.clone();
                let task_set_label_for_spawn = task_set_label.clone();
                let config_for_spawn = config.clone();

                // start heal task
                tokio::spawn(async move {
                    debug!(
                        target: "rustfs::heal::manager",
                        event = EVENT_HEAL_SCHEDULER_STATE,
                        component = LOG_COMPONENT_HEAL,
                        subsystem = LOG_SUBSYSTEM_MANAGER,
                        task_id,
                        priority = ?task_priority,
                        heal_type = %task_type_label_for_spawn,
                        set = %task_set_label_for_spawn,
                        state = "task_started",
                        "Heal scheduler task started"
                    );
                    let result = task.execute().await;
                    let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await;
                    match &result {
                        Ok(_) => {
                            debug!(
                                target: "rustfs::heal::manager",
                                event = EVENT_HEAL_SCHEDULER_STATE,
                                component = LOG_COMPONENT_HEAL,
                                subsystem = LOG_SUBSYSTEM_MANAGER,
                                task_id,
                                heal_type = %task_type_label_for_spawn,
                                set = %task_set_label_for_spawn,
                                state = "task_completed",
                                "Heal scheduler task completed"
                            );
                        }
                        Err(e) => {
                            let will_retry = retry_request.is_some();
                            if will_retry {
                                demote_to_debug_when!(task.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", {
                                    event = EVENT_HEAL_SCHEDULER_STATE,
                                    component = LOG_COMPONENT_HEAL,
                                    subsystem = LOG_SUBSYSTEM_MANAGER,
                                    task_id,
                                    heal_type = %task_type_label_for_spawn,
                                    set = %task_set_label_for_spawn,
                                    state = "task_retrying",
                                    retry_attempt = task.retry_attempts.saturating_add(1),
                                    error = %e,
                                    "Heal scheduler task retrying"
                                });
                            } else {
                                error!(
                                    target: "rustfs::heal::manager",
                                    event = EVENT_HEAL_SCHEDULER_STATE,
                                    component = LOG_COMPONENT_HEAL,
                                    subsystem = LOG_SUBSYSTEM_MANAGER,
                                    task_id,
                                    heal_type = %task_type_label_for_spawn,
                                    set = %task_set_label_for_spawn,
                                    state = "task_failed",
                                    error = %e,
                                    "Heal scheduler task failed"
                                );
                            }
                        }
                    }
                    let retry_request_for_status = retry_request.as_ref().map(|(request, _, error)| HealTaskStatus::Retrying {
                        error: error.clone(),
                        retry_attempt: request.retry_attempts,
                    });
                    let retry_request_for_queue = retry_request;
                    let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
                    if retry_request_for_queue.is_none() {
                        replacement_recovery_anchors_clone
                            .lock()
                            .unwrap_or_else(|poisoned| poisoned.into_inner())
                            .remove(&task_id);
                    }
                    let mut active_heals_guard = active_heals_clone.lock().await;
                    // Keep retry ownership continuous: status snapshots acquire
                    // these locks in the same active -> retrying order.
                    let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
                        (retry_request_for_queue.as_ref(), retry_cancel_token.as_ref())
                    {
                        let mut retrying = retrying_heals_clone.lock().await;
                        if active_heals_guard.contains_key(&task_id) {
                            retrying.insert(
                                request.id.clone(),
                                RetryingHeal {
                                    request: request.clone(),
                                    error: error.clone(),
                                    cancel_token: cancel_token.clone(),
                                },
                            );
                            #[cfg(test)]
                            pause_retry_ownership_transition(&task_id, false).await;
                        }
                        Some(retrying)
                    } else {
                        None
                    };
                    let completed_task = active_heals_guard.remove(&task_id);
                    if let Some(completed_task) = completed_task.as_ref() {
                        publish_active_heal_count(&active_heals_guard);
                        update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
                    }
                    let active_count = active_heals_guard.len();
                    drop(retrying_heals_guard.take());
                    drop(active_heals_guard);

                    if let Some(completed_task) = completed_task {
                        let completed_status = if let Some(status) = retry_request_for_status {
                            status
                        } else {
                            completed_task.get_status().await
                        };
                        let completed_progress = completed_task.get_progress().await;
                        let final_window = completed_task.get_result_items_since(None).await;
                        let completed_status_entry = CompletedHealStatus {
                            heal_type: completed_task.heal_type.clone(),
                            status: completed_status.clone(),
                            result_items: final_window.items.clone(),
                            result_items_truncated: completed_task.result_items_truncated(),
                            completed_at: SystemTime::now(),
                            seqed_items: completed_task.get_seqed_result_items().await,
                            next_seq: final_window.next_seq,
                            min_seq: final_window.min_seq,
                        };
                        let mut completed_heals_guard = completed_heals_clone.lock().await;
                        prune_completed_heal_statuses(&mut completed_heals_guard);
                        completed_heals_guard.insert(task_id.clone(), completed_status_entry);
                        // update statistics
                        let mut stats = statistics_clone.write().await;
                        match completed_status {
                            HealTaskStatus::Completed => {
                                stats.update_task_completion(true);
                                stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed);
                            }
                            HealTaskStatus::Retrying { .. } => {}
                            _ => {
                                stats.update_task_completion(false);
                            }
                        }
                        stats.update_running_tasks(usize_to_u64_saturated(active_count));
                    }

                    if let (Some((retry_request, retry_delay, retry_error)), Some(retry_cancel_token)) =
                        (retry_request_for_queue, retry_cancel_token)
                    {
                        let retry_request_id = retry_request.id.clone();
                        let retry_attempt = retry_request.retry_attempts;
                        let retry_key = PriorityHealQueue::make_dedup_key(&retry_request);
                        let retry_priority = retry_request.priority;
                        let retry_active_heals = active_heals_clone.clone();
                        let retry_heal_queue = heal_queue_clone.clone();
                        let retrying_heals_for_spawn = retrying_heals_clone.clone();
                        let retry_completed_heals = completed_heals_clone.clone();
                        let retry_notify = notify_clone.clone();
                        let retry_manager_cancel_token = manager_cancel_token.clone();
                        let retry_config = config_for_spawn.clone();
                        tokio::spawn(async move {
                            loop {
                                tokio::select! {
                                    _ = retry_cancel_token.cancelled() => {
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_QUEUE_ADMISSION,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_MANAGER,
                                            request_id = %retry_request_id,
                                            priority = ?retry_priority,
                                            retry_attempt,
                                            result = "retry_cancelled",
                                            "Heal retry admission decided"
                                        );
                                        return;
                                    }
                                    _ = retry_manager_cancel_token.cancelled() => {
                                        retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
                                        return;
                                    }
                                    _ = sleep(retry_delay) => {}
                                }

                                {
                                    let retrying_heals_guard = retrying_heals_for_spawn.lock().await;
                                    if !retrying_heals_guard.contains_key(&retry_request_id) {
                                        return;
                                    }
                                }

                                let active_duplicate = {
                                    let active_heals_guard = retry_active_heals.lock().await;
                                    active_heals_contains_dedup_key(&active_heals_guard, &retry_key)
                                };
                                if active_duplicate {
                                    retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
                                    debug!(
                                        target: "rustfs::heal::manager",
                                        event = EVENT_HEAL_QUEUE_ADMISSION,
                                        component = LOG_COMPONENT_HEAL,
                                        subsystem = LOG_SUBSYSTEM_MANAGER,
                                        request_id = %retry_request_id,
                                        priority = ?retry_priority,
                                        retry_attempt,
                                        result = "retry_merged_active_duplicate",
                                        "Heal retry admission decided"
                                    );
                                    return;
                                }

                                let mut queue = retry_heal_queue.lock().await;
                                let admission =
                                    Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry");
                                let should_notify = matches!(admission, HealAdmissionResult::Accepted)
                                    && retry_config.event_driven_scheduler_enable;
                                match admission {
                                    HealAdmissionResult::Accepted => {
                                        // Transfer ownership while holding queue -> retrying,
                                        // matching operations_snapshot's lock order.
                                        #[cfg(test)]
                                        pause_retry_ownership_transition(&retry_request_id, true).await;
                                        retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
                                        drop(queue);
                                        retry_completed_heals.lock().await.remove(&retry_request_id);
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_QUEUE_ADMISSION,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_MANAGER,
                                            request_id = %retry_request_id,
                                            priority = ?retry_priority,
                                            retry_attempt,
                                            retry_delay_ms = retry_delay.as_millis(),
                                            error = %retry_error,
                                            result = "retry_enqueued",
                                            "Heal retry admission decided"
                                        );
                                        if should_notify {
                                            retry_notify.notify_one();
                                        }
                                        return;
                                    }
                                    HealAdmissionResult::Merged => {
                                        retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
                                        drop(queue);
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_QUEUE_ADMISSION,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_MANAGER,
                                            request_id = %retry_request_id,
                                            priority = ?retry_priority,
                                            retry_attempt,
                                            result = "retry_merged_duplicate",
                                            "Heal retry admission decided"
                                        );
                                        return;
                                    }
                                    HealAdmissionResult::Full => {
                                        // admit_request_to_queue already logged the
                                        // rejection (context = "retry"); this repeats
                                        // every backoff cycle while the queue stays
                                        // full, so keep it at debug!.
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_QUEUE_ADMISSION,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_MANAGER,
                                            request_id = %retry_request_id,
                                            priority = ?retry_priority,
                                            retry_attempt,
                                            result = "retry_rejected_full",
                                            "Heal retry admission decided"
                                        );
                                    }
                                    HealAdmissionResult::Dropped(reason) => {
                                        debug!(
                                            target: "rustfs::heal::manager",
                                            event = EVENT_HEAL_QUEUE_ADMISSION,
                                            component = LOG_COMPONENT_HEAL,
                                            subsystem = LOG_SUBSYSTEM_MANAGER,
                                            request_id = %retry_request_id,
                                            priority = ?retry_priority,
                                            retry_attempt,
                                            reason = reason.as_str(),
                                            result = "retry_dropped",
                                            "Heal retry admission decided"
                                        );
                                    }
                                }
                            }
                        });
                    }
                    notify_clone.notify_one();
                });
                tasks_started += 1;
            } else {
                delayed_by_mainline_throttle = mainline_pressure.is_some();
                break;
            }
        }

        // Update statistics for all started tasks
        let mut stats = statistics.write().await;
        stats.total_tasks += tasks_started as u64;
        stats.update_running_tasks(active_heals_guard.len() as u64);
        publish_active_heal_count(&active_heals_guard);
        publish_heal_queue_length(&queue);

        if delayed_by_mainline_throttle && let Some(pressure) = mainline_pressure {
            Self::record_mainline_throttle_delay(pressure, &config);
            Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep);
        }

        // Log queue status if items remain
        if !queue.is_empty() {
            let remaining = queue.len();
            if remaining > 10 {
                info!(
                    target: "rustfs::heal::manager",
                    event = EVENT_HEAL_QUEUE_STATE,
                    component = LOG_COMPONENT_HEAL,
                    subsystem = LOG_SUBSYSTEM_MANAGER,
                    queue_len = remaining,
                    active_tasks = active_heals_guard.len(),
                    state = "backlog_high",
                    "Heal queue backlog high"
                );
            }
        }
    }
}

impl std::fmt::Debug for HealManager {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("HealManager")
            .field("config", &"<config>")
            .field("state", &"<state>")
            .field("active_heals_count", &"<active_heals>")
            .field("queue_length", &"<queue>")
            .finish()
    }
}

fn heal_request_set_key(request: &HealRequest) -> Option<String> {
    match &request.heal_type {
        HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
        HealType::Object { .. } => heal_options_set_key(&request.options),
        _ => None,
    }
}

fn heal_options_set_key(options: &HealOptions) -> Option<String> {
    match (options.pool_index, options.set_index) {
        (Some(pool), Some(set)) => Some(format!("pool_{pool}_set_{set}")),
        _ => None,
    }
}

fn heal_request_type_label(request: &HealRequest) -> &'static str {
    match &request.heal_type {
        HealType::Cluster => "cluster",
        HealType::Object { .. } => "object",
        HealType::Bucket { .. } => "bucket",
        HealType::Prefix { .. } => "prefix",
        HealType::ErasureSet { .. } => "erasure_set",
        HealType::Metadata { .. } => "metadata",
        HealType::MRF { .. } => "mrf",
        HealType::ECDecode { .. } => "ec_decode",
    }
}

fn heal_request_set_metric_label(request: &HealRequest) -> String {
    heal_request_set_key(request).unwrap_or_else(|| match (request.options.pool_index, request.options.set_index) {
        (Some(pool), Some(set)) => format!("pool_{pool}_set_{set}"),
        _ => "global".to_string(),
    })
}

fn record_scheduler_skip(set_label: &str) {
    counter!(
        "rustfs_heal_scheduler_skip_total",
        "reason" => "set_limit".to_string(),
        "set" => set_label.to_string()
    )
    .increment(1);
}

fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTask>>, task: &HealTask) {
    let type_label = task.metric_type_label();
    let set_label = task.metric_set_label();
    let count = active_heals
        .values()
        .filter(|active_task| active_task.metric_type_label() == type_label && active_task.metric_set_label() == set_label)
        .count();

    gauge!(
        "rustfs_heal_task_running",
        "type" => type_label.to_string(),
        "set" => set_label
    )
    .set(count as f64);
}

fn running_heal_set_counts(active_heals: &HashMap<String, Arc<HealTask>>) -> HashMap<String, usize> {
    let mut running = HashMap::new();
    for task in active_heals.values() {
        if let Some(set_key) = heal_request_set_key_for_task(task) {
            *running.entry(set_key).or_insert(0) += 1;
        }
    }
    running
}

fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
    match &task.heal_type {
        HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
        HealType::Object { .. } => heal_options_set_key(&task.options),
        _ => None,
    }
}

fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, CompletedHealStatus>) {
    let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
        return;
    };

    completed_heals.retain(|_, completed| {
        completed
            .completed_at
            .duration_since(SystemTime::UNIX_EPOCH)
            .map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
            .unwrap_or(false)
    });
}

fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap<String, usize>, max_concurrent_per_set: usize) -> bool {
    match heal_request_set_key(request) {
        Some(set_key) => running_per_set.get(&set_key).copied().unwrap_or(0) < max_concurrent_per_set,
        None => true,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::heal::EcstoreError;
    use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity};
    use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
    use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType};
    use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
    use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot};
    use rustfs_madmin::heal_commands::HealResultItem;
    use std::sync::Mutex as StdMutex;
    use tempfile::TempDir;

    use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};

    #[tokio::test]
    async fn auto_replacement_path_requires_a_non_root_mount() {
        let temp = TempDir::new().expect("temporary replacement root should be created");
        let ready = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("replacement endpoint should parse");
        let missing = Endpoint::try_from(temp.path().join("missing").to_string_lossy().as_ref())
            .expect("missing replacement endpoint should parse");

        let ready_disk = new_disk(
            &ready,
            &DiskOption {
                cleanup: false,
                health_check: false,
            },
        )
        .await
        .expect("temporary disk should initialize");
        assert!(
            !super::super::replacement_readiness::auto_replacement_target_ready(&ready_disk, std::slice::from_ref(&ready_disk),)
                .await
        );
        assert!(
            matches!(
                new_disk(
                    &missing,
                    &DiskOption {
                        cleanup: false,
                        health_check: false,
                    },
                )
                .await,
                Err(DiskError::VolumeNotFound)
            ),
            "a missing replacement path must be rejected before admission"
        );
    }

    #[derive(Debug)]
    struct FixedWorkloadProvider {
        class: WorkloadClass,
        active: usize,
        limit: usize,
        state: AdmissionState,
    }

    impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
        fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
            WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(self.class, self.state).with_counts(
                Some(self.active),
                None,
                Some(self.limit),
            )])
        }
    }

    async fn process_manager_queue_once(manager: &HealManager) {
        HealManager::process_heal_queue(HealQueueContext {
            heal_queue: &manager.heal_queue,
            active_heals: &manager.active_heals,
            completed_heals: &manager.completed_heals,
            retrying_heals: &manager.retrying_heals,
            replacement_recovery_anchors: &manager.replacement_recovery_anchors,
            config: &manager.config,
            statistics: &manager.statistics,
            storage: &manager.storage,
            notify: &manager.notify,
            cancel_token: &manager.cancel_token,
            workload_provider: &manager.workload_provider,
        })
        .await;
    }

    struct MockStorage;

    #[async_trait::async_trait]
    impl HealStorageAPI for MockStorage {
        async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
            Ok(None)
        }

        async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result<Option<Vec<u8>>> {
            Ok(None)
        }

        async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> {
            Ok(())
        }

        async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<()> {
            Ok(())
        }

        async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result<bool> {
            Ok(true)
        }

        async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result<Vec<u8>> {
            Ok(Vec::new())
        }

        async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result<crate::heal::storage::DiskStatus> {
            Ok(crate::heal::storage::DiskStatus::Ok)
        }

        async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> {
            Ok(())
        }

        async fn get_bucket_info(&self, _bucket: &str) -> Result<Option<BucketInfo>> {
            Ok(None)
        }

        async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> {
            Ok(())
        }

        async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
            if let Some(hook) = manager_recovery_test_hook() {
                *hook.listed.lock().expect("manager recovery listed lock should not poison") = true;
            }
            Ok(Vec::new())
        }

        async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
            Ok(bucket == "retry-transition")
        }

        async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result<Option<u64>> {
            Ok(None)
        }

        async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result<Option<String>> {
            Ok(None)
        }

        async fn heal_object(
            &self,
            bucket: &str,
            _object: &str,
            _version_id: Option<&str>,
            _opts: &HealOpts,
        ) -> Result<(HealResultItem, Option<Error>)> {
            if let Some(hook) = manager_recovery_test_hook() {
                *hook
                    .heal_object_calls
                    .lock()
                    .expect("manager recovery object call lock should not poison") += 1;
            }
            if bucket == "retry-transition" {
                return Ok((
                    HealResultItem::default(),
                    Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
                        bucket.to_string(),
                        "object".to_string(),
                    ))),
                ));
            }
            Ok((HealResultItem::default(), None))
        }

        async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result<HealResultItem> {
            if let Some(hook) = manager_recovery_test_hook() {
                *hook
                    .bucket_heal_calls
                    .lock()
                    .expect("manager recovery bucket call lock should not poison") += 1;
            }
            Ok(HealResultItem::default())
        }

        async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
            if let Some(hook) = manager_recovery_test_hook() {
                *hook
                    .global_format_calls
                    .lock()
                    .expect("manager recovery global format call lock should not poison") += 1;
            }
            Ok((HealResultItem::default(), None))
        }

        async fn heal_replacement_format(
            &self,
            _dry_run: bool,
            _pool_index: usize,
            _set_index: usize,
            _targets: &[String],
        ) -> Result<(HealResultItem, Option<Error>)> {
            if let Some(hook) = manager_recovery_test_hook() {
                *hook
                    .replacement_format_calls
                    .lock()
                    .expect("manager recovery replacement format call lock should not poison") += 1;
            }
            Ok((HealResultItem::default(), None))
        }

        async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result<Vec<crate::heal::storage::HealListItem>> {
            Ok(Vec::new())
        }

        async fn list_objects_for_heal_page(
            &self,
            _bucket: &str,
            _prefix: &str,
            _continuation_token: Option<&str>,
            _include_lifecycle_object_info: bool,
        ) -> Result<(Vec<crate::heal::storage::HealListItem>, Option<String>, bool)> {
            Ok((Vec::new(), None, false))
        }

        async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result<DiskStore> {
            Err(Error::other("not implemented in tests"))
        }

        async fn get_replacement_resume_disk(
            &self,
            _set_disk_id: &str,
            _task_id: &str,
            _excluded_targets: &[String],
        ) -> Result<crate::heal::storage::ReplacementResumeDisk> {
            let Some(hook) = manager_recovery_test_hook() else {
                return Err(Error::other("not implemented in tests"));
            };
            Ok(crate::heal::storage::ReplacementResumeDisk::Existing(
                hook.replacement_resume_disk.clone(),
            ))
        }
    }

    struct ManagerRecoveryTestHook {
        replacement_resume_disk: DiskStore,
        listed: StdMutex<bool>,
        global_format_calls: StdMutex<u32>,
        replacement_format_calls: StdMutex<u32>,
        bucket_heal_calls: StdMutex<u32>,
        heal_object_calls: StdMutex<u32>,
    }

    static MANAGER_RECOVERY_TEST_HOOK: LazyLock<StdMutex<Option<Arc<ManagerRecoveryTestHook>>>> =
        LazyLock::new(|| StdMutex::new(None));

    struct ManagerRecoveryTestHookGuard;

    impl ManagerRecoveryTestHook {
        fn install(replacement_resume_disk: DiskStore) -> (Arc<Self>, ManagerRecoveryTestHookGuard) {
            let hook = Arc::new(Self {
                replacement_resume_disk,
                listed: StdMutex::new(false),
                global_format_calls: StdMutex::new(0),
                replacement_format_calls: StdMutex::new(0),
                bucket_heal_calls: StdMutex::new(0),
                heal_object_calls: StdMutex::new(0),
            });
            let previous = MANAGER_RECOVERY_TEST_HOOK
                .lock()
                .expect("manager recovery hook lock should not poison")
                .replace(hook.clone());
            assert!(previous.is_none(), "manager recovery hook already installed");
            (hook, ManagerRecoveryTestHookGuard)
        }
    }

    impl Drop for ManagerRecoveryTestHookGuard {
        fn drop(&mut self) {
            *MANAGER_RECOVERY_TEST_HOOK
                .lock()
                .expect("manager recovery hook lock should not poison") = None;
        }
    }

    fn manager_recovery_test_hook() -> Option<Arc<ManagerRecoveryTestHook>> {
        MANAGER_RECOVERY_TEST_HOOK
            .lock()
            .expect("manager recovery hook lock should not poison")
            .clone()
    }

    async fn make_manager_resume_disk(temp: &TempDir, name: &str) -> DiskStore {
        let disk_path = temp.path().join(name);
        std::fs::create_dir_all(&disk_path).expect("manager recovery disk directory should be created");
        let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).expect("manager recovery endpoint should parse");
        let disk = new_disk(
            &endpoint,
            &DiskOption {
                cleanup: false,
                health_check: false,
            },
        )
        .await
        .expect("manager recovery disk should initialize");
        let metadata_volume = disk.make_volume(super::super::RUSTFS_META_BUCKET).await;
        assert!(
            matches!(metadata_volume, Ok(()) | Err(DiskError::VolumeExists)),
            "manager recovery metadata volume should exist: {metadata_volume:?}"
        );
        disk
    }

    fn bucket_request(bucket: &str, priority: HealPriority, source: HealRequestSource) -> HealRequest {
        let mut request = HealRequest::new(
            HealType::Bucket {
                bucket: bucket.to_string(),
            },
            HealOptions::default(),
            priority,
        );
        request.source = source;
        request
    }

    #[test]
    fn test_push_displacing_lower_priority_actually_enqueues_new_request() {
        // Regression for the release-build defect where the enqueue side effect lived inside
        // `debug_assert_eq!(self.push(request), ...)` and was compiled out under
        // `cargo test --release` (debug_assertions off), silently dropping the displacing
        // high-priority request while still having evicted a queued item.
        //
        // Must run with --release to expose the original bug.
        let mut queue = PriorityHealQueue::new();

        let low = bucket_request("victim-bucket", HealPriority::Low, HealRequestSource::Scanner);
        assert_eq!(queue.push(low), QueuePushOutcome::Accepted);
        assert_eq!(queue.len(), 1);

        let high = bucket_request("admin-bucket", HealPriority::High, HealRequestSource::Admin);
        let high_id = high.id.clone();
        assert!(queue.can_displace_lower_priority(high.priority));

        let displaced = queue
            .push_displacing_lower_priority(high)
            .expect("a lower-priority item should have been displaced");
        assert_eq!(displaced.priority, HealPriority::Low);

        // The displacing high-priority request must actually be enqueued (pre-fix under
        // --release, len() is 0 because self.push(request) was elided with debug_assert_eq!).
        assert_eq!(queue.len(), 1, "displacing request must remain enqueued");
        let admitted = queue.pop_next().expect("displacing high-priority request must be enqueued");
        assert_eq!(admitted.priority, HealPriority::High);
        assert_eq!(admitted.id, high_id);
        assert_eq!(queue.len(), 0);
    }

    #[test]
    fn test_priority_queue_ordering() {
        let mut queue = PriorityHealQueue::new();

        // Add requests with different priorities
        let low_req = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket1".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );

        let normal_req = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket2".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let high_req = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket3".to_string(),
            },
            HealOptions::default(),
            HealPriority::High,
        );

        let urgent_req = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket4".to_string(),
            },
            HealOptions::default(),
            HealPriority::Urgent,
        );

        // Add in random order: low, high, normal, urgent
        assert_eq!(queue.push(low_req), QueuePushOutcome::Accepted);
        assert_eq!(queue.push(high_req), QueuePushOutcome::Accepted);
        assert_eq!(queue.push(normal_req), QueuePushOutcome::Accepted);
        assert_eq!(queue.push(urgent_req), QueuePushOutcome::Accepted);

        assert_eq!(queue.len(), 4);

        // Should pop in priority order: urgent, high, normal, low
        let popped1 = queue.pop().unwrap();
        assert_eq!(popped1.priority, HealPriority::Urgent);

        let popped2 = queue.pop().unwrap();
        assert_eq!(popped2.priority, HealPriority::High);

        let popped3 = queue.pop().unwrap();
        assert_eq!(popped3.priority, HealPriority::Normal);

        let popped4 = queue.pop().unwrap();
        assert_eq!(popped4.priority, HealPriority::Low);

        assert_eq!(queue.len(), 0);
    }

    #[test]
    fn test_priority_queue_fifo_same_priority() {
        let mut queue = PriorityHealQueue::new();

        // Add multiple requests with same priority
        let req1 = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket1".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let req2 = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket2".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let req3 = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket3".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let id1 = req1.id.clone();
        let id2 = req2.id.clone();
        let id3 = req3.id.clone();

        assert_eq!(queue.push(req1), QueuePushOutcome::Accepted);
        assert_eq!(queue.push(req2), QueuePushOutcome::Accepted);
        assert_eq!(queue.push(req3), QueuePushOutcome::Accepted);

        // Should maintain FIFO order for same priority
        let popped1 = queue.pop().unwrap();
        assert_eq!(popped1.id, id1);

        let popped2 = queue.pop().unwrap();
        assert_eq!(popped2.id, id2);

        let popped3 = queue.pop().unwrap();
        assert_eq!(popped3.id, id3);
    }

    #[test]
    fn test_priority_queue_deduplication() {
        let mut queue = PriorityHealQueue::new();

        let req1 = HealRequest::new(
            HealType::Object {
                bucket: "bucket1".to_string(),
                object: "object1".to_string(),
                version_id: None,
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let req2 = HealRequest::new(
            HealType::Object {
                bucket: "bucket1".to_string(),
                object: "object1".to_string(),
                version_id: None,
            },
            HealOptions::default(),
            HealPriority::High,
        );

        // First request should be added
        assert_eq!(queue.push(req1), QueuePushOutcome::Accepted);
        assert_eq!(queue.len(), 1);

        // Second request with same object should be rejected (duplicate)
        assert_eq!(queue.push(req2), QueuePushOutcome::Merged);
        assert_eq!(queue.len(), 1);
    }

    #[test]
    fn test_priority_queue_contains_erasure_set() {
        let mut queue = PriorityHealQueue::new();

        let req = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec!["bucket1".to_string()],
                set_disk_id: "pool_0_set_1".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        assert_eq!(queue.push(req), QueuePushOutcome::Accepted);
        assert!(queue.contains_erasure_set("pool_0_set_1"));
        assert!(!queue.contains_erasure_set("pool_0_set_2"));
    }

    #[test]
    fn test_priority_queue_dedup_key_generation() {
        // Test different heal types generate different keys
        let obj_req = HealRequest::new(
            HealType::Object {
                bucket: "bucket1".to_string(),
                object: "object1".to_string(),
                version_id: None,
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let bucket_req = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket1".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let erasure_req = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec!["bucket1".to_string()],
                set_disk_id: "pool_0_set_1".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let obj_key = PriorityHealQueue::make_dedup_key(&obj_req);
        let bucket_key = PriorityHealQueue::make_dedup_key(&bucket_req);
        let erasure_key = PriorityHealQueue::make_dedup_key(&erasure_req);

        // All keys should be different
        assert_ne!(obj_key, bucket_key);
        assert_ne!(obj_key, erasure_key);
        assert_ne!(bucket_key, erasure_key);

        assert!(obj_key.starts_with("object:"));
        assert!(bucket_key.starts_with("bucket:"));
        assert!(erasure_key.starts_with("erasure_set:"));
    }

    #[test]
    fn test_priority_queue_mixed_priorities_and_types() {
        let mut queue = PriorityHealQueue::new();

        // Add various requests
        let requests = vec![
            (
                HealType::Object {
                    bucket: "b1".to_string(),
                    object: "o1".to_string(),
                    version_id: None,
                },
                HealPriority::Low,
            ),
            (
                HealType::Bucket {
                    bucket: "b2".to_string(),
                },
                HealPriority::Urgent,
            ),
            (
                HealType::ErasureSet {
                    buckets: vec!["b3".to_string()],
                    set_disk_id: "pool_0_set_1".to_string(),
                },
                HealPriority::Normal,
            ),
            (
                HealType::Object {
                    bucket: "b4".to_string(),
                    object: "o4".to_string(),
                    version_id: None,
                },
                HealPriority::High,
            ),
        ];

        for (heal_type, priority) in requests {
            let req = HealRequest::new(heal_type, HealOptions::default(), priority);
            let outcome = queue.push(req);
            assert_eq!(outcome, QueuePushOutcome::Accepted);
        }

        assert_eq!(queue.len(), 4);

        // Check they come out in priority order
        let priorities: Vec<HealPriority> = (0..4).filter_map(|_| queue.pop().map(|r| r.priority)).collect();

        assert_eq!(
            priorities,
            vec![
                HealPriority::Urgent,
                HealPriority::High,
                HealPriority::Normal,
                HealPriority::Low,
            ]
        );
    }

    #[test]
    fn test_priority_queue_stats() {
        let mut queue = PriorityHealQueue::new();

        // Add requests with different priorities
        for _ in 0..3 {
            assert_eq!(
                queue.push(HealRequest::new(
                    HealType::Bucket {
                        bucket: format!("bucket-low-{}", queue.len()),
                    },
                    HealOptions::default(),
                    HealPriority::Low,
                )),
                QueuePushOutcome::Accepted
            );
        }

        for _ in 0..2 {
            assert_eq!(
                queue.push(HealRequest::new(
                    HealType::Bucket {
                        bucket: format!("bucket-normal-{}", queue.len()),
                    },
                    HealOptions::default(),
                    HealPriority::Normal,
                )),
                QueuePushOutcome::Accepted
            );
        }

        assert_eq!(
            queue.push(HealRequest::new(
                HealType::Bucket {
                    bucket: "bucket-high".to_string(),
                },
                HealOptions::default(),
                HealPriority::High,
            )),
            QueuePushOutcome::Accepted
        );

        let stats = queue.get_priority_stats();

        assert_eq!(*stats.get(&HealPriority::Low).unwrap_or(&0), 3);
        assert_eq!(*stats.get(&HealPriority::Normal).unwrap_or(&0), 2);
        assert_eq!(*stats.get(&HealPriority::High).unwrap_or(&0), 1);
        assert_eq!(*stats.get(&HealPriority::Urgent).unwrap_or(&0), 0);
    }

    #[test]
    fn test_priority_queue_is_empty() {
        let mut queue = PriorityHealQueue::new();

        assert!(queue.is_empty());

        assert_eq!(
            queue.push(HealRequest::new(
                HealType::Bucket {
                    bucket: "test".to_string(),
                },
                HealOptions::default(),
                HealPriority::Normal,
            )),
            QueuePushOutcome::Accepted
        );

        assert!(!queue.is_empty());

        queue.pop();

        assert!(queue.is_empty());
    }

    #[test]
    fn test_priority_queue_pop_runnable_skips_blocked_erasure_set() {
        let mut queue = PriorityHealQueue::new();

        let blocked = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec!["bucket-a".to_string()],
                set_disk_id: "pool_0_set_1".to_string(),
            },
            HealOptions::default(),
            HealPriority::Urgent,
        );
        let runnable = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec!["bucket-b".to_string()],
                set_disk_id: "pool_0_set_2".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        assert_eq!(queue.push(blocked), QueuePushOutcome::Accepted);
        assert_eq!(queue.push(runnable), QueuePushOutcome::Accepted);

        let mut running = HashMap::new();
        running.insert("pool_0_set_1".to_string(), 1);

        let popped = queue
            .pop_runnable(|request| can_schedule_request(request, &running, 1))
            .expect("should find runnable request");

        assert!(matches!(
            popped.heal_type,
            HealType::ErasureSet { ref set_disk_id, .. } if set_disk_id == "pool_0_set_2"
        ));
    }

    #[test]
    fn test_can_schedule_request_respects_per_set_limit() {
        let request = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec!["bucket".to_string()],
                set_disk_id: "pool_0_set_1".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        let mut running = HashMap::new();
        running.insert("pool_0_set_1".to_string(), 1);

        assert!(!can_schedule_request(&request, &running, 1));
        assert!(can_schedule_request(&request, &running, 2));
    }

    #[test]
    fn test_can_schedule_scoped_object_request_respects_per_set_limit() {
        let options = HealOptions {
            pool_index: Some(0),
            set_index: Some(1),
            ..Default::default()
        };
        let request = HealRequest::new(
            HealType::Object {
                bucket: "bucket".to_string(),
                object: "object".to_string(),
                version_id: None,
            },
            options,
            HealPriority::Normal,
        );

        let mut running = HashMap::new();
        running.insert("pool_0_set_1".to_string(), 1);

        assert!(!can_schedule_request(&request, &running, 1));
        assert!(can_schedule_request(&request, &running, 2));
    }

    #[tokio::test]
    async fn test_submit_heal_request_returns_merged_for_duplicate() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let request = HealRequest::new(
            HealType::Object {
                bucket: "bucket".to_string(),
                object: "object".to_string(),
                version_id: None,
            },
            HealOptions::default(),
            HealPriority::Low,
        );

        assert_eq!(
            manager
                .submit_heal_request(request.clone())
                .await
                .expect("first request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(request)
                .await
                .expect("duplicate request should produce admission result"),
            HealAdmissionResult::Merged
        );
    }

    #[tokio::test]
    async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);
        let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        original.source = HealRequestSource::Admin;
        let original_id = original.id.clone();
        let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        duplicate.source = HealRequestSource::Admin;
        let duplicate_id = duplicate.id.clone();

        let accepted = manager
            .submit_heal_request_with_receipt(original)
            .await
            .expect("first request should be accepted");
        let merged = manager
            .submit_heal_request_with_receipt(duplicate)
            .await
            .expect("duplicate request should merge");

        assert_eq!(accepted.result, HealAdmissionResult::Accepted);
        assert_eq!(accepted.task_id, original_id);
        assert_eq!(merged.result, HealAdmissionResult::Merged);
        assert_eq!(merged.task_id, original_id);
        assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
    }

    #[tokio::test]
    async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = Arc::new(HealManager::new(storage.clone(), None));
        let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        original.source = HealRequestSource::Admin;
        let original_id = original.id.clone();
        assert_eq!(
            manager
                .submit_heal_request(original)
                .await
                .expect("original request should be accepted"),
            HealAdmissionResult::Accepted
        );

        let mut duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        duplicate.source = HealRequestSource::Admin;
        let hook = Arc::new(DuplicateAdmissionTestHook {
            request_id: duplicate.id.clone(),
            active_lock_reached: Notify::new(),
            active_lock_release: Notify::new(),
        });
        *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = Some(hook.clone());

        let duplicate_manager = manager.clone();
        let duplicate_task = tokio::spawn(async move { duplicate_manager.submit_heal_request_with_receipt(duplicate).await });
        tokio::time::timeout(Duration::from_secs(1), hook.active_lock_reached.notified())
            .await
            .expect("duplicate admission should reach the active lock hook");

        let transition_manager = manager.clone();
        let transition_storage = storage.clone();
        let (attempting_active_tx, attempting_active_rx) = tokio::sync::oneshot::channel();
        let (active_acquired_tx, mut active_acquired_rx) = tokio::sync::oneshot::channel();
        let transition = tokio::spawn(async move {
            let _ = attempting_active_tx.send(());
            let mut active = transition_manager.active_heals.lock().await;
            let _ = active_acquired_tx.send(());
            let mut queue = transition_manager.heal_queue.lock().await;
            let request = queue.pop_next().expect("original request should remain queued");
            let task = Arc::new(HealTask::from_request(request, transition_storage));
            active.insert(task.id.clone(), task);
        });
        tokio::time::timeout(Duration::from_secs(1), attempting_active_rx)
            .await
            .expect("transition should attempt the active lock")
            .expect("transition attempt signal should be delivered");
        assert!(matches!(
            active_acquired_rx.try_recv(),
            Err(tokio::sync::oneshot::error::TryRecvError::Empty)
        ));

        hook.active_lock_release.notify_one();
        let receipt = tokio::time::timeout(Duration::from_secs(1), duplicate_task)
            .await
            .expect("duplicate admission should not hang")
            .expect("duplicate task should join")
            .expect("duplicate admission should succeed");
        tokio::time::timeout(Duration::from_secs(1), transition)
            .await
            .expect("queue to active transition should not hang")
            .expect("queue to active transition should join");
        *DUPLICATE_ADMISSION_TEST_HOOK.lock().await = None;

        assert_eq!(receipt.result, HealAdmissionResult::Merged);
        assert_eq!(receipt.task_id, original_id);
        assert_eq!(manager.get_queue_length().await, 0);
        assert_eq!(manager.get_active_task_count().await, 1);
    }

    #[tokio::test]
    async fn test_submit_heal_request_returns_merged_for_active_duplicate() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage.clone(), None);
        let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        let active_task = Arc::new(HealTask::from_request(active_request, storage));
        manager.active_heals.lock().await.insert(active_task.id.clone(), active_task);

        let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);

        assert_eq!(
            manager
                .submit_heal_request(duplicate_request)
                .await
                .expect("active duplicate should produce admission result"),
            HealAdmissionResult::Merged
        );
        assert_eq!(manager.get_queue_length().await, 0);
    }

    #[tokio::test]
    async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage.clone(), None);
        let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        let active_task = Arc::new(HealTask::from_request(active_request, storage));
        let active_task_id = active_task.id.clone();
        manager.active_heals.lock().await.insert(active_task_id.clone(), active_task);

        let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        let duplicate_task_id = duplicate_request.id.clone();

        assert_eq!(
            manager
                .submit_heal_request(duplicate_request)
                .await
                .expect("active duplicate should produce admission result"),
            HealAdmissionResult::Merged
        );
        assert_eq!(
            manager
                .get_task_status_for_path("bucket/object", &duplicate_task_id)
                .await
                .expect("duplicate token should query merged active task"),
            HealTaskStatus::Pending
        );

        manager
            .cancel_task(&duplicate_task_id)
            .await
            .expect("duplicate token should cancel merged active task");

        assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
        assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. })));
    }

    #[tokio::test]
    async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);
        let original_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        let original_task_id = original_request.id.clone();
        let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        let duplicate_task_id = duplicate_request.id.clone();

        assert_eq!(
            manager
                .submit_heal_request(original_request)
                .await
                .expect("original request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(duplicate_request)
                .await
                .expect("queued duplicate should produce admission result"),
            HealAdmissionResult::Merged
        );
        assert_eq!(
            manager
                .get_task_status_for_path("bucket/object", &duplicate_task_id)
                .await
                .expect("duplicate token should query merged queued task"),
            HealTaskStatus::Pending
        );

        manager
            .cancel_task(&duplicate_task_id)
            .await
            .expect("duplicate token should cancel merged queued request");

        assert!(matches!(
            manager.get_task_status(&original_task_id).await,
            Err(Error::TaskNotFound { .. })
        ));
    }

    #[test]
    fn test_retry_request_for_recoverable_lock_timeout() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage);
        let result = Err(Error::TaskExecutionFailed {
            message: "Failed to heal object bucket/object: Lock acquisition timeout".to_string(),
        });

        let (retry_request, retry_delay, retry_error) =
            retry_request_for_result(&task, &result).expect("lock timeout should be retryable");

        assert_eq!(retry_request.id, task.id);
        assert_eq!(retry_request.retry_attempts, 1);
        assert_eq!(retry_request.priority, task.priority);
        assert!(retry_delay > Duration::ZERO);
        assert!(retry_error.contains("Lock acquisition timeout"));
    }

    #[tokio::test]
    async fn retry_request_for_result_preserves_remaining_timeout_budget() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
        request.options.timeout = Some(Duration::from_secs(60));
        let task = HealTask::from_request(request, storage);
        let result = task.execute().await;

        let (retry_request, _, _) = retry_request_for_result_with_budget(&task, &result)
            .await
            .expect("read quorum failure should retain the unused timeout budget");
        let remaining = retry_request
            .options
            .timeout
            .expect("configured timeout should remain present");
        assert!(remaining < Duration::from_secs(60));
        assert!(remaining > Duration::from_secs(59));
    }

    #[test]
    fn test_retry_request_for_incomplete_heal_rename() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage);
        let result = Err(Error::TaskExecutionFailed {
            message: "Failed to heal object bucket/object: heal rename incomplete: 1 of 2 targets committed".to_string(),
        });

        let (retry_request, retry_delay, retry_error) =
            retry_request_for_result(&task, &result).expect("incomplete target rename should be retryable");

        assert_eq!(retry_request.id, task.id);
        assert_eq!(retry_request.retry_attempts, 1);
        assert!(retry_delay > Duration::ZERO);
        assert!(retry_error.contains("heal rename incomplete"));
    }

    #[test]
    fn test_retry_request_for_typed_read_quorum_error() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage);
        let result = Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
            "bucket".to_string(),
            "object".to_string(),
        )));

        let (retry_request, retry_delay, retry_error) =
            retry_request_for_result(&task, &result).expect("typed read quorum should be retryable");

        assert_eq!(retry_request.id, task.id);
        assert_eq!(retry_request.retry_attempts, 1);
        assert!(retry_delay > Duration::ZERO);
        assert!(retry_error.contains("Storage resources are insufficient"));
    }

    #[test]
    fn test_retry_request_for_durable_replacement_retry_signal() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let task = HealTask::from_request(
            HealRequest::new(
                HealType::ErasureSet {
                    buckets: vec!["bucket".to_string()],
                    set_disk_id: "pool_0_set_0".to_string(),
                },
                HealOptions::default(),
                HealPriority::Low,
            ),
            storage,
        );
        let result = Err(Error::transient_skip("Replacement erasure set heal incomplete; retry scheduled"));

        let (retry_request, retry_delay, retry_error) =
            retry_request_for_result(&task, &result).expect("typed replacement retry signal must be scheduled");

        assert_eq!(retry_request.id, task.id);
        assert_eq!(retry_request.retry_attempts, 1);
        assert!(retry_delay > Duration::ZERO);
        assert!(retry_error.contains("Replacement erasure set heal incomplete"));
    }

    #[test]
    fn durable_replacement_recovery_re_admits_only_the_matching_generation() {
        let task_id = "replacement-generation";
        let mut state = crate::heal::resume::ResumeState::new(
            task_id.to_string(),
            "erasure_set".to_string(),
            "pool_0_set_0".to_string(),
            vec!["bucket".to_string()],
        );
        state.replacement_generation = Some(task_id.to_string());
        state.replacement_phase = ReplacementPhase::Intent;
        state.replacement_targets = vec!["replacement-a".to_string()];
        assert!(!durable_replacement_recovery_is_due(&state, task_id));

        state.retry_count = state.max_retries;
        assert!(durable_replacement_recovery_is_due(&state, task_id));

        state.completed = true;
        state.retry_count = 0;
        state.replacement_phase = ReplacementPhase::Verified;
        assert!(
            durable_replacement_recovery_is_due(&state, task_id),
            "verified terminal cleanup must be re-admitted without re-running recovery"
        );

        state.replacement_phase = ReplacementPhase::CleanupPending;
        assert!(
            durable_replacement_recovery_is_due(&state, task_id),
            "cleanup-pending terminal cleanup must be periodically re-admitted"
        );

        state.completed = false;
        state.replacement_generation = Some("another-generation".to_string());
        assert!(
            !durable_replacement_recovery_is_due(&state, task_id),
            "a task must not adopt another generation's durable intent"
        );

        state.replacement_generation = Some(task_id.to_string());
        state.replacement_targets.clear();
        assert!(
            !durable_replacement_recovery_is_due(&state, task_id),
            "a durable retry without a target is not safe to re-admit"
        );

        state.replacement_targets = vec!["replacement-a".to_string()];
        state.replacement_phase = ReplacementPhase::Verified;
        assert!(
            !durable_replacement_recovery_is_due(&state, task_id),
            "a task must not adopt another generation's terminal cleanup"
        );
    }

    #[test]
    fn replacement_recovery_blocker_is_set_scoped() {
        let manager = HealManager::new(Arc::new(MockStorage), None);

        manager.block_replacement_recovery_set("pool_0_set_0");

        assert!(manager.replacement_recovery_set_is_blocked("pool_0_set_0"));
        assert!(!manager.replacement_recovery_set_is_blocked("pool_0_set_1"));
    }

    #[test]
    fn replacement_recovery_blocks_only_confirmed_conflicts() {
        assert!(crate::heal::resume::replacement_recovery_error_requires_block(
            &Error::TaskExecutionFailed {
                message: "replacement recovery conflict: proof mismatch".to_string(),
            }
        ));
        assert!(crate::heal::resume::replacement_recovery_error_requires_block(
            &Error::TaskExecutionFailed {
                message: "replacement recovery corruption: malformed legacy intent".to_string(),
            }
        ));
        assert!(!crate::heal::resume::replacement_recovery_error_requires_block(&Error::Disk(
            DiskError::Timeout
        )));
        assert!(!crate::heal::resume::replacement_recovery_error_requires_block(
            &Error::TaskExecutionFailed {
                message: "Failed to list replacement recovery records: temporary I/O error".to_string(),
            }
        ));
    }

    #[test]
    fn replacement_recovery_discovery_unformatted_is_quiet_only_for_deferred_endpoint() {
        let error = Error::Disk(DiskError::UnformattedDisk);
        let deferred = HashSet::from(["endpoint-a".to_string()]);

        assert!(replacement_discovery_error_is_expected_for_deferred_endpoint(
            &error,
            "endpoint-a",
            &deferred
        ));
        assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint(
            &error,
            "endpoint-b",
            &deferred
        ));
        assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint(
            &Error::Disk(DiskError::Timeout),
            "endpoint-a",
            &deferred
        ));
    }

    #[test]
    fn replacement_recovery_retry_barrier_requires_all_set_records_to_validate() {
        let mut blocked = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]);
        let retry_succeeded = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]);
        let retry_failed = HashSet::from(["pool_0_set_0".to_string()]);

        unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed);

        assert!(
            blocked.contains("pool_0_set_0"),
            "one failed disk record must keep the whole replacement set blocked"
        );
        assert!(
            !blocked.contains("pool_0_set_1"),
            "a blocked set may resume only after every retried record validates"
        );
    }

    #[tokio::test]
    async fn scheduler_completes_cleanup_pending_recovery_from_manager_anchor() {
        let temp = TempDir::new().expect("temporary manager recovery directory should be created");
        let anchor = make_manager_resume_disk(&temp, "anchor").await;
        let task_id = ResumeUtils::generate_task_id();
        let target = "replacement-a".to_string();
        let identity = ReplacementTargetIdentity {
            endpoint: target.clone(),
            canonical_path: "/replacement/replacement-a".to_string(),
            physical_device_ids: vec!["replacement-a".to_string()],
            filesystem_identity: "identity-replacement-a".to_string(),
        };
        let resume_manager = ResumeManager::new_replacement_intent(
            anchor.clone(),
            task_id.clone(),
            "pool_0_set_0".to_string(),
            vec!["bucket-a".to_string()],
            vec![target.clone()],
            vec![identity],
        )
        .await
        .expect("cleanup-pending replacement state should persist on the survivor anchor");
        resume_manager
            .mark_replacement_completed_and_verified()
            .await
            .expect("completion proof should persist before cleanup");
        resume_manager
            .mark_replacement_cleanup_pending()
            .await
            .expect("cleanup-pending state should persist before restart");
        CheckpointManager::new(anchor.clone(), task_id.clone())
            .await
            .expect("checkpoint fixture should persist");

        let (hook, _hook_guard) = ManagerRecoveryTestHook::install(anchor.clone());
        let storage = Arc::new(MockStorage);
        let manager = HealManager::new(storage.clone(), None);
        let mut request = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec!["bucket-a".to_string()],
                set_disk_id: "pool_0_set_0".to_string(),
            },
            HealOptions {
                pool_index: Some(0),
                set_index: Some(0),
                ..HealOptions::default()
            },
            HealPriority::Low,
        );
        request.id = task_id.clone();
        request.source = HealRequestSource::AutoHeal;
        request.heal_endpoints = vec![target];
        assert_eq!(
            manager
                .submit_heal_request(request)
                .await
                .expect("durable recovery request should be admitted"),
            HealAdmissionResult::Accepted
        );
        manager
            .replacement_recovery_anchors
            .lock()
            .expect("replacement recovery anchor lock should not poison")
            .insert(task_id.clone(), anchor.endpoint().to_string());

        process_manager_queue_once(&manager).await;
        tokio::time::timeout(Duration::from_secs(2), async {
            loop {
                let resume_removed = !ResumeManager::has_resume_state(&anchor, &task_id).await;
                let checkpoint_removed = !CheckpointManager::has_checkpoint(&anchor, &task_id).await;
                let anchor_removed = !manager
                    .replacement_recovery_anchors
                    .lock()
                    .expect("replacement recovery anchor lock should not poison")
                    .contains_key(&task_id);
                if resume_removed && checkpoint_removed && anchor_removed {
                    break;
                }
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("cleanup-pending recovery should finish through the manager scheduler");

        assert!(
            !*hook.listed.lock().expect("manager recovery listed lock should not poison"),
            "cleanup-pending recovery must not list buckets or restart object healing"
        );
        assert_eq!(
            *hook
                .global_format_calls
                .lock()
                .expect("manager recovery global format call lock should not poison"),
            0
        );
        assert_eq!(
            *hook
                .replacement_format_calls
                .lock()
                .expect("manager recovery replacement format call lock should not poison"),
            0,
            "manager-resumed terminal cleanup must not format replacement targets"
        );
        assert_eq!(
            *hook
                .bucket_heal_calls
                .lock()
                .expect("manager recovery bucket call lock should not poison"),
            0
        );
        assert_eq!(
            *hook
                .heal_object_calls
                .lock()
                .expect("manager recovery object call lock should not poison"),
            0
        );
    }

    #[test]
    fn test_retry_request_for_scoped_slowdown_preserves_scope() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let task = HealTask::from_request(
            HealRequest::new(
                HealType::Object {
                    bucket: "bucket".to_string(),
                    object: "object".to_string(),
                    version_id: None,
                },
                HealOptions {
                    pool_index: Some(0),
                    set_index: Some(1),
                    ..Default::default()
                },
                HealPriority::Normal,
            ),
            storage,
        );
        let result = Err(Error::Storage(EcstoreError::SlowDown));

        let (retry_request, retry_delay, _) =
            retry_request_for_result(&task, &result).expect("SlowDown should defer scoped heal");

        assert_eq!(retry_request.options.pool_index, Some(0));
        assert_eq!(retry_request.options.set_index, Some(1));
        assert_eq!(retry_request.retry_attempts, 1);
        assert!(retry_delay > Duration::ZERO);
    }

    #[test]
    fn test_retry_request_for_typed_not_found_error_is_not_retryable() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let task = HealTask::from_request(HealRequest::object("bucket".to_string(), "object".to_string(), None), storage);
        let result = Err(Error::Storage(EcstoreError::ObjectNotFound("bucket".to_string(), "object".to_string())));

        assert!(retry_request_for_result(&task, &result).is_none());
    }

    #[test]
    fn test_retry_request_for_recoverable_error_stops_at_limit() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
        request.retry_attempts = MAX_RECOVERABLE_HEAL_RETRIES;
        let task = HealTask::from_request(request, storage);
        let result = Err(Error::TaskExecutionFailed {
            message: "Remote lock RPC timed out".to_string(),
        });

        assert!(retry_request_for_result(&task, &result).is_none());
    }

    #[tokio::test]
    async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
        let result = Err(task
            .record_batch_failure(BatchHealFailure {
                scope: "bucket:bucket".to_string(),
                failed: 1,
                retryable: 1,
                permanent: 0,
                first_object: "object".to_string(),
                first_error: "Lock acquisition timeout".to_string(),
            })
            .await);

        assert!(retry_request_for_result(&task, &result).is_none());
    }

    #[test]
    fn test_heal_type_matches_path_normalizes_prefix_trailing_slash() {
        let heal_type = HealType::Prefix {
            bucket: "bucket".to_string(),
            prefix: "logs/".to_string(),
        };

        assert!(heal_type_matches_path(&heal_type, "bucket"));
        assert!(heal_type_matches_path(&heal_type, "bucket/logs"));
        assert!(heal_type_matches_path(&heal_type, "bucket/logs/"));
    }

    #[test]
    fn test_heal_type_matches_path_normalizes_object_trailing_slash() {
        let heal_type = HealType::Object {
            bucket: "bucket".to_string(),
            object: "object/".to_string(),
            version_id: None,
        };

        assert!(heal_type_matches_path(&heal_type, "bucket/object"));
        assert!(heal_type_matches_path(&heal_type, "bucket/object/"));
    }

    async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> CancellationToken {
        let task_id = request.id.clone();
        let cancel_token = CancellationToken::new();
        manager.retrying_heals.lock().await.insert(
            task_id.clone(),
            RetryingHeal {
                request: request.clone(),
                error: "Lock acquisition timeout".to_string(),
                cancel_token: cancel_token.clone(),
            },
        );
        manager.completed_heals.lock().await.insert(
            task_id,
            CompletedHealStatus {
                heal_type: request.heal_type,
                status: HealTaskStatus::Retrying {
                    error: "Lock acquisition timeout".to_string(),
                    retry_attempt: request.retry_attempts,
                },
                result_items: Vec::new(),
                result_items_truncated: false,
                seqed_items: Vec::new(),
                next_seq: 0,
                min_seq: 0,
                completed_at: SystemTime::now(),
            },
        );
        cancel_token
    }

    #[tokio::test]
    async fn test_cancel_task_cancels_retrying_backoff() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);
        let mut request = HealRequest::bucket("bucket".to_string());
        request.retry_attempts = 1;
        let task_id = request.id.clone();
        let cancel_token = insert_retrying_request(&manager, request).await;

        assert!(matches!(
            manager
                .get_task_status(&task_id)
                .await
                .expect("retrying task should be queryable"),
            HealTaskStatus::Retrying { .. }
        ));

        manager
            .cancel_task(&task_id)
            .await
            .expect("retrying task should be cancellable by token");

        assert!(cancel_token.is_cancelled());
        assert!(manager.retrying_heals.lock().await.get(&task_id).is_none());
        assert!(matches!(manager.get_task_status(&task_id).await, Err(Error::TaskNotFound { .. })));
    }

    #[tokio::test]
    async fn test_cancel_tasks_for_path_cancels_retrying_backoff() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);
        let mut request = HealRequest::bucket("bucket".to_string());
        request.retry_attempts = 1;
        let task_id = request.id.clone();
        let cancel_token = insert_retrying_request(&manager, request).await;

        assert_eq!(
            manager
                .cancel_tasks_for_path("bucket")
                .await
                .expect("retrying task should be cancellable by path"),
            1
        );

        assert!(cancel_token.is_cancelled());
        assert!(manager.retrying_heals.lock().await.get(&task_id).is_none());
        assert!(matches!(manager.get_task_status(&task_id).await, Err(Error::TaskNotFound { .. })));
    }

    #[tokio::test]
    async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
        let cluster_request_id = cluster_request.id.clone();
        let bucket_request = HealRequest::bucket("bucket".to_string());
        let bucket_request_id = bucket_request.id.clone();

        manager
            .submit_heal_request(cluster_request)
            .await
            .expect("cluster request should be accepted");
        manager
            .submit_heal_request(bucket_request)
            .await
            .expect("bucket request should be accepted");

        assert_eq!(
            manager
                .cancel_tasks_for_path("")
                .await
                .expect("root path should cancel queued cluster task"),
            1
        );
        assert!(matches!(
            manager.get_task_status(&cluster_request_id).await,
            Err(Error::TaskNotFound { .. })
        ));
        assert_eq!(
            manager
                .get_task_status(&bucket_request_id)
                .await
                .expect("bucket request should not match root path"),
            HealTaskStatus::Pending
        );
    }

    #[tokio::test]
    async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage.clone(), None);

        let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
        let cluster_request_id = cluster_request.id.clone();
        let bucket_request = HealRequest::bucket("bucket".to_string());
        let bucket_request_id = bucket_request.id.clone();

        manager.active_heals.lock().await.insert(
            cluster_request_id.clone(),
            Arc::new(HealTask::from_request(cluster_request, storage.clone())),
        );
        manager
            .active_heals
            .lock()
            .await
            .insert(bucket_request_id.clone(), Arc::new(HealTask::from_request(bucket_request, storage)));

        assert_eq!(
            manager
                .cancel_tasks_for_path("")
                .await
                .expect("root path should cancel active cluster task"),
            1
        );
        assert!(manager.active_heals.lock().await.get(&cluster_request_id).is_none());
        assert!(manager.active_heals.lock().await.get(&bucket_request_id).is_some());
    }

    #[tokio::test]
    async fn test_cancel_tasks_for_empty_path_cancels_retrying_cluster_only() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let mut cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
        cluster_request.retry_attempts = 1;
        let cluster_request_id = cluster_request.id.clone();
        let cluster_cancel_token = insert_retrying_request(&manager, cluster_request).await;

        let mut bucket_request = HealRequest::bucket("bucket".to_string());
        bucket_request.retry_attempts = 1;
        let bucket_request_id = bucket_request.id.clone();
        let bucket_cancel_token = insert_retrying_request(&manager, bucket_request).await;

        assert_eq!(
            manager
                .cancel_tasks_for_path("")
                .await
                .expect("root path should cancel retrying cluster task"),
            1
        );
        assert!(cluster_cancel_token.is_cancelled());
        assert!(!bucket_cancel_token.is_cancelled());
        assert!(manager.retrying_heals.lock().await.get(&cluster_request_id).is_none());
        assert!(manager.retrying_heals.lock().await.get(&bucket_request_id).is_some());
    }

    #[test]
    fn test_heal_type_matches_path_accepts_legacy_root() {
        assert!(heal_type_matches_path(&HealType::Cluster, LEGACY_ROOT_HEAL_PATH));
        assert!(!heal_type_matches_path(
            &HealType::Bucket {
                bucket: "bucket".to_string(),
            },
            LEGACY_ROOT_HEAL_PATH,
        ));
    }

    #[tokio::test]
    async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);
        let mut original_request = HealRequest::bucket("bucket".to_string());
        original_request.retry_attempts = 1;
        let original_task_id = original_request.id.clone();
        let cancel_token = insert_retrying_request(&manager, original_request).await;

        let duplicate_request = HealRequest::bucket("bucket".to_string());
        let duplicate_task_id = duplicate_request.id.clone();

        assert_eq!(
            manager
                .submit_heal_request(duplicate_request)
                .await
                .expect("retrying duplicate should produce admission result"),
            HealAdmissionResult::Merged
        );
        assert!(matches!(
            manager
                .get_task_status_for_path("bucket", &duplicate_task_id)
                .await
                .expect("duplicate token should query merged retrying task"),
            HealTaskStatus::Retrying { .. }
        ));

        manager
            .cancel_task(&duplicate_task_id)
            .await
            .expect("duplicate token should cancel merged retrying task");

        assert!(cancel_token.is_cancelled());
        assert!(manager.retrying_heals.lock().await.get(&original_task_id).is_none());
    }

    #[tokio::test]
    async fn test_get_task_status_reports_pending_for_queued_request() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let request = HealRequest::bucket("bucket".to_string());
        let request_id = request.id.clone();

        assert_eq!(
            manager
                .submit_heal_request(request)
                .await
                .expect("request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .get_task_status(&request_id)
                .await
                .expect("queued request should have status"),
            HealTaskStatus::Pending
        );
    }

    #[tokio::test]
    async fn test_operations_snapshot_counts_queue_by_source_and_priority() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let mut scanner_request = HealRequest::new(
            HealType::Object {
                bucket: "bucket-a".to_string(),
                object: "object-a".to_string(),
                version_id: None,
            },
            HealOptions::default(),
            HealPriority::Low,
        );
        scanner_request.source = HealRequestSource::Scanner;

        let mut admin_request = HealRequest::bucket("bucket-b".to_string());
        admin_request.priority = HealPriority::High;
        admin_request.source = HealRequestSource::Admin;

        let mut auto_request = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec!["bucket-c".to_string()],
                set_disk_id: "0-0".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );
        auto_request.source = HealRequestSource::AutoHeal;

        manager
            .submit_heal_request(scanner_request)
            .await
            .expect("scanner request should be accepted");
        manager
            .submit_heal_request(admin_request)
            .await
            .expect("admin request should be accepted");
        manager
            .submit_heal_request(auto_request)
            .await
            .expect("auto request should be accepted");

        let snapshot = manager.operations_snapshot().await;

        assert_eq!(snapshot.queue_length, 3);
        assert_eq!(snapshot.active_tasks, 0);
        assert_eq!(snapshot.queued_by_priority.low, 1);
        assert_eq!(snapshot.queued_by_priority.normal, 1);
        assert_eq!(snapshot.queued_by_priority.high, 1);
        assert_eq!(snapshot.queued_by_priority.urgent, 0);
        assert_eq!(snapshot.queued_by_source.scanner, 1);
        assert_eq!(snapshot.queued_by_source.admin, 1);
        assert_eq!(snapshot.queued_by_source.auto_heal, 1);
        assert_eq!(snapshot.queued_by_source.internal, 0);
    }

    // HS-06 (backlog#1870): overlap policy + forceStart semantics.
    fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        HealManager::new(
            storage,
            Some(HealConfig {
                overlap_policy: policy,
                ..Default::default()
            }),
        )
    }

    fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest {
        let mut request = HealRequest::new(
            HealType::Prefix {
                bucket: bucket.to_string(),
                prefix: prefix.to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );
        request.source = HealRequestSource::Admin;
        request
    }

    async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String {
        let task = Arc::new(HealTask::from_request(request, manager.storage.clone()));
        let task_id = task.id.clone();
        manager.active_heals.lock().await.insert(task_id.clone(), task);
        task_id
    }

    #[tokio::test]
    async fn overlap_policy_minio_error_rejects_same_and_containing_paths() {
        let manager = manager_with_policy(HealOverlapPolicy::MinioError);
        insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;

        // Same target: typed AlreadyRunning.
        let same = manager
            .submit_heal_request(admin_prefix_request("bucket-a", "logs/"))
            .await
            .expect("admission must decide");
        assert_eq!(
            same,
            HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning),
            "an identical target must reject with already-running"
        );

        // Contained path: typed OverlappingPaths.
        let nested = manager
            .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/"))
            .await
            .expect("admission must decide");
        assert_eq!(
            nested,
            HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths),
            "a path inside the active task's path must reject with overlapping-paths"
        );

        // Containing path (bucket-wide vs nested active): also overlapping.
        let wide = manager
            .submit_heal_request(admin_prefix_request("bucket-a", ""))
            .await
            .expect("admission must decide");
        assert_eq!(
            wide,
            HealAdmissionResult::Dropped(HealAdmissionDropReason::OverlappingPaths),
            "a bucket-wide start overlapping a nested active heal must reject"
        );

        // Disjoint bucket: unaffected.
        let disjoint = manager
            .submit_heal_request(admin_prefix_request("bucket-b", "logs/"))
            .await
            .expect("admission must decide");
        assert_eq!(disjoint, HealAdmissionResult::Accepted);
    }

    #[tokio::test]
    async fn overlap_policy_default_merge_keeps_today_semantics() {
        let manager = manager_with_policy(HealOverlapPolicy::Merge);
        insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;

        // Different-dedup-key overlap still merges under the default policy:
        // the nested path dedups to its own key but nothing rejects it.
        let nested = manager
            .submit_heal_request(admin_prefix_request("bucket-a", "logs/app/"))
            .await
            .expect("admission must decide");
        assert_eq!(nested, HealAdmissionResult::Accepted, "default policy must not reject overlaps");

        // Non-admin sources never get overlap rejections even under minio_error.
        let manager = manager_with_policy(HealOverlapPolicy::MinioError);
        insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;
        let mut scanner_request = admin_prefix_request("bucket-a", "logs/app/");
        scanner_request.source = HealRequestSource::Scanner;
        let admitted = manager
            .submit_heal_request(scanner_request)
            .await
            .expect("admission must decide");
        assert_eq!(admitted, HealAdmissionResult::Accepted, "scanner sources must never be overlap-rejected");
    }

    #[tokio::test]
    async fn admin_force_start_cancels_overlapping_active_task_first() {
        let manager = manager_with_policy(HealOverlapPolicy::Merge);
        let old_id = insert_active_task(&manager, admin_prefix_request("bucket-a", "logs/")).await;

        let mut replacement = admin_prefix_request("bucket-a", "logs/");
        replacement.force_start = true;
        let receipt = manager
            .submit_heal_request_with_receipt(replacement)
            .await
            .expect("force-start submission must decide");

        assert!(receipt.result.is_admitted(), "the new task must be admitted (Accepted or Merged)");
        let old_task_gone = {
            let active_heals = manager.active_heals.lock().await;
            !active_heals.contains_key(&old_id)
        };
        assert!(
            old_task_gone,
            "the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
        );
        assert!(
            matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })),
            "a cancelled task must no longer resolve as an active heal"
        );
    }

    #[tokio::test]
    async fn test_operations_snapshot_counts_active_by_source_and_priority() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let mut request = HealRequest::bucket("bucket-a".to_string());
        request.priority = HealPriority::High;
        request.source = HealRequestSource::Admin;
        let task = Arc::new(HealTask::from_request(request, manager.storage.clone()));
        let task_id = task.id.clone();

        manager.active_heals.lock().await.insert(task_id, task);

        let snapshot = manager.operations_snapshot().await;

        assert_eq!(snapshot.queue_length, 0);
        assert_eq!(snapshot.active_tasks, 1);
        assert_eq!(snapshot.active_by_priority.high, 1);
        assert_eq!(snapshot.active_by_source.admin, 1);
        assert_eq!(snapshot.active_by_source.scanner, 0);
    }

    #[tokio::test]
    async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);
        let mut request = HealRequest::bucket("bucket-retry".to_string());
        request.priority = HealPriority::Urgent;
        request.source = HealRequestSource::Admin;
        let task_id = request.id.clone();
        manager.retrying_heals.lock().await.insert(
            task_id,
            RetryingHeal {
                request,
                error: "transient".to_string(),
                cancel_token: CancellationToken::new(),
            },
        );

        let snapshot = manager.operations_snapshot().await;

        assert_eq!(snapshot.queue_length, 0);
        assert_eq!(snapshot.active_tasks, 0);
        assert_eq!(snapshot.retrying_tasks, 1);
        assert_eq!(snapshot.retrying_by_priority.urgent, 1);
        assert_eq!(snapshot.retrying_by_source.admin, 1);
    }

    #[tokio::test]
    async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = Arc::new(HealManager::new(storage, None));
        {
            let mut config = manager.config.write().await;
            config.enable_auto_heal = false;
            config.heal_interval = Duration::from_millis(10);
            config.event_driven_scheduler_enable = true;
        }
        manager.start().await.expect("manager should start");

        let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
        let task_id = request.id.clone();
        let hook = Arc::new(RetryOwnershipTestHook {
            task_id: task_id.clone(),
            active_to_retrying_reached: Notify::new(),
            active_to_retrying_release: Notify::new(),
            retrying_to_queue_reached: Notify::new(),
            retrying_to_queue_release: Notify::new(),
        });
        *RETRY_OWNERSHIP_TEST_HOOK.lock().await = Some(hook.clone());

        manager
            .submit_heal_request(request)
            .await
            .expect("retry test request should be accepted");
        tokio::time::timeout(Duration::from_secs(2), hook.active_to_retrying_reached.notified())
            .await
            .expect("active to retrying transition should be reached");

        let snapshot_manager = manager.clone();
        let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await });
        assert!(
            tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(),
            "snapshot must wait while active and retrying ownership locks are held"
        );
        hook.active_to_retrying_release.notify_one();
        let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot)
            .await
            .expect("snapshot should resume after active to retrying handoff")
            .expect("snapshot task should complete");
        assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1);
        assert_eq!(snapshot.retrying_tasks, 1);

        tokio::time::timeout(Duration::from_secs(5), hook.retrying_to_queue_reached.notified())
            .await
            .expect("retrying to queue transition should be reached");
        let snapshot_manager = manager.clone();
        let mut snapshot = tokio::spawn(async move { snapshot_manager.operations_snapshot().await });
        assert!(
            tokio::time::timeout(Duration::from_millis(20), &mut snapshot).await.is_err(),
            "snapshot must wait while queue ownership is transferred"
        );
        hook.retrying_to_queue_release.notify_one();
        // The scheduler may immediately execute the retried request again;
        // leave a permit so a second test-only active handoff cannot stall it.
        hook.active_to_retrying_release.notify_one();
        let snapshot = tokio::time::timeout(Duration::from_secs(1), snapshot)
            .await
            .expect("snapshot should resume after retrying to queue handoff")
            .expect("snapshot task should complete");
        assert_eq!(snapshot.active_tasks + snapshot.queue_length + snapshot.retrying_tasks, 1);

        *RETRY_OWNERSHIP_TEST_HOOK.lock().await = None;
        hook.active_to_retrying_release.notify_one();
        hook.retrying_to_queue_release.notify_one();
        tokio::time::timeout(Duration::from_secs(1), manager.stop())
            .await
            .expect("manager stop should not stall")
            .expect("manager should stop");
    }

    #[tokio::test]
    async fn test_active_progress_snapshot_sums_active_task_progress() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let first = Arc::new(HealTask::from_request(
            HealRequest::bucket("bucket-a".to_string()),
            manager.storage.clone(),
        ));
        {
            let mut progress = first.progress.write().await;
            progress.start_time = Some(SystemTime::now() - Duration::from_secs(20));
            progress.set_total_baseline(12, 8192);
            progress.update_progress(7, 3, 1, 4096);
        }

        let second = Arc::new(HealTask::from_request(
            HealRequest::bucket("bucket-b".to_string()),
            manager.storage.clone(),
        ));
        {
            let mut progress = second.progress.write().await;
            progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
            progress.set_total_baseline(8, 4096);
            progress.update_progress(11, 5, 2, 2048);
        }

        manager.active_heals.lock().await.insert(first.id.clone(), first);
        manager.active_heals.lock().await.insert(second.id.clone(), second);

        let progress = manager
            .active_progress_snapshot()
            .await
            .expect("active progress should exist");

        assert_eq!(progress.objects_scanned, 18);
        assert_eq!(progress.objects_healed, 8);
        assert_eq!(progress.objects_failed, 3);
        assert_eq!(progress.objects_total_count, 20);
        assert_eq!(progress.objects_total_size, 12288);
        assert_eq!(progress.bytes_processed, 6144);
        assert!((progress.progress_percentage - 50.0).abs() < 0.001);
        assert!(progress.estimated_completion_time.is_some());
    }

    #[tokio::test]
    async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        manager
            .submit_heal_request(HealRequest::bucket("bucket".to_string()))
            .await
            .expect("request should be accepted");

        assert!(matches!(
            manager.get_task_status_for_path("bucket", "wrong-token").await,
            Err(Error::InvalidClientToken)
        ));
    }

    #[tokio::test]
    async fn test_get_task_status_for_path_rejects_token_from_other_active_path() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let bucket_request = HealRequest::bucket("bucket".to_string());
        let other_request = HealRequest::bucket("other".to_string());
        let other_request_id = other_request.id.clone();

        manager
            .submit_heal_request(bucket_request)
            .await
            .expect("bucket request should be accepted");
        manager
            .submit_heal_request(other_request)
            .await
            .expect("other request should be accepted");

        assert!(matches!(
            manager.get_task_status_for_path("bucket", &other_request_id).await,
            Err(Error::InvalidClientToken)
        ));
    }

    #[tokio::test]
    async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let request = HealRequest::bucket("bucket".to_string());
        let request_id = request.id.clone();

        manager
            .submit_heal_request(request)
            .await
            .expect("request should be accepted");

        assert!(matches!(
            manager.get_task_status_for_path("other", &request_id).await,
            Err(Error::TaskNotFound { .. })
        ));
    }

    #[tokio::test]
    async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        assert!(matches!(
            manager.get_task_status_for_path("bucket", "old-token").await,
            Err(Error::TaskNotFound { .. })
        ));
    }

    #[tokio::test]
    async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let request = HealRequest::bucket("bucket".to_string());
        let request_id = request.id.clone();

        manager
            .submit_heal_request(request)
            .await
            .expect("request should be accepted");

        assert!(matches!(
            manager.get_task_status_for_path("", &request_id).await,
            Err(Error::TaskNotFound { .. })
        ));
        assert!(matches!(
            manager.get_task_status_for_path("", "wrong-token").await,
            Err(Error::TaskNotFound { .. })
        ));
    }

    #[tokio::test]
    async fn test_get_task_report_queries_queued_task_by_token_without_path() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let request = HealRequest::new(
            HealType::ErasureSet {
                buckets: vec![],
                set_disk_id: "pool_0_set_1".to_string(),
            },
            HealOptions::default(),
            HealPriority::High,
        );
        let request_id = request.id.clone();

        manager
            .submit_heal_request(request)
            .await
            .expect("request should be accepted");

        let report = manager
            .get_task_report(&request_id)
            .await
            .expect("queued task should be queryable by token");

        assert_eq!(report.status, HealTaskStatus::Pending);
        assert!(report.result_items.is_empty());
    }

    #[tokio::test]
    async fn test_get_task_status_reads_recent_completed_status() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        manager.completed_heals.lock().await.insert(
            "completed-token".to_string(),
            CompletedHealStatus {
                heal_type: HealType::Bucket {
                    bucket: "bucket".to_string(),
                },
                status: HealTaskStatus::Completed,
                result_items: Vec::new(),
                result_items_truncated: false,
                seqed_items: Vec::new(),
                next_seq: 0,
                min_seq: 0,
                completed_at: SystemTime::now(),
            },
        );

        assert_eq!(
            manager
                .get_task_status_for_path("bucket", "completed-token")
                .await
                .expect("recent completed task should be queryable"),
            HealTaskStatus::Completed
        );
    }

    #[tokio::test]
    async fn test_get_task_report_for_path_reads_completed_items() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        manager.completed_heals.lock().await.insert(
            "completed-token".to_string(),
            CompletedHealStatus {
                heal_type: HealType::Object {
                    bucket: "bucket".to_string(),
                    object: "object".to_string(),
                    version_id: None,
                },
                status: HealTaskStatus::Completed,
                result_items: vec![HealResultItem {
                    bucket: "bucket".to_string(),
                    object: "object".to_string(),
                    object_size: 1024,
                    ..Default::default()
                }],
                result_items_truncated: true,
                seqed_items: Vec::new(),
                next_seq: 0,
                min_seq: 0,
                completed_at: SystemTime::now(),
            },
        );

        let report = manager
            .get_task_report_for_path("bucket/object", "completed-token")
            .await
            .expect("recent completed task report should be queryable");
        assert!(report.result_items_truncated);

        assert_eq!(report.status, HealTaskStatus::Completed);
        assert_eq!(report.result_items.len(), 1);
        assert_eq!(report.result_items[0].object_size, 1024);
    }

    #[tokio::test]
    async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        manager
            .submit_heal_request(HealRequest::bucket("bucket".to_string()))
            .await
            .expect("request should be accepted");

        assert!(matches!(
            manager.get_task_report_for_path("", "wrong-token").await,
            Err(Error::TaskNotFound { .. })
        ));
    }

    #[tokio::test]
    async fn test_cancel_task_removes_queued_request() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let request = HealRequest::bucket("bucket".to_string());
        let request_id = request.id.clone();

        manager
            .submit_heal_request(request)
            .await
            .expect("request should be accepted");
        manager
            .cancel_task(&request_id)
            .await
            .expect("queued request should be cancelled");

        assert!(matches!(manager.get_task_status(&request_id).await, Err(Error::TaskNotFound { .. })));
    }

    #[tokio::test]
    async fn test_cancel_tasks_for_path_removes_matching_queued_requests() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(storage, None);

        let bucket_request = HealRequest::bucket("bucket".to_string());
        let bucket_request_id = bucket_request.id.clone();
        let other_request = HealRequest::bucket("other".to_string());
        let other_request_id = other_request.id.clone();

        manager
            .submit_heal_request(bucket_request)
            .await
            .expect("bucket request should be accepted");
        manager
            .submit_heal_request(other_request)
            .await
            .expect("other request should be accepted");

        assert_eq!(
            manager
                .cancel_tasks_for_path("bucket")
                .await
                .expect("matching request should be cancelled"),
            1
        );
        assert!(matches!(
            manager.get_task_status(&bucket_request_id).await,
            Err(Error::TaskNotFound { .. })
        ));
        assert_eq!(
            manager
                .get_task_status(&other_request_id)
                .await
                .expect("unmatched request should remain queued"),
            HealTaskStatus::Pending
        );
    }

    #[tokio::test]
    async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 1,
                ..HealConfig::default()
            }),
        );

        let request = HealRequest::new(
            HealType::Object {
                bucket: "bucket".to_string(),
                object: "object".to_string(),
                version_id: None,
            },
            HealOptions::default(),
            HealPriority::Low,
        );

        assert_eq!(
            manager
                .submit_heal_request(request.clone())
                .await
                .expect("first request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(request)
                .await
                .expect("duplicate request should merge even when queue is full"),
            HealAdmissionResult::Merged
        );
    }

    #[tokio::test]
    async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 1,
                low_priority_drop_when_full: true,
                ..HealConfig::default()
            }),
        );

        let accepted = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket-a".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );
        let dropped = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket-b".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );

        assert_eq!(
            manager
                .submit_heal_request(accepted)
                .await
                .expect("first request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(dropped)
                .await
                .expect("low priority request should be dropped with explicit admission result"),
            HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
        );
    }

    #[tokio::test]
    async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 1,
                ..HealConfig::default()
            }),
        );

        let accepted = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket-a".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );
        let full = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket-b".to_string(),
            },
            HealOptions::default(),
            HealPriority::Normal,
        );

        assert_eq!(
            manager
                .submit_heal_request(accepted)
                .await
                .expect("first request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(full)
                .await
                .expect("normal priority request should surface full admission"),
            HealAdmissionResult::Full
        );
    }

    #[tokio::test]
    async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 1,
                ..HealConfig::default()
            }),
        );

        let low = HealRequest::new(
            HealType::Bucket {
                bucket: "background-bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );
        let low_id = low.id.clone();
        let high = HealRequest::new(
            HealType::Bucket {
                bucket: "manual-bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::High,
        );
        let high_id = high.id.clone();

        assert_eq!(
            manager
                .submit_heal_request(low)
                .await
                .expect("low priority request should be accepted first"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(high)
                .await
                .expect("high priority request should be admitted by displacing lower priority work"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(manager.get_queue_length().await, 1);
        assert!(matches!(manager.get_task_status(&low_id).await, Err(Error::TaskNotFound { .. })));
        assert_eq!(
            manager
                .get_task_status(&high_id)
                .await
                .expect("high priority request should remain queued"),
            HealTaskStatus::Pending
        );
    }

    #[tokio::test]
    async fn test_submit_heal_request_drops_read_repair_under_pressure() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 10,
                ..HealConfig::default()
            }),
        );

        for index in 0..8 {
            assert_eq!(
                manager
                    .submit_heal_request(bucket_request(
                        &format!("queued-{index}"),
                        HealPriority::Normal,
                        HealRequestSource::Internal,
                    ))
                    .await
                    .expect("seed request should be accepted"),
                HealAdmissionResult::Accepted
            );
        }

        let admission = manager
            .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair))
            .await
            .expect("read repair admission should return a result");

        assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
        assert_eq!(manager.get_queue_length().await, 8);
    }

    #[tokio::test]
    async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 10,
                ..HealConfig::default()
            }),
        );

        for index in 0..8 {
            assert_eq!(
                manager
                    .submit_heal_request(bucket_request(
                        &format!("queued-{index}"),
                        HealPriority::Normal,
                        HealRequestSource::Internal,
                    ))
                    .await
                    .expect("seed request should be accepted"),
                HealAdmissionResult::Accepted
            );
        }

        let admission = manager
            .submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner))
            .await
            .expect("scanner admission should return a result");

        assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
        assert_eq!(manager.get_queue_length().await, 8);
    }

    #[tokio::test]
    async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 10,
                ..HealConfig::default()
            }),
        );

        for index in 0..8 {
            assert_eq!(
                manager
                    .submit_heal_request(bucket_request(
                        &format!("queued-{index}"),
                        HealPriority::Normal,
                        HealRequestSource::Internal,
                    ))
                    .await
                    .expect("seed request should be accepted"),
                HealAdmissionResult::Accepted
            );
        }

        let admission = manager
            .submit_heal_request(bucket_request("admin", HealPriority::High, HealRequestSource::Admin))
            .await
            .expect("admin admission should return a result");

        assert_eq!(admission, HealAdmissionResult::Accepted);
        assert_eq!(manager.get_queue_length().await, 9);
    }

    #[tokio::test]
    async fn test_mainline_throttle_delays_background_heal_start() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider {
            class: WorkloadClass::ForegroundRead,
            active: 8,
            limit: 10,
            state: AdmissionState::Open,
        });
        let manager = HealManager::new_with_workload_provider(
            storage,
            Some(HealConfig {
                max_concurrent_heals: 1,
                mainline_throttle_enable: true,
                mainline_read_utilization_high_percent: 80,
                mainline_write_utilization_high_percent: 80,
                mainline_max_sleep: Duration::from_millis(1),
                ..HealConfig::default()
            }),
            Some(provider),
        );

        manager
            .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair))
            .await
            .expect("read repair request should be queued");

        process_manager_queue_once(&manager).await;

        assert_eq!(manager.get_queue_length().await, 1);
        assert_eq!(manager.get_active_task_count().await, 0);
    }

    #[tokio::test]
    async fn test_mainline_throttle_delays_background_heal_start_under_write_pressure() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider {
            class: WorkloadClass::ForegroundWrite,
            active: 9,
            limit: 10,
            state: AdmissionState::Open,
        });
        let manager = HealManager::new_with_workload_provider(
            storage,
            Some(HealConfig {
                max_concurrent_heals: 1,
                mainline_throttle_enable: true,
                mainline_read_utilization_high_percent: 80,
                mainline_write_utilization_high_percent: 80,
                mainline_max_sleep: Duration::from_millis(1),
                ..HealConfig::default()
            }),
            Some(provider),
        );

        manager
            .submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair))
            .await
            .expect("read repair request should be queued");

        process_manager_queue_once(&manager).await;

        assert_eq!(manager.get_queue_length().await, 1);
        assert_eq!(manager.get_active_task_count().await, 0);
    }

    #[tokio::test]
    async fn test_mainline_throttle_allows_admin_high_start() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider {
            class: WorkloadClass::ForegroundRead,
            active: 10,
            limit: 10,
            state: AdmissionState::Saturated,
        });
        let manager = HealManager::new_with_workload_provider(
            storage,
            Some(HealConfig {
                max_concurrent_heals: 1,
                mainline_throttle_enable: true,
                mainline_read_utilization_high_percent: 80,
                mainline_write_utilization_high_percent: 80,
                mainline_max_sleep: Duration::from_millis(1),
                ..HealConfig::default()
            }),
            Some(provider),
        );

        manager
            .submit_heal_request(bucket_request("admin", HealPriority::High, HealRequestSource::Admin))
            .await
            .expect("admin request should be queued");

        process_manager_queue_once(&manager).await;

        assert_eq!(manager.get_queue_length().await, 0);
    }

    #[tokio::test]
    async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                max_concurrent_heals: 1,
                task_timeout: Duration::ZERO,
                ..HealConfig::default()
            }),
        );

        let mut defaulted = bucket_request("defaulted-timeout", HealPriority::Normal, HealRequestSource::Admin);
        defaulted.options.timeout = None;
        let defaulted_id = defaulted.id.clone();
        manager
            .submit_heal_request(defaulted)
            .await
            .expect("request without timeout should be queued");
        process_manager_queue_once(&manager).await;
        let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async {
            loop {
                if let Ok(status @ HealTaskStatus::Timeout) = manager.get_task_status(&defaulted_id).await {
                    break status;
                }
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("configured timeout should finish the task");
        assert_eq!(defaulted_status, HealTaskStatus::Timeout);
        assert!(manager.retrying_heals.lock().await.get(&defaulted_id).is_none());

        let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin);
        explicit.options.timeout = Some(Duration::from_secs(60));
        let explicit_id = explicit.id.clone();
        manager
            .submit_heal_request(explicit)
            .await
            .expect("request with explicit timeout should be queued");
        process_manager_queue_once(&manager).await;
        let explicit_status = tokio::time::timeout(Duration::from_secs(1), async {
            loop {
                if let Ok(status @ HealTaskStatus::Failed { .. }) = manager.get_task_status(&explicit_id).await {
                    break status;
                }
                tokio::task::yield_now().await;
            }
        })
        .await
        .expect("explicit timeout request should finish without using the zero default");
        assert!(matches!(explicit_status, HealTaskStatus::Failed { .. }));
    }

    #[tokio::test]
    async fn test_force_start_bypasses_duplicate_and_full_admission() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 1,
                low_priority_drop_when_full: true,
                ..HealConfig::default()
            }),
        );

        let normal = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );
        let mut forced_duplicate = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );
        forced_duplicate.force_start = true;

        let subsequent_duplicate = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );

        assert_eq!(
            manager
                .submit_heal_request(normal)
                .await
                .expect("first request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(forced_duplicate)
                .await
                .expect("force start should bypass duplicate/full policy"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(subsequent_duplicate)
                .await
                .expect("subsequent non-force duplicate should be merged"),
            HealAdmissionResult::Merged
        );
    }

    #[tokio::test]
    async fn test_force_start_marks_dedup_key_for_future_duplicates() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let manager = HealManager::new(
            storage,
            Some(HealConfig {
                queue_size: 1,
                ..HealConfig::default()
            }),
        );

        let normal = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );
        let mut forced = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );
        forced.force_start = true;
        let duplicate = HealRequest::new(
            HealType::Bucket {
                bucket: "bucket".to_string(),
            },
            HealOptions::default(),
            HealPriority::Low,
        );

        assert_eq!(
            manager
                .submit_heal_request(normal)
                .await
                .expect("first request should be accepted"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(forced)
                .await
                .expect("forced request should bypass duplicate/full admission"),
            HealAdmissionResult::Accepted
        );
        assert_eq!(
            manager
                .submit_heal_request(duplicate)
                .await
                .expect("non-forced duplicate should merge while forced request is queued"),
            HealAdmissionResult::Merged
        );
    }

    #[test]
    fn test_running_heal_set_counts_groups_set_scoped_tasks() {
        let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
        let erasure_task = Arc::new(HealTask::from_request(
            HealRequest::new(
                HealType::ErasureSet {
                    buckets: vec!["bucket".to_string()],
                    set_disk_id: "pool_0_set_1".to_string(),
                },
                HealOptions::default(),
                HealPriority::Normal,
            ),
            storage.clone(),
        ));
        let scoped_options = HealOptions {
            pool_index: Some(0),
            set_index: Some(1),
            ..Default::default()
        };
        let scoped_object_task = Arc::new(HealTask::from_request(
            HealRequest::new(
                HealType::Object {
                    bucket: "bucket".to_string(),
                    object: "scoped-object".to_string(),
                    version_id: None,
                },
                scoped_options,
                HealPriority::Normal,
            ),
            storage.clone(),
        ));
        let object_task = Arc::new(HealTask::from_request(
            HealRequest::new(
                HealType::Object {
                    bucket: "bucket".to_string(),
                    object: "object".to_string(),
                    version_id: None,
                },
                HealOptions::default(),
                HealPriority::Normal,
            ),
            storage,
        ));

        let mut active = HashMap::new();
        active.insert(erasure_task.id.clone(), erasure_task);
        active.insert(scoped_object_task.id.clone(), scoped_object_task);
        active.insert(object_task.id.clone(), object_task);

        let counts = running_heal_set_counts(&active);
        assert_eq!(counts.get("pool_0_set_1"), Some(&2));
        assert_eq!(counts.len(), 1);
    }

    #[test]
    fn test_heal_config_respects_feature_flags() {
        temp_env::with_vars(
            [
                (rustfs_config::ENV_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE, Some("false")),
                (rustfs_config::ENV_HEAL_SET_BULKHEAD_ENABLE, Some("false")),
                (rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("false")),
            ],
            || {
                let config = HealConfig::default();
                assert!(!config.event_driven_scheduler_enable);
                assert!(!config.set_bulkhead_enable);
                assert!(!config.page_parallel_enable);
            },
        );
    }
}
