fix(storage): harden offline drive fail-fast paths (#2564)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

crates/ecstore/src/disk/disk_store.rs

@@ -16,10 +16,16 @@ use crate::disk::{
     CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
     FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo,
     WalkDirOptions,
+    health_state::{
+        RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
+        get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
+        record_drive_recovery_class, record_drive_runtime_state, record_drive_state_transition,
+    },
     local::{LocalDisk, ScanGuard},
 };
 use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
+use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use std::{
     path::PathBuf,
@@ -40,7 +46,6 @@ const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
 pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
 pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
@@ -51,9 +56,52 @@ lazy_static::lazy_static! {
 }
 
 pub fn get_max_timeout_duration() -> Duration {
-    std::env::var(ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION)
-        .map(|v| Duration::from_secs(v.parse::<u64>().unwrap_or(30)))
-        .unwrap_or(Duration::from_secs(30))
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION,
+        rustfs_config::DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS,
+    ))
+}
+
+fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
+    Duration::from_secs(
+        rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
+            .unwrap_or(default_secs),
+    )
+}
+
+pub fn get_drive_metadata_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_disk_info_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_list_dir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_stall_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+    )
 }
 
 /// DiskHealthTracker tracks the health status of a disk.
@@ -68,6 +116,16 @@ pub struct DiskHealthTracker {
     pub status: AtomicU32,
     /// Atomic number of waiting operations
     pub waiting: AtomicU32,
+    /// Runtime drive health state
+    pub runtime_state: AtomicU32,
+    /// Consecutive failures while transitioning away from online
+    pub consecutive_failures: AtomicU32,
+    /// Consecutive successes while returning online
+    pub consecutive_successes: AtomicU32,
+    /// When the drive first left the online state
+    pub offline_since_unix_secs: AtomicI64,
+    /// Last runtime state transition timestamp
+    pub last_transition_unix_secs: AtomicI64,
 }
 
 impl DiskHealthTracker {
@@ -83,6 +141,11 @@ impl DiskHealthTracker {
             last_started: AtomicI64::new(now),
             status: AtomicU32::new(DISK_HEALTH_OK),
             waiting: AtomicU32::new(0),
+            runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
+            consecutive_failures: AtomicU32::new(0),
+            consecutive_successes: AtomicU32::new(0),
+            offline_since_unix_secs: AtomicI64::new(0),
+            last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
         }
     }
 
@@ -110,12 +173,139 @@ impl DiskHealthTracker {
         self.status.store(DISK_HEALTH_OK, Ordering::Release);
     }
 
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.runtime_state.store(state as u32, Ordering::Release);
+        match state {
+            RuntimeDriveHealthState::Offline => self.set_faulty(),
+            RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => {
+                self.set_ok();
+            }
+        }
+    }
+
     pub fn swap_ok_to_faulty(&self) -> bool {
         self.status
             .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed)
             .is_ok()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire))
+    }
+
+    pub fn offline_duration(&self) -> Option<Duration> {
+        let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
+        if offline_since <= 0 {
+            return None;
+        }
+        let now = current_unix_secs();
+        Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
+    }
+
+    pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let now = current_unix_secs();
+        let next = match current {
+            RuntimeDriveHealthState::Online => {
+                self.consecutive_failures.store(1, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                self.offline_since_unix_secs
+                    .compare_exchange(0, now as i64, Ordering::AcqRel, Ordering::Relaxed)
+                    .ok();
+                RuntimeDriveHealthState::Suspect
+            }
+            RuntimeDriveHealthState::Suspect => {
+                let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
+                if failures >= get_drive_suspect_failure_threshold() {
+                    RuntimeDriveHealthState::Offline
+                } else {
+                    RuntimeDriveHealthState::Suspect
+                }
+            }
+            RuntimeDriveHealthState::Returning => {
+                self.consecutive_failures.store(0, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                RuntimeDriveHealthState::Offline
+            }
+            RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
+        };
+
+        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        self.transition_state(endpoint, current, next, reason);
+        current == RuntimeDriveHealthState::Online
+    }
+
+    pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let next = match current {
+            RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Suspect => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Offline => {
+                self.consecutive_successes.store(1, Ordering::Release);
+                RuntimeDriveHealthState::Returning
+            }
+            RuntimeDriveHealthState::Returning => {
+                let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1;
+                if successes >= get_drive_returning_success_threshold() {
+                    RuntimeDriveHealthState::Online
+                } else {
+                    RuntimeDriveHealthState::Returning
+                }
+            }
+        };
+
+        let became_online = next == RuntimeDriveHealthState::Online;
+        if became_online {
+            self.status.store(DISK_HEALTH_OK, Ordering::Release);
+            self.consecutive_failures.store(0, Ordering::Release);
+            self.consecutive_successes.store(0, Ordering::Release);
+        }
+        self.transition_state(endpoint, current, next, reason);
+        if became_online {
+            self.log_success();
+        }
+        became_online
+    }
+
+    fn transition_state(
+        &self,
+        endpoint: &Endpoint,
+        current: RuntimeDriveHealthState,
+        next: RuntimeDriveHealthState,
+        reason: &'static str,
+    ) {
+        if current == next {
+            return;
+        }
+
+        self.runtime_state.store(next as u32, Ordering::Release);
+        self.last_transition_unix_secs
+            .store(current_unix_secs() as i64, Ordering::Release);
+
+        if matches!(
+            next,
+            RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Offline | RuntimeDriveHealthState::Returning
+        ) && self.offline_since_unix_secs.load(Ordering::Acquire) == 0
+        {
+            self.offline_since_unix_secs
+                .store(current_unix_secs() as i64, Ordering::Release);
+        }
+
+        if next == RuntimeDriveHealthState::Online {
+            if let Some(duration) = self.offline_duration() {
+                record_drive_offline_duration(endpoint, duration);
+                record_drive_recovery_class(classify_drive_recovery(duration));
+            }
+            self.offline_since_unix_secs.store(0, Ordering::Release);
+        } else if let Some(duration) = self.offline_duration() {
+            record_drive_offline_duration(endpoint, duration);
+        }
+
+        record_drive_state_transition(endpoint, current, next, reason);
+        record_drive_runtime_state(endpoint, next);
+    }
+
     /// Increment waiting operations counter
     pub fn increment_waiting(&self) {
         self.waiting.fetch_add(1, Ordering::Relaxed);
@@ -137,6 +327,13 @@ impl DiskHealthTracker {
     }
 }
 
+fn current_unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_secs()
+}
+
 impl Default for DiskHealthTracker {
     fn default() -> Self {
         Self::new()
@@ -186,19 +383,34 @@ impl LocalDiskWrapper {
         let env_health_check =
             rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
 
-        Self {
+        let wrapper = Self {
             disk,
             health: Arc::new(DiskHealthTracker::new()),
             health_check: health_check && env_health_check,
             cancel_token: CancellationToken::new(),
             disk_id: Arc::new(RwLock::new(None)),
-        }
+        };
+        record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
+        wrapper
     }
 
     pub fn get_disk(&self) -> Arc<LocalDisk> {
         self.disk.clone()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        self.health.runtime_state()
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        self.health.offline_duration().map(|duration| duration.as_secs())
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.health.force_runtime_state_for_test(state);
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -219,6 +431,20 @@ impl LocalDiskWrapper {
         self.cancel_token.cancel();
     }
 
+    fn spawn_recovery_monitor_if_needed(&self) {
+        if !self.health_check {
+            return;
+        }
+
+        self.health.increment_waiting();
+        let health = Arc::clone(&self.health);
+        let disk = Arc::clone(&self.disk);
+        let cancel_token = self.cancel_token.clone();
+        tokio::spawn(async move {
+            Self::monitor_disk_status(disk, health, cancel_token).await;
+        });
+    }
+
     /// Monitor disk writability periodically
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         // TODO: config interval
@@ -256,7 +482,9 @@ impl LocalDiskWrapper {
 
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err() && health.swap_ok_to_faulty() {
+                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                        && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
+                    {
                         // Health check failed, disk is considered faulty
                         warn!("health check: failed, disk is considered faulty");
 
@@ -345,9 +573,9 @@ impl LocalDiskWrapper {
 
     /// Monitor disk status and try to bring it back online
     async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        const CHECK_EVERY: Duration = Duration::from_secs(5);
+        let check_every = get_drive_returning_probe_interval();
 
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(check_every);
 
         loop {
             tokio::select! {
@@ -362,12 +590,18 @@ impl LocalDiskWrapper {
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
                     match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
                         Ok(_) => {
+                            let state_before = health.runtime_state();
+                            let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
+                            info!("Disk {} recovery probe succeeded; state={:?}", disk.to_string(), state_before);
+                            if !is_online {
+                                continue;
+                            }
                             info!("Disk {} is back online", disk.to_string());
-                            health.set_ok();
                             health.decrement_waiting();
                             return;
                         }
                         Err(e) => {
+                            health.mark_failure(&disk.endpoint(), "recovery_probe_failed");
                             warn!("Disk {} still faulty: {:?}", disk.to_string(), e);
                         }
                     }
@@ -435,6 +669,19 @@ impl LocalDiskWrapper {
     /// Track disk health for an operation.
     /// This method should wrap disk operations to ensure health checking.
     pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        self.track_disk_health_with_op("unknown", operation, timeout_duration).await
+    }
+
+    pub async fn track_disk_health_with_op<T, F, Fut>(
+        &self,
+        op: &'static str,
+        operation: F,
+        timeout_duration: Duration,
+    ) -> Result<T>
     where
         F: FnOnce() -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
@@ -479,7 +726,21 @@ impl LocalDiskWrapper {
             Err(_) => {
                 // Timeout occurred, mark disk as potentially faulty and decrement waiting counter
                 self.health.decrement_waiting();
-                warn!("disk operation timeout after {:?}", timeout_duration);
+                if self.health.mark_failure(&self.endpoint(), "operation_timeout") {
+                    self.spawn_recovery_monitor_if_needed();
+                }
+                counter!(
+                    "rustfs_drive_op_timeout_total",
+                    "endpoint" => self.endpoint().to_string(),
+                    "op" => op.to_string()
+                )
+                .increment(1);
+                warn!(
+                    endpoint = %self.endpoint(),
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    "Local disk operation timed out"
+                );
                 Err(DiskError::other(format!("disk operation timeout after {timeout_duration:?}")))
             }
         }
@@ -489,8 +750,12 @@ impl LocalDiskWrapper {
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        self.track_disk_health(|| async { self.disk.read_metadata(volume, path).await }, Duration::ZERO)
-            .await
+        self.track_disk_health_with_op(
+            "read_metadata",
+            || async { self.disk.read_metadata(volume, path).await },
+            get_drive_metadata_timeout(),
+        )
+        .await
     }
 
     fn start_scan(&self) -> ScanGuard {
@@ -565,15 +830,22 @@ impl DiskAPI for LocalDiskWrapper {
             return Err(DiskError::FaultyDisk);
         }
 
-        let result = self.disk.disk_info(opts).await?;
+        self.track_disk_health_with_op(
+            "disk_info",
+            || async {
+                let result = self.disk.disk_info(opts).await?;
 
-        if let Some(current_disk_id) = *self.disk_id.read().await
-            && Some(current_disk_id) != result.id
-        {
-            return Err(DiskError::DiskNotFound);
-        };
+                if let Some(current_disk_id) = *self.disk_id.read().await
+                    && Some(current_disk_id) != result.id
+                {
+                    return Err(DiskError::DiskNotFound);
+                };
 
-        Ok(result)
+                Ok(result)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
     }
 
     async fn make_volume(&self, volume: &str) -> Result<()> {
@@ -587,7 +859,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
-        self.track_disk_health(|| async { self.disk.list_volumes().await }, Duration::ZERO)
+        self.track_disk_health_with_op("list_volumes", || async { self.disk.list_volumes().await }, Duration::ZERO)
             .await
     }
 
@@ -602,7 +874,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
-        self.track_disk_health(|| async { self.disk.walk_dir(opts, wr).await }, Duration::ZERO)
+        self.track_disk_health_with_op("walk_dir", || async { self.disk.walk_dir(opts, wr).await }, get_drive_walkdir_timeout())
             .await
     }
 
@@ -710,9 +982,10 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
-        self.track_disk_health(
+        self.track_disk_health_with_op(
+            "list_dir",
             || async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
-            get_max_timeout_duration(),
+            get_drive_list_dir_timeout(),
         )
         .await
     }
@@ -802,3 +1075,74 @@ impl DiskAPI for LocalDiskWrapper {
             .await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::endpoint::Endpoint;
+    use crate::disk::health_state::RuntimeDriveHealthState;
+
+    #[test]
+    fn drive_metadata_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
+                assert_eq!(
+                    get_drive_metadata_timeout(),
+                    Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(17));
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_prefers_canonical_over_legacy() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("7"), || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(7));
+            });
+        });
+    }
+
+    #[test]
+    fn runtime_state_transitions_from_online_to_suspect_then_offline() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+
+        assert!(!health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+        assert!(health.offline_duration().is_some());
+    }
+
+    #[test]
+    fn runtime_state_transitions_back_online_after_recovery_threshold() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        health.mark_failure(&endpoint, "timeout");
+        health.mark_failure(&endpoint, "timeout");
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.offline_duration().is_none());
+    }
+}

crates/ecstore/src/disk/health_state.rs

@@ -0,0 +1,237 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{DiskAPI, DiskStore};
+use crate::disk::endpoint::Endpoint;
+use metrics::{counter, gauge};
+use std::time::Duration;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u32)]
+pub enum RuntimeDriveHealthState {
+    Online = 0,
+    Suspect = 1,
+    Offline = 2,
+    Returning = 3,
+}
+
+impl RuntimeDriveHealthState {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Online => "online",
+            Self::Suspect => "suspect",
+            Self::Offline => "offline",
+            Self::Returning => "returning",
+        }
+    }
+
+    pub fn from_u32(value: u32) -> Self {
+        match value {
+            1 => Self::Suspect,
+            2 => Self::Offline,
+            3 => Self::Returning,
+            _ => Self::Online,
+        }
+    }
+
+    pub fn is_snapshot_eligible(self) -> bool {
+        matches!(self, Self::Online | Self::Suspect | Self::Returning)
+    }
+
+    pub fn is_strictly_online(self) -> bool {
+        matches!(self, Self::Online)
+    }
+
+    pub fn should_probe_for_admin(self) -> bool {
+        matches!(self, Self::Online | Self::Returning)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DriveRecoveryClass {
+    ShortOffline,
+    MediumOffline,
+    LongOffline,
+}
+
+impl DriveRecoveryClass {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::ShortOffline => "short_offline",
+            Self::MediumOffline => "medium_offline",
+            Self::LongOffline => "long_offline",
+        }
+    }
+}
+
+pub fn get_drive_suspect_failure_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_success_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_probe_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_offline_grace_period() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+        rustfs_config::DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+    ))
+}
+
+pub fn get_drive_long_offline_threshold() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+        rustfs_config::DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+    ))
+}
+
+pub fn classify_drive_recovery(duration: Duration) -> DriveRecoveryClass {
+    if duration <= get_drive_offline_grace_period() {
+        DriveRecoveryClass::ShortOffline
+    } else if duration >= get_drive_long_offline_threshold() {
+        DriveRecoveryClass::LongOffline
+    } else {
+        DriveRecoveryClass::MediumOffline
+    }
+}
+
+pub fn record_drive_runtime_state(endpoint: &Endpoint, state: RuntimeDriveHealthState) {
+    let endpoint_label = endpoint.to_string();
+    let pool_label = endpoint.pool_idx.to_string();
+    let set_label = endpoint.set_idx.to_string();
+    let disk_label = endpoint.disk_idx.to_string();
+
+    for candidate in [
+        RuntimeDriveHealthState::Online,
+        RuntimeDriveHealthState::Suspect,
+        RuntimeDriveHealthState::Offline,
+        RuntimeDriveHealthState::Returning,
+    ] {
+        gauge!(
+            "rustfs_drive_runtime_state",
+            "endpoint" => endpoint_label.clone(),
+            "pool" => pool_label.clone(),
+            "set" => set_label.clone(),
+            "disk" => disk_label.clone(),
+            "state" => candidate.as_str().to_string()
+        )
+        .set(if candidate == state { 1.0 } else { 0.0 });
+    }
+}
+
+pub fn record_drive_state_transition(
+    endpoint: &Endpoint,
+    from: RuntimeDriveHealthState,
+    to: RuntimeDriveHealthState,
+    reason: &'static str,
+) {
+    counter!(
+        "rustfs_drive_state_transition_total",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string(),
+        "from" => from.as_str().to_string(),
+        "to" => to.as_str().to_string(),
+        "reason" => reason.to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_recovery_class(class: DriveRecoveryClass) {
+    counter!(
+        "rustfs_drive_recovery_class_total",
+        "class" => class.as_str().to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_offline_duration(endpoint: &Endpoint, duration: Duration) {
+    gauge!(
+        "rustfs_drive_offline_duration_seconds",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string()
+    )
+    .set(duration.as_secs_f64());
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DriveMembershipSnapshot {
+    pub online: Vec<DiskStore>,
+    pub suspect: Vec<DiskStore>,
+    pub returning: Vec<DiskStore>,
+    pub offline: Vec<DiskStore>,
+}
+
+impl DriveMembershipSnapshot {
+    pub fn from_optional_disks(disks: &[Option<DiskStore>]) -> Self {
+        let mut snapshot = Self::default();
+
+        for disk in disks.iter().flatten() {
+            match disk.runtime_state() {
+                RuntimeDriveHealthState::Online => snapshot.online.push(disk.clone()),
+                RuntimeDriveHealthState::Suspect => snapshot.suspect.push(disk.clone()),
+                RuntimeDriveHealthState::Returning => snapshot.returning.push(disk.clone()),
+                RuntimeDriveHealthState::Offline => snapshot.offline.push(disk.clone()),
+            }
+        }
+
+        snapshot
+    }
+
+    pub fn scanner_heal_candidates(&self) -> Vec<DiskStore> {
+        let mut disks = Vec::with_capacity(self.online.len() + self.suspect.len() + self.returning.len());
+        disks.extend(self.online.iter().cloned());
+        disks.extend(self.suspect.iter().cloned());
+        disks.extend(self.returning.iter().cloned());
+        disks
+    }
+
+    pub fn strict_online_candidates(&self) -> Vec<DiskStore> {
+        self.online.clone()
+    }
+
+    pub fn strict_online_local_candidates(&self) -> Vec<DiskStore> {
+        self.online.iter().filter(|disk| disk.is_local()).cloned().collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn runtime_drive_health_state_snapshot_eligibility_matches_membership_policy() {
+        assert!(RuntimeDriveHealthState::Online.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Suspect.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Returning.is_snapshot_eligible());
+        assert!(!RuntimeDriveHealthState::Offline.is_snapshot_eligible());
+    }
+}

crates/ecstore/src/disk/mod.rs

@@ -19,6 +19,7 @@ pub mod error_conv;
 pub mod error_reduce;
 pub mod format;
 pub mod fs;
+pub mod health_state;
 pub mod local;
 pub mod os;
 
@@ -33,6 +34,7 @@ pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
 use crate::disk::disk_store::LocalDiskWrapper;
+use crate::disk::health_state::RuntimeDriveHealthState;
 use crate::disk::local::ScanGuard;
 use crate::rpc::RemoteDisk;
 use bytes::Bytes;
@@ -410,6 +412,30 @@ impl DiskAPI for Disk {
     }
 }
 
+impl Disk {
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        match self {
+            Disk::Local(local_disk) => local_disk.runtime_state(),
+            Disk::Remote(remote_disk) => remote_disk.runtime_state(),
+        }
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        match self {
+            Disk::Local(local_disk) => local_disk.offline_duration_secs(),
+            Disk::Remote(remote_disk) => remote_disk.offline_duration_secs(),
+        }
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        match self {
+            Disk::Local(local_disk) => local_disk.force_runtime_state_for_test(state),
+            Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
+        }
+    }
+}
+
 impl Disk {
     /// Enable health monitoring on this disk.
     /// Called after startup format loading completes so that remote peers