fix(storage): harden offline drive fail-fast paths (#2564)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

crates/scanner/src/scanner_io.rs

@@ -19,6 +19,7 @@ use crate::{
     DataUsageInfo, SizeSummary, TierStats,
 };
 use futures::future::join_all;
+use metrics::counter;
 use rand::seq::SliceRandom as _;
 use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete};
@@ -50,6 +51,24 @@ use tokio::time::Duration;
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, warn};
 
+fn record_set_scan_failure(first_err: &mut Option<Error>, err: Error) {
+    if first_err.is_none() {
+        *first_err = Some(err);
+    }
+}
+
+fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
+    if results.iter().any(|result| result.info.last_update.is_some()) {
+        return Ok(());
+    }
+
+    if let Some(err) = first_err {
+        return Err(err);
+    }
+
+    Ok(())
+}
+
 #[async_trait::async_trait]
 pub trait ScannerIO: Send + Sync + Debug + 'static {
     async fn nsscanner(
@@ -125,6 +144,8 @@ impl ScannerIO for ECStore {
                 let results_index_clone = results_index as usize;
                 // Clone the Arc to move it into the spawned task
                 let set_clone: Arc<SetDisks> = Arc::clone(set);
+                let pool_label = set.pool_index.to_string();
+                let set_label = set.set_index.to_string();
 
                 let child_token_clone = child_token.clone();
                 let want_cycle_clone = want_cycle;
@@ -150,9 +171,21 @@ impl ScannerIO for ECStore {
                         .nsscanner_cache(child_token_clone.clone(), all_buckets_clone, tx, want_cycle_clone, scan_mode_clone)
                         .await
                     {
-                        error!("Failed to scan set: {e}");
-                        let _ = first_err_mutex_clone.lock().await.insert(e);
-                        child_token_clone.cancel();
+                        counter!(
+                            "rustfs_scanner_set_failure_total",
+                            "pool" => pool_label.clone(),
+                            "set" => set_label.clone(),
+                            "stage" => "nsscanner_cache".to_string()
+                        )
+                        .increment(1);
+                        error!(
+                            pool = %pool_label,
+                            set = %set_label,
+                            error = %e,
+                            "Failed to scan set; continuing scanner cycle"
+                        );
+                        let mut first_err = first_err_mutex_clone.lock().await;
+                        record_set_scan_failure(&mut first_err, e);
                     }
                 });
                 wait_futs.push(scanner_fut);
@@ -162,6 +195,7 @@ impl ScannerIO for ECStore {
         let (update_tx, mut update_rx) = tokio::sync::oneshot::channel::<()>();
 
         let all_buckets_clone = all_buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>();
+        let results_mutex_for_updates = results_mutex.clone();
         tokio::spawn(async move {
             let mut last_update = SystemTime::UNIX_EPOCH;
             let mut has_sent_once = false;
@@ -177,7 +211,7 @@ impl ScannerIO for ECStore {
                             break;
                         }
 
-                        let results = results_mutex.lock().await;
+                        let results = results_mutex_for_updates.lock().await;
                         let mut all_merged = DataUsageCache::default();
                         for result in results.iter() {
                             if result.info.last_update.is_none() {
@@ -196,7 +230,7 @@ impl ScannerIO for ECStore {
                         break;
                     }
                     _ = ticker.tick() => {
-                        let results = results_mutex.lock().await;
+                        let results = results_mutex_for_updates.lock().await;
                         let mut all_merged = DataUsageCache::default();
                         for result in results.iter() {
                             if result.info.last_update.is_none() {
@@ -223,7 +257,9 @@ impl ScannerIO for ECStore {
 
         let _ = update_tx.send(());
 
-        Ok(())
+        let first_err = first_err_mutex.lock().await.take();
+        let results = results_mutex.lock().await.clone();
+        finalize_nsscanner_result(&results, first_err)
     }
 }
 
@@ -561,13 +597,7 @@ impl ScannerIODisk for Disk {
             Err(_) => None,
         };
 
-        let Some(ecstore) = new_object_layer_fn() else {
-            error!("ECStore not available");
-            return Err(StorageError::other("ECStore not available".to_string()));
-        };
-
-        item.apply_actions(ecstore, object_infos, lock_config, &mut size_summary)
-            .await;
+        item.apply_actions(object_infos, lock_config, &mut size_summary).await;
 
         if !free_version_infos.is_empty() {
             let mut expiry_state = GLOBAL_ExpiryState.write().await;
@@ -668,3 +698,36 @@ impl ScannerIODisk for Disk {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn record_set_scan_failure_preserves_first_error() {
+        let mut first = None;
+        record_set_scan_failure(&mut first, Error::other("first"));
+        record_set_scan_failure(&mut first, Error::other("second"));
+
+        let first = first.expect("first error should be recorded");
+        assert!(first.to_string().contains("first"));
+    }
+
+    #[test]
+    fn finalize_nsscanner_result_returns_ok_when_any_set_succeeds() {
+        let mut results = vec![DataUsageCache::default(), DataUsageCache::default()];
+        results[1].info.last_update = Some(SystemTime::now());
+
+        let result = finalize_nsscanner_result(&results, Some(Error::other("set failed")));
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn finalize_nsscanner_result_returns_first_error_when_all_sets_fail() {
+        let results = vec![DataUsageCache::default(), DataUsageCache::default()];
+
+        let err = finalize_nsscanner_result(&results, Some(Error::other("set failed")))
+            .expect_err("all failed sets should bubble first error");
+        assert!(err.to_string().contains("set failed"));
+    }
+}