fix(ecstore): reset drive health between store init format retries (#2848)

Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>

crates/ecstore/src/disk/disk_store.rs

@@ -265,6 +265,26 @@ impl DiskHealthTracker {
         true
     }
 
+    /// Clear faulty/offline state so a store-init format load retry can issue RPC again.
+    ///
+    /// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
+    /// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
+    pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
+        self.status.store(DISK_HEALTH_OK, Ordering::Release);
+        self.runtime_state
+            .store(RuntimeDriveHealthState::Online as u32, Ordering::Release);
+        self.consecutive_failures.store(0, Ordering::Release);
+        self.consecutive_successes.store(0, Ordering::Release);
+        self.offline_since_unix_secs.store(0, Ordering::Release);
+        self.waiting.store(0, Ordering::Release);
+        let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap();
+        let now_nanos = now.as_nanos() as i64;
+        self.last_success.store(now_nanos, Ordering::Relaxed);
+        self.last_started.store(now_nanos, Ordering::Relaxed);
+        self.last_transition_unix_secs.store(now.as_secs() as i64, Ordering::Release);
+        record_drive_runtime_state(endpoint, RuntimeDriveHealthState::Online);
+    }
+
     pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
         let current = self.runtime_state();
         let next = match current {
@@ -448,6 +468,11 @@ impl LocalDiskWrapper {
         self.health.force_runtime_state_for_test(state);
     }
 
+    /// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
+    pub fn reset_health_for_store_init_retry(&self) {
+        self.health.reset_for_store_init_retry(&self.disk.endpoint());
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -1255,4 +1280,21 @@ mod tests {
         assert!(!health.is_faulty());
         assert!(health.offline_duration().is_none());
     }
+
+    #[test]
+    fn reset_for_store_init_retry_clears_faulty_and_back_online() {
+        let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        assert!(health.mark_offline(&endpoint, "simulated_fault"));
+        assert!(health.is_faulty());
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+        health.reset_for_store_init_retry(&endpoint);
+        assert!(!health.is_faulty());
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+
+        assert!(health.mark_offline(&endpoint, "again"));
+        assert!(health.is_faulty());
+    }
 }

crates/ecstore/src/disk/mod.rs

@@ -437,6 +437,15 @@ impl Disk {
 }
 
 impl Disk {
+    /// Reset drive health so `connect_load_init_formats` retries are not blocked by a prior
+    /// transient mark-faulty (same disk handles are reused across retries).
+    pub fn reset_health_for_store_init_retry(&self) {
+        match self {
+            Disk::Local(local_disk) => local_disk.reset_health_for_store_init_retry(),
+            Disk::Remote(remote_disk) => remote_disk.reset_health_for_store_init_retry(),
+        }
+    }
+
     /// Enable health monitoring on this disk.
     /// Called after startup format loading completes so that remote peers
     /// have time to come online before being marked as faulty.

crates/ecstore/src/rpc/remote_disk.rs

@@ -140,6 +140,11 @@ impl RemoteDisk {
         self.health.force_runtime_state_for_test(state);
     }
 
+    /// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
+    pub fn reset_health_for_store_init_retry(&self) {
+        self.health.reset_for_store_init_retry(&self.endpoint);
+    }
+
     fn spawn_recovery_monitor_if_needed(&self) {
         if !self.health_check {
             return;

crates/ecstore/src/store/init.rs

@@ -185,6 +185,11 @@ impl ECStore {
                         _ = sleep(Duration::from_secs(interval)) => {
                         }
                     }
+                    // After waiting for peers, clear transient faulty marks so the next attempt can open RPCs again
+                    // (these `DiskStore` handles are reused; `is_faulty()` would otherwise short-circuit).
+                    for disk in disks.iter().flatten() {
+                        disk.reset_health_for_store_init_retry();
+                    }
                 }
             }?;