fix(storage): make acknowledged writes durable across power loss (#4221)

crates/heal/src/heal/manager.rs

@@ -48,6 +48,7 @@ const EVENT_HEAL_QUEUE_ADMISSION: &str = "heal_queue_admission";
 const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle";
 const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state";
 const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state";
+const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
 const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
 const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
 
@@ -1251,6 +1252,10 @@ impl HealManager {
             );
         }
 
+        // Detect a previous unclean shutdown (crash/power loss) and proactively
+        // verify all local erasure sets instead of waiting for the periodic scanner.
+        self.process_unclean_shutdown().await;
+
         info!(
             target: "rustfs::heal::manager",
             event = EVENT_HEAL_MANAGER_STATE,
@@ -1262,6 +1267,117 @@ impl HealManager {
         Ok(())
     }
 
+    /// Detect whether the previous run ended without a clean shutdown and, if so,
+    /// enqueue a full erasure-set heal for every local set. Also (re)writes the
+    /// marker for the current run; [`super::clear_unclean_shutdown_markers`]
+    /// removes it again during graceful shutdown. Best-effort: failures only log.
+    async fn process_unclean_shutdown(&self) {
+        let mut unclean = false;
+        let mut set_disk_ids = HashSet::new();
+
+        {
+            let local_disk_map = local_disk_map_read().await;
+            for disk in local_disk_map.values().flatten() {
+                let endpoint = disk.endpoint();
+                match disk
+                    .read_all(super::RUSTFS_META_BUCKET, super::UNCLEAN_SHUTDOWN_MARKER_PATH)
+                    .await
+                {
+                    Ok(_) => unclean = true,
+                    Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
+                    Err(err) => {
+                        debug!(
+                            target: "rustfs::heal::manager",
+                            event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
+                            component = LOG_COMPONENT_HEAL,
+                            subsystem = LOG_SUBSYSTEM_MANAGER,
+                            endpoint = %endpoint,
+                            error = ?err,
+                            "Unclean-shutdown marker check failed"
+                        );
+                    }
+                }
+
+                let marker = SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .map(|d| d.as_secs().to_string())
+                    .unwrap_or_default();
+                if let Err(err) = disk
+                    .write_all(super::RUSTFS_META_BUCKET, super::UNCLEAN_SHUTDOWN_MARKER_PATH, marker.into())
+                    .await
+                {
+                    warn!(
+                        target: "rustfs::heal::manager",
+                        event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
+                        component = LOG_COMPONENT_HEAL,
+                        subsystem = LOG_SUBSYSTEM_MANAGER,
+                        endpoint = %endpoint,
+                        error = ?err,
+                        "Unclean-shutdown marker write failed"
+                    );
+                }
+
+                if let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx) {
+                    set_disk_ids.insert(set_disk_id);
+                }
+            }
+        }
+
+        if !unclean || set_disk_ids.is_empty() {
+            return;
+        }
+
+        info!(
+            target: "rustfs::heal::manager",
+            event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
+            component = LOG_COMPONENT_HEAL,
+            subsystem = LOG_SUBSYSTEM_MANAGER,
+            set_count = set_disk_ids.len(),
+            "Unclean shutdown detected; scheduling erasure-set heal for local sets"
+        );
+
+        let buckets = match self.storage.list_buckets().await {
+            Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>(),
+            Err(err) => {
+                error!(
+                    target: "rustfs::heal::manager",
+                    event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
+                    component = LOG_COMPONENT_HEAL,
+                    subsystem = LOG_SUBSYSTEM_MANAGER,
+                    error = %err,
+                    "Unclean-shutdown heal skipped: bucket listing failed"
+                );
+                return;
+            }
+        };
+
+        for set_disk_id in set_disk_ids {
+            let mut req = HealRequest::new(
+                HealType::ErasureSet {
+                    buckets: buckets.clone(),
+                    set_disk_id: set_disk_id.clone(),
+                },
+                HealOptions {
+                    timeout: None,
+                    ..HealOptions::default()
+                },
+                HealPriority::Low,
+            );
+            req.source = HealRequestSource::AutoHeal;
+            if let Err(err) = self.submit_heal_request(req).await {
+                warn!(
+                    target: "rustfs::heal::manager",
+                    event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
+                    component = LOG_COMPONENT_HEAL,
+                    subsystem = LOG_SUBSYSTEM_MANAGER,
+                    set_disk_id,
+                    error = %err,
+                    "Unclean-shutdown heal enqueue failed"
+                );
+            }
+        }
+    }
+
     /// Stop HealManager
     pub async fn stop(&self) -> Result<()> {
         info!(

crates/heal/src/heal/mod.rs

@@ -40,6 +40,35 @@ pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME;
 pub(crate) const BUCKET_META_PREFIX: &str = ECSTORE_BUCKET_META_PREFIX;
 pub(crate) const RUSTFS_META_BUCKET: &str = ECSTORE_RUSTFS_META_BUCKET;
 
+/// Marker written to every local disk while the process runs; removed by
+/// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at
+/// startup means the previous run crashed or lost power, so the heal manager
+/// proactively re-verifies all local erasure sets.
+pub(crate) const UNCLEAN_SHUTDOWN_MARKER_PATH: &str = "unclean-shutdown";
+
+/// Remove the unclean-shutdown markers from all local disks. Call at the end of
+/// a graceful shutdown, after the data plane has stopped accepting writes.
+pub async fn clear_unclean_shutdown_markers() {
+    let local_disk_map = local_disk_map_read().await;
+    for disk in local_disk_map.values().flatten() {
+        if let Err(err) = EcstoreDiskAPI::delete(
+            disk.as_ref(),
+            RUSTFS_META_BUCKET,
+            UNCLEAN_SHUTDOWN_MARKER_PATH,
+            EcstoreDeleteOptions::default(),
+        )
+        .await
+            && err != DiskError::FileNotFound
+        {
+            tracing::warn!(
+                endpoint = %EcstoreDiskAPI::endpoint(disk.as_ref()),
+                error = ?err,
+                "failed to clear unclean-shutdown marker"
+            );
+        }
+    }
+}
+
 pub(crate) type DiskError = EcstoreDiskError;
 pub(crate) type DiskResult<T> = EcstoreDiskResult<T>;
 pub(crate) type DiskStore = EcstoreDiskStore;