fix(ecstore): start with unresolved Kubernetes peers (#5460)

* fix(ecstore): start with unresolved Kubernetes peers

* fix(ecstore): infer Kubernetes endpoint identity safely

* fix(ecstore): fail closed on unsafe format migration

* fix(ecstore): reject poisoned format heal candidates

* fix(ecstore): reject unsafe legacy migration outliers

* fix(ecstore): resume interrupted format migrations

* fix(ecstore): preserve Kubernetes startup compatibility

crates/ecstore/src/store/heal.rs

@@ -30,6 +30,7 @@ impl ECStore {
         };
 
         let mut count_no_heal = 0;
+        let mut first_error = None;
         for pool in self.pools.iter() {
             let (mut result, err) = pool.heal_format(dry_run).await?;
             if let Some(err) = err {
@@ -37,8 +38,8 @@ impl ECStore {
                     StorageError::NoHealRequired => {
                         count_no_heal += 1;
                     }
-                    _ => {
-                        continue;
+                    err => {
+                        first_error.get_or_insert(err);
                     }
                 }
             }
@@ -47,6 +48,9 @@ impl ECStore {
             r.before.drives.append(&mut result.before.drives);
             r.after.drives.append(&mut result.after.drives);
         }
+        if let Some(err) = first_error {
+            return Ok((r, Some(err)));
+        }
         if count_no_heal == self.pools.len() {
             info!(
                 event = EVENT_HEAL_FORMAT_COMPLETED,
@@ -165,3 +169,134 @@ impl ECStore {
         Err(StorageError::NotImplemented)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::{DiskOption, format::FormatV3, new_disk};
+    use crate::layout::endpoints::{Endpoints, PoolEndpoints};
+    use crate::store::init_format::{load_format_erasure, save_format_file};
+
+    #[tokio::test]
+    async fn handle_heal_format_continues_after_a_pool_error() {
+        let canonical_format = FormatV3::new(1, 3);
+        let mut foreign_format = canonical_format.clone();
+        foreign_format.id = Uuid::new_v4();
+        let mut temp_dirs = Vec::new();
+        let mut endpoints = Vec::new();
+        let mut disks = Vec::new();
+
+        for disk_index in 0..3 {
+            let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
+            let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
+                .expect("temporary endpoint should parse");
+            endpoint.set_pool_index(0);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(disk_index);
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("temporary disk should open");
+            let mut disk_format = foreign_format.clone();
+            disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
+            save_format_file(&Some(disk.clone()), &Some(disk_format))
+                .await
+                .expect("foreign format should be written");
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let pool_endpoints = PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 3,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: "foreign-format-majority-test".to_string(),
+            platform: "test".to_string(),
+        };
+        let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
+            .await
+            .expect("test pool should build around the cached canonical format");
+
+        let mut recoverable_format = FormatV3::new(1, 3);
+        recoverable_format.id = canonical_format.id;
+        let mut recoverable_temp_dirs = Vec::new();
+        let mut recoverable_endpoints = Vec::new();
+        let mut recoverable_disks = Vec::new();
+        let mut unformatted_disk = None;
+        for disk_index in 0..3 {
+            let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
+            let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
+                .expect("temporary endpoint should parse");
+            endpoint.set_pool_index(1);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(disk_index);
+            let disk = new_disk(
+                &endpoint,
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await
+            .expect("temporary disk should open");
+            if disk_index < 2 {
+                let mut disk_format = recoverable_format.clone();
+                disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
+                save_format_file(&Some(disk.clone()), &Some(disk_format))
+                    .await
+                    .expect("recoverable format should be written");
+            } else {
+                unformatted_disk = Some(disk.clone());
+            }
+            recoverable_temp_dirs.push(temp_dir);
+            recoverable_endpoints.push(endpoint);
+            recoverable_disks.push(Some(disk));
+        }
+        let recoverable_pool_endpoints = PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 3,
+            endpoints: Endpoints::from(recoverable_endpoints),
+            cmd_line: "recoverable-format-test".to_string(),
+            platform: "test".to_string(),
+        };
+        let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
+            .await
+            .expect("recoverable test pool should build");
+
+        let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
+        let store = ECStore {
+            id: canonical_format.id,
+            disk_map: HashMap::new(),
+            pools: vec![pool, recoverable_pool],
+            peer_sys: S3PeerSys::new(&endpoint_pools),
+            pool_meta: RwLock::new(PoolMeta::default()),
+            rebalance_meta: RwLock::new(None),
+            decommission_cancelers: RwLock::new(Vec::new()),
+            start_gate: Mutex::new(()),
+            pool_meta_save_gate: Mutex::new(()),
+            ctx: crate::runtime::instance::bootstrap_ctx(),
+        };
+
+        let (result, err) = store
+            .handle_heal_format(false)
+            .await
+            .expect("format heal should return the typed pool error");
+        assert!(
+            matches!(err, Some(StorageError::CorruptedFormat)),
+            "foreign format majority must not be downgraded to a successful heal: {err:?}"
+        );
+        assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
+        let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
+            .await
+            .expect("the later pool should be healed despite the first pool error");
+        assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
+    }
+}

crates/ecstore/src/store/init.rs

@@ -101,6 +101,10 @@ fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool {
     matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES
 }
 
+fn should_retry_format_load(err: &Error) -> bool {
+    !matches!(err, Error::CorruptedFormat)
+}
+
 fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_meta_loaded: bool) -> bool {
     rebalance_meta_loaded && !decommission_running
 }
@@ -294,7 +298,7 @@ impl ECStore {
             // periodic monitoring until format loading succeeds. Startup RPC
             // failures can still spawn recovery probes for peers that come up
             // after this node.
-            let (disks, errs) = init_format::init_disks(
+            let (mut disks, errs) = init_format::init_disks(
                 &pool_eps.endpoints,
                 &DiskOption {
                     cleanup: true,
@@ -311,7 +315,7 @@ impl ECStore {
                 loop {
                     match init_format::connect_load_init_formats(
                         pool_first_is_local,
-                        &disks,
+                        &mut disks,
                         pool_eps.set_count,
                         pool_eps.drives_per_set,
                         deployment_id,
@@ -319,6 +323,7 @@ impl ECStore {
                     .await
                     {
                         Ok(fm) => break Ok(fm),
+                        Err(e) if !should_retry_format_load(&e) => break Err(e),
                         // Wrap the final error if we are giving up
                         Err(e) if times >= 10 => {
                             break Err(Error::other(format!("store init failed to load formats after {times} retries: {e}")));
@@ -551,7 +556,7 @@ mod tests {
         LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, load_pool_meta_for_startup, pool_first_endpoint_is_local,
         pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with,
         resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
-        should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
+        should_retry_format_load, should_retry_local_decommission_resume, wait_for_local_decommission_resume_delay,
     };
     #[cfg(feature = "test-util")]
     use crate::{
@@ -773,6 +778,13 @@ mod tests {
         assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0));
     }
 
+    #[test]
+    fn test_should_retry_format_load_rejects_permanent_corruption() {
+        assert!(!should_retry_format_load(&StorageError::CorruptedFormat));
+        assert!(should_retry_format_load(&StorageError::ErasureReadQuorum));
+        assert!(should_retry_format_load(&StorageError::FirstDiskWait));
+    }
+
     #[test]
     fn test_should_auto_start_rebalance_after_init_allows_loaded_rebalance_without_decommission() {
         assert!(should_auto_start_rebalance_after_init(false, true));

crates/ecstore/src/store/mod.rs

@@ -989,7 +989,7 @@ mod tests {
 
         init_local_disks(endpoint_pools.clone()).await.expect("init local disks");
 
-        let (disks, errs) = init_disks(
+        let (mut disks, errs) = init_disks(
             &endpoint_pools.as_ref().first().expect("pool endpoints").endpoints,
             &DiskOption {
                 cleanup: true,
@@ -999,7 +999,7 @@ mod tests {
         .await;
 
         assert!(errs.iter().all(|err| err.is_none()), "disk init should succeed: {errs:?}");
-        connect_load_init_formats(true, &disks, 1, 4, None)
+        connect_load_init_formats(true, &mut disks, 1, 4, None)
             .await
             .expect("initialize format metadata");