logging(disks): Propogate storage disk init error, improve logging (#1825)

Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/store.rs

@@ -217,7 +217,7 @@ impl ECStore {
                 let mut times = 0;
                 let mut interval = 1;
                 loop {
-                    if let Ok(fm) = store_init::connect_load_init_formats(
+                    match store_init::connect_load_init_formats(
                         first_is_local,
                         &disks,
                         pool_eps.set_count,
@@ -226,15 +226,18 @@ impl ECStore {
                     )
                     .await
                     {
-                        break fm;
+                        Ok(fm) => break Ok(fm),
+                        // Wrap the final error if we are giving up
+                        Err(e) if times >= 10 => {
+                            break Err(Error::other(format!("can not get formats after {} retries, last error: {e}", times)));
+                        }
+                        // Retrying so just drop the error
+                        Err(_) => {}
                     }
                     times += 1;
                     if interval < 16 {
                         interval *= 2;
                     }
-                    if times > 10 {
-                        return Err(Error::other("can not get formats"));
-                    }
                     info!("retrying get formats after {:?}", interval);
                     select! {
                         _ = tokio::signal::ctrl_c() => {
@@ -245,7 +248,7 @@ impl ECStore {
                         }
                     }
                 }
-            };
+            }?;
 
             if deployment_id.is_none() {
                 deployment_id = Some(fm.id);

crates/ecstore/src/store_init.rs

@@ -27,7 +27,7 @@ use crate::{
 };
 use futures::future::join_all;
 use std::collections::{HashMap, hash_map::Entry};
-use tracing::{info, warn};
+use tracing::{debug, info, warn};
 use uuid::Uuid;
 
 pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
@@ -66,8 +66,6 @@ pub async fn connect_load_init_formats(
 ) -> Result<FormatV3> {
     let (formats, errs) = load_format_erasure_all(disks, false).await;
 
-    // debug!("load_format_erasure_all errs {:?}", &errs);
-
     check_disk_fatal_errs(&errs)?;
 
     check_format_erasure_values(&formats, set_drive_count)?;
@@ -199,16 +197,27 @@ pub fn check_format_erasure_values(
 
         check_format_erasure_value(f)?;
 
-        if formats.len() != f.erasure.sets.len() * f.erasure.sets[0].len() {
-            return Err(Error::other("formats length for erasure.sets not mtach"));
+        let first_set = f.erasure.sets.first().ok_or_else(|| Error::other("erasure.sets is empty"))?;
+
+        if formats.len() != f.erasure.sets.len() * first_set.len() {
+            return Err(Error::other(format!(
+                "formats length for erasure.sets does not match: got {}, expected {}",
+                formats.len(),
+                f.erasure.sets.len() * first_set.len()
+            )));
         }
 
-        if f.erasure.sets[0].len() != set_drive_count {
-            return Err(Error::other("erasure set length not match set_drive_count"));
+        if first_set.len() != set_drive_count {
+            return Err(Error::other(format!(
+                "erasure set length for set_drive_count does not match: got {}, expected {}",
+                first_set.len(),
+                set_drive_count
+            )));
         }
     }
     Ok(())
 }
+
 fn check_format_erasure_value(format: &FormatV3) -> Result<()> {
     if format.version != FormatMetaVersion::V1 {
         return Err(Error::other("invalid FormatMetaVersion"));
@@ -254,6 +263,24 @@ pub async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) ->
         }
     }
 
+    // Log aggregation summary of format load results
+    let ok_count = errors.iter().filter(|e| e.is_none()).count();
+    let err_count = errors.iter().filter(|e| e.is_some()).count();
+    // Count occurrences of each unique error
+    let mut err_counts: HashMap<String, usize> = HashMap::new();
+    for err in errors.iter().flatten() {
+        *err_counts.entry(format!("{err}")).or_default() += 1;
+    }
+    if !err_counts.is_empty() {
+        debug!(
+            disks_ok = ok_count,
+            disks_err = err_count,
+            disks_total = disks.len(),
+            "load format erasure all errors: {:?}",
+            err_counts
+        );
+    }
+
     (datas, errors)
 }