diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index b0cac484c..4f82909c7 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -217,7 +217,7 @@ impl ECStore { let mut times = 0; let mut interval = 1; loop { - if let Ok(fm) = store_init::connect_load_init_formats( + match store_init::connect_load_init_formats( first_is_local, &disks, pool_eps.set_count, @@ -226,15 +226,18 @@ impl ECStore { ) .await { - break fm; + Ok(fm) => break Ok(fm), + // Wrap the final error if we are giving up + Err(e) if times >= 10 => { + break Err(Error::other(format!("can not get formats after {} retries, last error: {e}", times))); + } + // Retrying so just drop the error + Err(_) => {} } times += 1; if interval < 16 { interval *= 2; } - if times > 10 { - return Err(Error::other("can not get formats")); - } info!("retrying get formats after {:?}", interval); select! { _ = tokio::signal::ctrl_c() => { @@ -245,7 +248,7 @@ impl ECStore { } } } - }; + }?; if deployment_id.is_none() { deployment_id = Some(fm.id); diff --git a/crates/ecstore/src/store_init.rs b/crates/ecstore/src/store_init.rs index 31ea864e8..0d4303dea 100644 --- a/crates/ecstore/src/store_init.rs +++ b/crates/ecstore/src/store_init.rs @@ -27,7 +27,7 @@ use crate::{ }; use futures::future::join_all; use std::collections::{HashMap, hash_map::Entry}; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use uuid::Uuid; pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec>, Vec>) { @@ -66,8 +66,6 @@ pub async fn connect_load_init_formats( ) -> Result { let (formats, errs) = load_format_erasure_all(disks, false).await; - // debug!("load_format_erasure_all errs {:?}", &errs); - check_disk_fatal_errs(&errs)?; check_format_erasure_values(&formats, set_drive_count)?; @@ -199,16 +197,27 @@ pub fn check_format_erasure_values( check_format_erasure_value(f)?; - if formats.len() != f.erasure.sets.len() * f.erasure.sets[0].len() { - return Err(Error::other("formats length for erasure.sets not mtach")); + let first_set = f.erasure.sets.first().ok_or_else(|| Error::other("erasure.sets is empty"))?; + + if formats.len() != f.erasure.sets.len() * first_set.len() { + return Err(Error::other(format!( + "formats length for erasure.sets does not match: got {}, expected {}", + formats.len(), + f.erasure.sets.len() * first_set.len() + ))); } - if f.erasure.sets[0].len() != set_drive_count { - return Err(Error::other("erasure set length not match set_drive_count")); + if first_set.len() != set_drive_count { + return Err(Error::other(format!( + "erasure set length for set_drive_count does not match: got {}, expected {}", + first_set.len(), + set_drive_count + ))); } } Ok(()) } + fn check_format_erasure_value(format: &FormatV3) -> Result<()> { if format.version != FormatMetaVersion::V1 { return Err(Error::other("invalid FormatMetaVersion")); @@ -254,6 +263,24 @@ pub async fn load_format_erasure_all(disks: &[Option], heal: bool) -> } } + // Log aggregation summary of format load results + let ok_count = errors.iter().filter(|e| e.is_none()).count(); + let err_count = errors.iter().filter(|e| e.is_some()).count(); + // Count occurrences of each unique error + let mut err_counts: HashMap = HashMap::new(); + for err in errors.iter().flatten() { + *err_counts.entry(format!("{err}")).or_default() += 1; + } + if !err_counts.is_empty() { + debug!( + disks_ok = ok_count, + disks_err = err_count, + disks_total = disks.len(), + "load format erasure all errors: {:?}", + err_counts + ); + } + (datas, errors) }