From 3fed21c68ae858f4bbb7f1be375d73b2af522f02 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 10 Jun 2026 22:24:12 +0800 Subject: [PATCH] fix(startup): add disk init diagnostics (#3324) * ci(build): remove macOS x86_64 target * ci(build): reduce workflow timeout * fix(startup): add disk init diagnostics --- crates/ecstore/src/disk/local.rs | 83 +++++++++++++++++++++++++++----- rustfs/src/main.rs | 41 +++++++++++++++- 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 166ae3908..badc6bdbb 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -46,7 +46,7 @@ use rustfs_utils::path::{ use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; -use std::io::SeekFrom; +use std::io::{Error as IoError, SeekFrom}; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -67,6 +67,31 @@ const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60); const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old"; const STARTUP_CLEANUP_WAIT_TIMEOUT: Duration = Duration::from_secs(2); +fn log_startup_disk_io_error(stage: &str, path: &Path, err: &IoError) { + warn!( + stage, + path = %path.display(), + error_kind = ?err.kind(), + raw_os_error = ?err.raw_os_error(), + error = ?err, + "local disk startup filesystem operation failed" + ); +} + +fn log_startup_disk_error(stage: &str, path: &Path, err: &DiskError) { + if let DiskError::Io(io_err) = err { + log_startup_disk_io_error(stage, path, io_err); + return; + } + + warn!( + stage, + path = %path.display(), + error = ?err, + "local disk startup operation failed" + ); +} + #[derive(Debug, Clone)] pub struct FormatInfo { pub id: Option, @@ -409,9 +434,17 @@ fn resolve_local_disk_root(ep_path: &str) -> Result { impl LocalDisk { pub async fn new(ep: &Endpoint, cleanup: bool) -> Result { debug!("Creating local disk"); - let root = resolve_local_disk_root(&ep.get_file_path())?; + let endpoint_path = ep.get_file_path(); + let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| { + log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err); + })?; - ensure_data_usage_layout(&root).await.map_err(DiskError::from)?; + ensure_data_usage_layout(&root) + .await + .map_err(DiskError::from) + .inspect_err(|err| { + log_startup_disk_error("ensure_data_usage_layout", &root, err); + })?; let startup_cleanup_ready = Arc::new(AtomicU32::new(u32::from(!cleanup))); let startup_cleanup_notify = Arc::new(Notify::new()); @@ -428,7 +461,9 @@ impl LocalDisk { // Use optimized path resolution instead of absolutize_virtually let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE); debug!("format_path: {:?}", format_path); - let (format_data, format_meta) = read_file_exists(&format_path).await?; + let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| { + log_startup_disk_error("read_format_json", &format_path, err); + })?; let mut id = None; // let mut format_legacy = false; @@ -521,7 +556,9 @@ impl LocalDisk { startup_cleanup_notify, exit_signal: None, }; - let (info, _root) = get_disk_info(root).await?; + let (info, _root) = get_disk_info(root.clone()).await.inspect_err(|err| { + log_startup_disk_error("get_disk_info", &root, err); + })?; disk.major = info.major; disk.minor = info.minor; disk.fstype = info.fstype; @@ -538,7 +575,9 @@ impl LocalDisk { disk.rotational = true; } - disk.make_meta_volumes().await?; + disk.make_meta_volumes().await.inspect_err(|err| { + log_startup_disk_error("make_meta_volumes", &disk.root, err); + })?; let (exit_tx, exit_rx) = tokio::sync::broadcast::channel(1); disk.exit_signal = Some(exit_tx); @@ -587,16 +626,21 @@ impl LocalDisk { let tmp_path = Self::meta_path(root, RUSTFS_META_TMP_BUCKET); let tmp_old_path = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET).join(Uuid::new_v4().to_string()); - rename_all(&tmp_path, &tmp_old_path, root).await?; + rename_all(&tmp_path, &tmp_old_path, root).await.inspect_err(|err| { + log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err); + })?; - tokio::fs::create_dir_all(Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET)).await?; + let tmp_deleted_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET); + tokio::fs::create_dir_all(&tmp_deleted_path).await.inspect_err(|err| { + log_startup_disk_io_error("cleanup_tmp_create_deleted_dir", &tmp_deleted_path, err); + })?; let tmp_old_root = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET); tokio::spawn(async move { if let Err(err) = tokio::fs::remove_dir_all(&tmp_old_root).await && err.kind() != ErrorKind::NotFound { - warn!(path = ?tmp_old_root, error = ?err, "failed to remove old temporary data"); + log_startup_disk_io_error("cleanup_tmp_remove_old_dir", &tmp_old_root, &err); } startup_cleanup_ready.store(1, Ordering::Release); startup_cleanup_notify.notify_waiters(); @@ -1672,13 +1716,26 @@ async fn read_file_all(path: impl AsRef) -> Result<(Bytes, Metadata)> { let p = path.as_ref(); let meta = read_file_metadata(&path).await?; - let data = fs::read(&p).await.map_err(to_file_error)?; + let data = fs::read(&p) + .await + .inspect_err(|err| { + log_startup_disk_io_error("read_file_all", p, err); + }) + .map_err(to_file_error)?; Ok((data.into(), meta)) } async fn read_file_metadata(p: impl AsRef) -> Result { - let meta = fs::metadata(&p).await.map_err(to_file_error)?; + let path = p.as_ref(); + let meta = fs::metadata(path) + .await + .inspect_err(|err| { + if err.kind() != ErrorKind::NotFound { + log_startup_disk_io_error("read_file_metadata", path, err); + } + }) + .map_err(to_file_error)?; Ok(meta) } @@ -3299,7 +3356,9 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf let drive_path = drive_path.to_string_lossy().to_string(); check_path_length(&drive_path)?; - let disk_info = get_info(&drive_path)?; + let disk_info = get_info(&drive_path).inspect_err(|err| { + log_startup_disk_io_error("get_disk_info_stat", Path::new(&drive_path), err); + })?; let root_drive = if !*GLOBAL_IsErasureSD.read().await { let root_disk_threshold = *GLOBAL_RootDiskThreshold.read().await; if root_disk_threshold > 0 { diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index ff8b312f7..a99a161fc 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -293,8 +293,22 @@ async fn run(config: rustfs::config::Config) -> Result<()> { set_global_addr(&config.address).await; // For RPC + info!( + target: "rustfs::main::run", + server_address = %server_address, + volumes = ?config.volumes, + "starting endpoint parsing" + ); let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), config.volumes.clone()) .await + .inspect_err(|err| { + error!( + target: "rustfs::main::run", + stage = "endpoint_parsing", + error = ?err, + "startup storage stage failed" + ); + }) .map_err(Error::other)?; enforce_unsupported_fs_policy(&endpoint_pools)?; @@ -302,7 +316,21 @@ async fn run(config: rustfs::config::Config) -> Result<()> { update_erasure_type(setup_type).await; // Initialize the local disk - init_local_disks(endpoint_pools.clone()).await.map_err(Error::other)?; + info!( + target: "rustfs::main::run", + "starting local disk initialization" + ); + init_local_disks(endpoint_pools.clone()) + .await + .inspect_err(|err| { + error!( + target: "rustfs::main::run", + stage = "local_disk_initialization", + error = ?err, + "startup storage stage failed" + ); + }) + .map_err(Error::other)?; prewarm_local_disk_id_map().await; // Initialize the lock clients @@ -366,10 +394,19 @@ async fn run(config: rustfs::config::Config) -> Result<()> { // init store // 2. Start Storage Engine (ECStore) + info!( + target: "rustfs::main::run", + "starting ECStore initialization" + ); let store = ECStore::new(server_addr, endpoint_pools.clone(), ctx.clone()) .await .inspect_err(|err| { - error!("ECStore::new {:?}", err); + error!( + target: "rustfs::main::run", + stage = "ecstore_initialization", + error = ?err, + "startup storage stage failed" + ); })?; ecconfig::init();