gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(startup): add disk init diagnostics (#3324)

Browse Source 
* ci(build): remove macOS x86_64 target

* ci(build): reduce workflow timeout

* fix(startup): add disk init diagnostics
This commit is contained in:
houseme
2026-06-10 22:24:12 +08:00
committed by GitHub
parent 64c0ede026
commit 3fed21c68a
2 changed files with 110 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/disk/local.rs
+71 -12
View File
@@ -46,7 +46,7 @@ use rustfs_utils::path::{
use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt::Debug;
use std::io::SeekFrom;
use std::io::{Error as IoError, SeekFrom};
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
@@ -67,6 +67,31 @@ const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old";
const STARTUP_CLEANUP_WAIT_TIMEOUT: Duration = Duration::from_secs(2);
fn log_startup_disk_io_error(stage: &str, path: &Path, err: &IoError) {
warn!(
stage,
path = %path.display(),
error_kind = ?err.kind(),
raw_os_error = ?err.raw_os_error(),
error = ?err,
"local disk startup filesystem operation failed"
);
}
fn log_startup_disk_error(stage: &str, path: &Path, err: &DiskError) {
if let DiskError::Io(io_err) = err {
log_startup_disk_io_error(stage, path, io_err);
return;
}
warn!(
stage,
path = %path.display(),
error = ?err,
"local disk startup operation failed"
);
}
#[derive(Debug, Clone)]
pub struct FormatInfo {
pub id: Option<Uuid>,
@@ -409,9 +434,17 @@ fn resolve_local_disk_root(ep_path: &str) -> Result<PathBuf> {
impl LocalDisk {
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
debug!("Creating local disk");
let root = resolve_local_disk_root(&ep.get_file_path())?;
let endpoint_path = ep.get_file_path();
let root = resolve_local_disk_root(&endpoint_path).inspect_err(|err| {
log_startup_disk_error("resolve_local_disk_root", Path::new(&endpoint_path), err);
})?;
ensure_data_usage_layout(&root).await.map_err(DiskError::from)?;
ensure_data_usage_layout(&root)
.await
.map_err(DiskError::from)
.inspect_err(|err| {
log_startup_disk_error("ensure_data_usage_layout", &root, err);
})?;
let startup_cleanup_ready = Arc::new(AtomicU32::new(u32::from(!cleanup)));
let startup_cleanup_notify = Arc::new(Notify::new());
@@ -428,7 +461,9 @@ impl LocalDisk {
// Use optimized path resolution instead of absolutize_virtually
let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE);
debug!("format_path: {:?}", format_path);
let (format_data, format_meta) = read_file_exists(&format_path).await?;
let (format_data, format_meta) = read_file_exists(&format_path).await.inspect_err(|err| {
log_startup_disk_error("read_format_json", &format_path, err);
})?;
let mut id = None;
// let mut format_legacy = false;
@@ -521,7 +556,9 @@ impl LocalDisk {
startup_cleanup_notify,
exit_signal: None,
};
let (info, _root) = get_disk_info(root).await?;
let (info, _root) = get_disk_info(root.clone()).await.inspect_err(|err| {
log_startup_disk_error("get_disk_info", &root, err);
})?;
disk.major = info.major;
disk.minor = info.minor;
disk.fstype = info.fstype;
@@ -538,7 +575,9 @@ impl LocalDisk {
disk.rotational = true;
}
disk.make_meta_volumes().await?;
disk.make_meta_volumes().await.inspect_err(|err| {
log_startup_disk_error("make_meta_volumes", &disk.root, err);
})?;
let (exit_tx, exit_rx) = tokio::sync::broadcast::channel(1);
disk.exit_signal = Some(exit_tx);
@@ -587,16 +626,21 @@ impl LocalDisk {
let tmp_path = Self::meta_path(root, RUSTFS_META_TMP_BUCKET);
let tmp_old_path = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET).join(Uuid::new_v4().to_string());
rename_all(&tmp_path, &tmp_old_path, root).await?;
rename_all(&tmp_path, &tmp_old_path, root).await.inspect_err(|err| {
log_startup_disk_error("cleanup_tmp_rename_all", &tmp_path, err);
})?;
tokio::fs::create_dir_all(Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET)).await?;
let tmp_deleted_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET);
tokio::fs::create_dir_all(&tmp_deleted_path).await.inspect_err(|err| {
log_startup_disk_io_error("cleanup_tmp_create_deleted_dir", &tmp_deleted_path, err);
})?;
let tmp_old_root = Self::meta_path(root, RUSTFS_META_TMP_OLD_BUCKET);
tokio::spawn(async move {
if let Err(err) = tokio::fs::remove_dir_all(&tmp_old_root).await
&& err.kind() != ErrorKind::NotFound
{
warn!(path = ?tmp_old_root, error = ?err, "failed to remove old temporary data");
log_startup_disk_io_error("cleanup_tmp_remove_old_dir", &tmp_old_root, &err);
}
startup_cleanup_ready.store(1, Ordering::Release);
startup_cleanup_notify.notify_waiters();
@@ -1672,13 +1716,26 @@ async fn read_file_all(path: impl AsRef<Path>) -> Result<(Bytes, Metadata)> {
let p = path.as_ref();
let meta = read_file_metadata(&path).await?;
let data = fs::read(&p).await.map_err(to_file_error)?;
let data = fs::read(&p)
.await
.inspect_err(|err| {
log_startup_disk_io_error("read_file_all", p, err);
})
.map_err(to_file_error)?;
Ok((data.into(), meta))
}
async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
let meta = fs::metadata(&p).await.map_err(to_file_error)?;
let path = p.as_ref();
let meta = fs::metadata(path)
.await
.inspect_err(|err| {
if err.kind() != ErrorKind::NotFound {
log_startup_disk_io_error("read_file_metadata", path, err);
}
})
.map_err(to_file_error)?;
Ok(meta)
}
@@ -3299,7 +3356,9 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
let drive_path = drive_path.to_string_lossy().to_string();
check_path_length(&drive_path)?;
let disk_info = get_info(&drive_path)?;
let disk_info = get_info(&drive_path).inspect_err(|err| {
log_startup_disk_io_error("get_disk_info_stat", Path::new(&drive_path), err);
})?;
let root_drive = if !*GLOBAL_IsErasureSD.read().await {
let root_disk_threshold = *GLOBAL_RootDiskThreshold.read().await;
if root_disk_threshold > 0 {
rustfs/src/main.rs
+39 -2
View File
@@ -293,8 +293,22 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
set_global_addr(&config.address).await;
// For RPC
info!(
target: "rustfs::main::run",
server_address = %server_address,
volumes = ?config.volumes,
"starting endpoint parsing"
);
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), config.volumes.clone())
.await
.inspect_err(|err| {
error!(
target: "rustfs::main::run",
stage = "endpoint_parsing",
error = ?err,
"startup storage stage failed"
);
})
.map_err(Error::other)?;
enforce_unsupported_fs_policy(&endpoint_pools)?;
@@ -302,7 +316,21 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
update_erasure_type(setup_type).await;
// Initialize the local disk
init_local_disks(endpoint_pools.clone()).await.map_err(Error::other)?;
info!(
target: "rustfs::main::run",
"starting local disk initialization"
);
init_local_disks(endpoint_pools.clone())
.await
.inspect_err(|err| {
error!(
target: "rustfs::main::run",
stage = "local_disk_initialization",
error = ?err,
"startup storage stage failed"
);
})
.map_err(Error::other)?;
prewarm_local_disk_id_map().await;
// Initialize the lock clients
@@ -366,10 +394,19 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
// init store
// 2. Start Storage Engine (ECStore)
info!(
target: "rustfs::main::run",
"starting ECStore initialization"
);
let store = ECStore::new(server_addr, endpoint_pools.clone(), ctx.clone())
.await
.inspect_err(|err| {
error!("ECStore::new {:?}", err);
error!(
target: "rustfs::main::run",
stage = "ecstore_initialization",
error = ?err,
"startup storage stage failed"
);
})?;
ecconfig::init();