gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 16:37:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(config): recover from corrupt server config instead of crashing (#4229)

Browse Source
This commit is contained in:
Zhengchao An
2026-07-03 13:08:04 +08:00
committed by GitHub
parent ba1f162f0f
commit 1c36c6b208
5 changed files with 501 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/startup_storage.rs
+21 -4
View File
@@ -14,8 +14,9 @@
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use crate::storage_api::startup::storage::{
ECStore, EndpointServerPools, init_background_replication, init_ecstore_config, init_global_config_sys, init_local_disks,
init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, update_erasure_type,
ECStore, EndpointServerPools, global_config_init_error_is_deterministic, init_background_replication, init_ecstore_config,
init_global_config_sys, init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints,
try_migrate_server_config, update_erasure_type,
};
use rustfs_common::{GlobalReadiness, SystemStage};
use std::{
@@ -200,6 +201,20 @@ async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
let mut retry_count = 0;
while let Err(e) = init_global_config_sys(store.clone()).await {
if global_config_init_error_is_deterministic(&e) {
error!(
target: "rustfs::main::run",
event = EVENT_STARTUP_STORAGE_STAGE,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STORAGE,
stage = "global_config_initialization",
state = "failed",
retryable = false,
error = ?e,
"Global config initialization failed with a deterministic error, not retrying"
);
return Err(Error::other(format!("ecconfig::init_global_config_sys failed: {e}")));
}
let next_retry_count = retry_count + 1;
error!(
target: "rustfs::main::run",
@@ -212,10 +227,9 @@ async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
error = ?e,
"Global config initialization retry failed"
);
// TODO: check error type
retry_count = next_retry_count;
if global_config_retry_exhausted(retry_count) {
return Err(Error::other("ecconfig::init_global_config_sys failed"));
return Err(Error::other(format!("ecconfig::init_global_config_sys failed: {e}")));
}
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
}
@@ -229,6 +243,9 @@ async fn init_embedded_startup_storage_global_config(store: Arc<ECStore>) -> Res
let mut retry = 0;
while let Err(err) = init_global_config_sys(store.clone()).await {
if global_config_init_error_is_deterministic(&err) {
return Err(Error::other(format!("init_global_config_sys failed with a deterministic error: {err}")));
}
retry += 1;
if retry > GLOBAL_CONFIG_INIT_MAX_RETRIES {
return Err(Error::other(format!(
rustfs/src/storage/storage_api.rs
+7
View File
@@ -724,6 +724,13 @@ pub(crate) async fn init_global_config_sys(store: Arc<ECStore>) -> Result<()> {
ecstore_config::init_global_config_sys(store).await
}
/// Returns true when an `init_global_config_sys` failure is deterministic
/// (corrupt persisted server config with the recovery fallback disabled), so
/// retrying it cannot succeed and startup should fail fast.
pub(crate) fn global_config_init_error_is_deterministic(err: &Error) -> bool {
ecstore_config::com::is_server_config_corrupt_error(err)
}
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
ecstore_storage::init_local_disks(endpoint_pools).await
}
rustfs/src/storage_api.rs
+3 -3
View File
@@ -215,9 +215,9 @@ pub(crate) mod startup {
pub(crate) mod storage {
pub(crate) use crate::storage::storage_api::{
ECStore, EndpointServerPools, init_background_replication, init_compression_total_memory_from_backend,
init_ecstore_config, init_global_config_sys, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
set_global_endpoints, try_migrate_server_config, update_erasure_type,
ECStore, EndpointServerPools, global_config_init_error_is_deterministic, init_background_replication,
init_compression_total_memory_from_backend, init_ecstore_config, init_global_config_sys, init_local_disks,
init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, update_erasure_type,
};
}
}