gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-16 18:08:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(capacity): stop background schedulers on shutdown (#5797)

Browse Source
This commit is contained in:
cxymds
2026-08-07 19:02:42 +08:00
committed by GitHub
parent bd15dd5784
commit 2a44985037
11 changed files with 293 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/capacity/capacity_integration.rs
+9 -1
View File
@@ -14,7 +14,10 @@
//! Capacity management integration for application startup
use crate::capacity::{get_cached_capacity_with_metrics, init_capacity_management_for_local_disks};
use crate::capacity::{
get_cached_capacity_with_metrics, init_capacity_management_for_local_disks, init_capacity_management_for_local_disks_managed,
};
use rustfs_object_capacity::capacity_manager::CapacityBackgroundTasks;
/// Initialize capacity management system
/// This should be called during application startup after local disks are initialized
@@ -22,6 +25,11 @@ pub async fn init_capacity_management() {
init_capacity_management_for_local_disks().await;
}
/// Initialize capacity management with lifecycle ownership.
pub async fn init_capacity_management_managed() -> Option<CapacityBackgroundTasks> {
init_capacity_management_for_local_disks_managed().await
}
/// Get capacity statistics with metrics
#[allow(dead_code)]
pub async fn get_capacity_with_metrics() -> Option<(u64, String)> {
rustfs/src/capacity/mod.rs
+2 -1
View File
@@ -53,5 +53,6 @@ pub mod capacity_integration;
pub mod service;
pub use service::{
capacity_disk_ref, get_cached_capacity_with_metrics, init_capacity_management_for_local_disks, record_capacity_write,
capacity_disk_ref, get_cached_capacity_with_metrics, init_capacity_management_for_local_disks,
init_capacity_management_for_local_disks_managed, record_capacity_write,
};
rustfs/src/capacity/service.rs
+13 -3
View File
@@ -14,7 +14,10 @@
use crate::storage_api::capacity::service::{all_local_disk, disk_drive_path, disk_endpoint};
use rustfs_io_metrics::capacity_metrics::{record_capacity_cache_hit, record_capacity_cache_miss};
use rustfs_object_capacity::{CapacityDiskRef, capacity_manager};
use rustfs_object_capacity::{
CapacityDiskRef,
capacity_manager::{self, CapacityBackgroundTasks},
};
use tracing::{info, warn};
const LOG_COMPONENT_CAPACITY: &str = "capacity";
@@ -34,6 +37,12 @@ pub async fn record_capacity_write(scope_token: Option<uuid::Uuid>) {
}
pub async fn init_capacity_management_for_local_disks() {
if let Some(tasks) = init_capacity_management_for_local_disks_managed().await {
tasks.detach();
}
}
pub async fn init_capacity_management_for_local_disks_managed() -> Option<CapacityBackgroundTasks> {
info!(
component = LOG_COMPONENT_CAPACITY,
subsystem = LOG_SUBSYSTEM_CAPACITY,
@@ -52,7 +61,7 @@ pub async fn init_capacity_management_for_local_disks() {
reason = "no_local_disks",
"Capacity manager state changed"
);
return;
return None;
}
info!(
@@ -75,7 +84,7 @@ pub async fn init_capacity_management_for_local_disks() {
state = "starting_background_task",
"Capacity manager state changed"
);
capacity_manager::start_background_task(disk_refs).await;
let tasks = capacity_manager::start_background_tasks(disk_refs).await;
info!(
component = LOG_COMPONENT_CAPACITY,
@@ -84,6 +93,7 @@ pub async fn init_capacity_management_for_local_disks() {
state = "initialized",
"Capacity manager state changed"
);
Some(tasks)
}
pub async fn get_cached_capacity_with_metrics() -> Option<(u64, &'static str)> {
rustfs/src/startup_entrypoint.rs
+3
View File
@@ -144,6 +144,8 @@ async fn run(config: Config) -> Result<()> {
shutdown_token: ctx,
} = init_startup_storage_runtime(server_addr, &endpoint_pools, readiness.clone(), instance_ctx).await?;
let capacity_tasks = crate::capacity::capacity_integration::init_capacity_management_managed().await;
let service_runtime = init_startup_runtime_services(
&config,
endpoint_pools,
@@ -160,6 +162,7 @@ async fn run(config: Config) -> Result<()> {
state_manager,
s3_shutdown_tx,
console_shutdown_tx,
capacity_tasks,
service_runtime,
store,
shutdown_token: ctx,
rustfs/src/startup_lifecycle.rs
+4
View File
@@ -21,6 +21,7 @@ use crate::{
startup_shutdown::run_startup_shutdown_sequence,
};
use rustfs_common::GlobalReadiness;
use rustfs_object_capacity::capacity_manager::CapacityBackgroundTasks;
use rustfs_scanner::init_data_scanner;
use std::{
io::{Error, Result},
@@ -106,6 +107,7 @@ pub(crate) struct StartupRuntimeLifecycle {
pub(crate) state_manager: Arc<ServiceStateManager>,
pub(crate) s3_shutdown_tx: Option<ShutdownHandle>,
pub(crate) console_shutdown_tx: Option<ShutdownHandle>,
pub(crate) capacity_tasks: Option<CapacityBackgroundTasks>,
pub(crate) service_runtime: StartupServiceRuntime,
pub(crate) store: Arc<ECStore>,
pub(crate) shutdown_token: CancellationToken,
@@ -118,6 +120,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
state_manager,
s3_shutdown_tx,
console_shutdown_tx,
capacity_tasks,
service_runtime,
store,
shutdown_token,
@@ -155,6 +158,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
s3_shutdown_tx,
console_shutdown_tx,
optional_runtimes,
capacity_tasks,
shutdown_token,
)
.await;
rustfs/src/startup_server.rs
-2
View File
@@ -13,7 +13,6 @@
// limitations under the License.
use crate::{
capacity::capacity_integration::init_capacity_management,
config::Config,
server::{ServiceState, ServiceStateManager, ShutdownHandle, start_http_server},
startup_runtime_sources,
@@ -259,7 +258,6 @@ pub(crate) async fn init_startup_http_servers(
readiness: Arc<GlobalReadiness>,
server_ctx: Arc<ServerContextSlot>,
) -> Result<StartupHttpServers> {
init_capacity_management().await;
let state_manager = Arc::new(ServiceStateManager::new());
state_manager.update(ServiceState::Starting);
rustfs/src/startup_shutdown.rs
+58 -1
View File
@@ -14,7 +14,10 @@
use crate::storage_api::startup::shutdown::shutdown_background_services;
use crate::{
server::{ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier, stop_audit_system},
server::{
SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier,
stop_audit_system,
},
startup_optional_runtime_sidecars::{
OptionalRuntimeServices, prepare_optional_runtime_shutdowns, shutdown_optional_runtime_services,
},
@@ -22,6 +25,7 @@ use crate::{
};
use rustfs_heal::shutdown_ahm_services;
use rustfs_notify::NotificationLifecycleTransition;
use rustfs_object_capacity::capacity_manager::CapacityBackgroundTasks;
use rustfs_utils::get_env_bool_with_aliases;
use std::future::Future;
use std::path::PathBuf;
@@ -42,6 +46,7 @@ const EVENT_EMBEDDED_SERVER_STATE: &str = "embedded_server_state";
const EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED: &str = "embedded_shutdown_cleanup_failed";
const EVENT_SHUTDOWN_SIGNAL_RECEIVED: &str = "shutdown_signal_received";
const EVENT_BACKGROUND_SERVICE_SHUTDOWN: &str = "background_service_shutdown";
const BACKGROUND_SERVICE_CAPACITY: &str = "capacity";
const EVENT_EVENT_NOTIFIER_SHUTDOWN: &str = "event_notifier_shutdown";
const EVENT_PROFILING_SHUTDOWN: &str = "profiling_shutdown";
const EVENT_SERVER_SHUTDOWN_STATE: &str = "server_shutdown_state";
@@ -200,6 +205,7 @@ pub(crate) async fn run_startup_shutdown_sequence(
s3_shutdown_handle: Option<ShutdownHandle>,
console_shutdown_handle: Option<ShutdownHandle>,
optional_runtimes: OptionalRuntimeServices,
capacity_tasks: Option<CapacityBackgroundTasks>,
ctx: CancellationToken,
) {
ctx.cancel();
@@ -220,6 +226,57 @@ pub(crate) async fn run_startup_shutdown_sequence(
);
state_manager.update(ServiceState::Stopping);
if let Some(handle) = &s3_shutdown_handle {
handle.signal();
}
if let Some(handle) = &console_shutdown_handle {
handle.signal();
}
if let Some(capacity_tasks) = capacity_tasks {
info!(
target: "rustfs::main::handle_shutdown",
event = EVENT_BACKGROUND_SERVICE_SHUTDOWN,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
service = BACKGROUND_SERVICE_CAPACITY,
state = "stopping",
"Background service shutdown started"
);
match tokio::time::timeout(SHUTDOWN_TIMEOUT, capacity_tasks.shutdown()).await {
Ok(Ok(())) => info!(
target: "rustfs::main::handle_shutdown",
event = EVENT_BACKGROUND_SERVICE_SHUTDOWN,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
service = BACKGROUND_SERVICE_CAPACITY,
state = "stopped",
"Background service shutdown completed"
),
Ok(Err(err)) => error!(
target: "rustfs::main::handle_shutdown",
event = EVENT_BACKGROUND_SERVICE_SHUTDOWN,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
service = BACKGROUND_SERVICE_CAPACITY,
state = "stop_failed",
reason = join_failure_reason(&err),
"Background service shutdown failed"
),
Err(_) => error!(
target: "rustfs::main::handle_shutdown",
event = EVENT_BACKGROUND_SERVICE_SHUTDOWN,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
service = BACKGROUND_SERVICE_CAPACITY,
state = "stop_failed",
reason = "timeout",
timeout_secs = SHUTDOWN_TIMEOUT.as_secs(),
"Background service shutdown timed out"
),
}
}
let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true);
let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true);