From e9037f9eb09fcbb31d980efac8b8e6b8f12bcaef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Sat, 20 Jun 2026 01:15:39 +0800 Subject: [PATCH] refactor: extract startup shutdown lifecycle boundary (#3634) --- docs/architecture/migration-progress.md | 49 +++++- docs/architecture/runtime-lifecycle.md | 8 + docs/architecture/startup-timeline.md | 8 +- rustfs/src/lib.rs | 1 + rustfs/src/startup_services.rs | 200 +--------------------- rustfs/src/startup_shutdown.rs | 216 ++++++++++++++++++++++++ 6 files changed, 276 insertions(+), 206 deletions(-) create mode 100644 rustfs/src/startup_shutdown.rs diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 9e3fe60e1..9347a0433 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,15 +5,15 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-optional-runtime-start-boundary` -- Baseline: latest `origin/main` - (`74fcfddd7fd98d69358e02908617261d962725d1`). +- Branch: `overtrue/arch-startup-shutdown-lifecycle-boundary` +- Baseline: `origin/main` + (`eec1792c7bd1bf650dbb3c46d7873db6fed6150f`). - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: move optional protocol startup and shutdown ownership from - `startup_services` into the optional runtime boundary. +- Rust code changes: move startup shutdown sequencing from `startup_services` + into a dedicated shutdown lifecycle boundary. - CI/script changes: none. -- Docs changes: record the R-022 optional runtime startup boundary slice. +- Docs changes: record the R-023 startup shutdown lifecycle boundary slice. ## Phase 0 Tasks @@ -2046,6 +2046,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block risk scan, branch freshness check, pre-commit quality gate, and three-expert review. +- [x] `R-023` Extract startup shutdown lifecycle boundary. + - Do: add `startup_shutdown` and move runtime token cancellation, service + state transitions, background shutdown, notifier/audit/profiling shutdown, + HTTP shutdown, and optional runtime wait sequencing out of + `startup_services`. + - Acceptance: shutdown order stays runtime token cancellation, `Stopping` + state, scanner/AHM shutdown, optional runtime shutdown planning, + notifier/audit/profiling shutdown, S3 and console HTTP shutdown, optional + runtime waits, then `Stopped` state. + - Must preserve: service state transitions, readiness state behavior, + scanner/heal enable flag handling, notifier/audit/profiling shutdown logs, + HTTP shutdown ordering, optional protocol shutdown ordering, and fatal + boundaries. + - Verification: focused shutdown/service/optional runtime tests, RustFS lib + check, migration/layer guards, formatting, diff hygiene, Rust risk scan, + branch freshness check, pre-commit quality gate, and three-expert review. + ## Next PRs 1. `pure-move`: continue larger lifecycle hook slices for optional runtime @@ -2057,9 +2074,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | R-022 keeps protocol implementation details in `startup_protocols` and moves optional runtime startup ownership into `startup_optional_runtimes`. | -| Migration preservation | passed | KMS-before-protocol ordering, protocol fatal boundary, protocol startup order, shutdown order, readiness, and HTTP shutdown behavior remain unchanged. | -| Testing/verification | passed | Focused optional runtime, protocol, and startup service tests plus final checks are tracked below before push. | +| Quality/architecture | passed | R-023 keeps startup orchestration in `startup_services` and moves shutdown sequencing into `startup_shutdown`. | +| Migration preservation | passed | Runtime token cancellation, background shutdown, optional runtime planning, notifier/audit/profiling shutdown, HTTP shutdown, optional runtime waits, and service state transitions remain ordered. | +| Testing/verification | passed | Focused shutdown, optional runtime, and startup service tests plus final checks are tracked below before push. | ## Verification Notes @@ -2114,6 +2131,20 @@ Passed before push: - `make pre-commit`: passed. - Three-expert review: passed. +- Issue #660 R-023 current slice: + - `cargo test -p rustfs --lib startup_shutdown -- --nocapture`: passed. + - `cargo test -p rustfs --lib startup_services -- --nocapture`: passed. + - `cargo test -p rustfs --lib startup_optional_runtimes -- --nocapture`: + passed. + - `cargo check -p rustfs --lib`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Rust risk scan on changed Rust files: passed. + - `make pre-commit`: passed. + - Three-expert review: passed. + - Issue #660 R-020 current slice: - `cargo test -p rustfs --lib startup_profiling -- --nocapture`: passed. - `cargo check -p rustfs --lib`: passed. diff --git a/docs/architecture/runtime-lifecycle.md b/docs/architecture/runtime-lifecycle.md index 97b546eaa..dbe0d61db 100644 --- a/docs/architecture/runtime-lifecycle.md +++ b/docs/architecture/runtime-lifecycle.md @@ -31,6 +31,14 @@ Disallowed responsibilities: - Hide globals behind a service-locator API. - Change startup side effects while moving code. +## Shutdown Lifecycle Boundary + +`startup_shutdown` owns the main shutdown sequence after the process receives a +shutdown signal. Startup modules may pass handles into this boundary, but they +must not reorder runtime-token cancellation, background service shutdown, +optional runtime shutdown planning, notifier/audit/profiling shutdown, HTTP +shutdown, optional runtime waits, or final service-state publication. + ## Optional Runtime Boundary `startup_optional_runtimes` owns startup and shutdown handoff for optional diff --git a/docs/architecture/startup-timeline.md b/docs/architecture/startup-timeline.md index f80f548f5..d7afe9507 100644 --- a/docs/architecture/startup-timeline.md +++ b/docs/architecture/startup-timeline.md @@ -69,11 +69,11 @@ new startup semantics. | Step | Source | Current action | Side effects | Fatal boundary | Ready stage | |---|---|---|---|---|---| -| `STOP-001` | `rustfs/src/startup_services.rs` | Cancel runtime token and move service state to `Stopping`. | Notifies cancellation-aware background tasks. | Non-fatal. | Service state moves to `Stopping`; readiness stages are not cleared here | -| `STOP-002` | `rustfs/src/startup_services.rs` | Stop scanner/background services and AHM services according to enable flags. | Calls ECStore background shutdown and heal/scanner shutdown helpers. | Non-fatal in this path. | No readiness-stage change | +| `STOP-001` | `rustfs/src/startup_shutdown.rs` | Cancel runtime token and move service state to `Stopping`. | Notifies cancellation-aware background tasks. | Non-fatal. | Service state moves to `Stopping`; readiness stages are not cleared here | +| `STOP-002` | `rustfs/src/startup_shutdown.rs` | Stop scanner/background services and AHM services according to enable flags. | Calls ECStore background shutdown and heal/scanner shutdown helpers. | Non-fatal in this path. | No readiness-stage change | | `STOP-003` | `rustfs/src/startup_optional_runtimes.rs` | Plan optional runtime shutdown and log stopping state for FTP/FTPS/WebDAV/SFTP protocol servers. | Collects protocol shutdown handles. | Non-fatal in this path. | No readiness-stage change | -| `STOP-004` | `rustfs/src/startup_services.rs` and `rustfs/src/startup_profiling.rs` | Stop event notifier, audit system, and profiling tasks. | Stops notifier and profiling tasks; audit stop failures are logged. | Non-fatal in this path. | No readiness-stage change | -| `STOP-005` | `rustfs/src/startup_services.rs` and `rustfs/src/startup_optional_runtimes.rs` | Stop S3 and console HTTP servers, signal and wait for optional protocol shutdowns, then mark service state `Stopped`. | HTTP shutdown happens after notifier/audit/profiling shutdown in current order. | Join failures are logged by shutdown handles; this path does not return errors. | Service state moves to `Stopped`; readiness stages are not cleared here | +| `STOP-004` | `rustfs/src/startup_shutdown.rs` and `rustfs/src/startup_profiling.rs` | Stop event notifier, audit system, and profiling tasks. | Stops notifier and profiling tasks; audit stop failures are logged. | Non-fatal in this path. | No readiness-stage change | +| `STOP-005` | `rustfs/src/startup_shutdown.rs` and `rustfs/src/startup_optional_runtimes.rs` | Stop S3 and console HTTP servers, signal and wait for optional protocol shutdowns, then mark service state `Stopped`. | HTTP shutdown happens after notifier/audit/profiling shutdown in current order. | Join failures are logged by shutdown handles; this path does not return errors. | Service state moves to `Stopped`; readiness stages are not cleared here | ## Migration Rules diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index 939e4b33e..10e16aff6 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -78,6 +78,7 @@ pub mod startup_protocols; pub mod startup_runtime; pub mod startup_server; pub mod startup_services; +pub mod startup_shutdown; pub mod startup_storage; pub mod storage; pub(crate) mod storage_compat; diff --git a/rustfs/src/startup_services.rs b/rustfs/src/startup_services.rs index 7828dd679..1f9433337 100644 --- a/rustfs/src/startup_services.rs +++ b/rustfs/src/startup_services.rs @@ -14,7 +14,7 @@ use crate::storage_compat::{ ECStore, EndpointServerPools, get_global_replication_pool, init_bucket_metadata_sys, new_global_notification_sys, - shutdown_background_services, try_migrate_bucket_metadata, try_migrate_iam_config, + try_migrate_bucket_metadata, try_migrate_iam_config, }; use crate::{ config::Config, @@ -22,21 +22,14 @@ use crate::{ add_bucket_notification_configuration, init_auto_tuner, init_buffer_profile_system, init_kms_system, init_update_check, print_server_info, }, - server::{ - ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, init_event_notifier, shutdown_event_notifier, - start_audit_system, stop_audit_system, wait_for_shutdown, - }, + server::{ServiceStateManager, ShutdownHandle, init_event_notifier, start_audit_system, wait_for_shutdown}, startup_iam::{IamBootstrapDisposition, bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap}, - startup_optional_runtimes::{ - OptionalRuntimeServices, init_optional_runtime_services, prepare_optional_runtime_shutdowns, - shutdown_optional_runtime_services, - }, + startup_optional_runtimes::{OptionalRuntimeServices, init_optional_runtime_services}, + startup_shutdown::run_startup_shutdown_sequence, }; use rustfs_audit::AuditResult; use rustfs_common::GlobalReadiness; -use rustfs_heal::{ - create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services, -}; +use rustfs_heal::{create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager}; use rustfs_iam::init_oidc_sys; use rustfs_obs::init_metrics_runtime; use rustfs_scanner::init_data_scanner; @@ -65,10 +58,6 @@ const EVENT_OIDC_INITIALIZATION_FAILED: &str = "oidc_initialization_failed"; const EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED: &str = "notification_system_initialization_failed"; const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured"; const EVENT_SERVER_READY: &str = "server_ready"; -const EVENT_SHUTDOWN_SIGNAL_RECEIVED: &str = "shutdown_signal_received"; -const EVENT_BACKGROUND_SERVICE_SHUTDOWN: &str = "background_service_shutdown"; -const EVENT_EVENT_NOTIFIER_SHUTDOWN: &str = "event_notifier_shutdown"; -const EVENT_PROFILING_SHUTDOWN: &str = "profiling_shutdown"; const EVENT_SERVER_SHUTDOWN_STATE: &str = "server_shutdown_state"; pub struct StartupServiceRuntime { @@ -154,7 +143,7 @@ pub async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifecycle) - } let shutdown_signal = wait_for_shutdown().await; - handle_shutdown( + run_startup_shutdown_sequence( &state_manager, shutdown_signal, s3_shutdown_tx, @@ -356,165 +345,6 @@ async fn init_observability_runtime(ctx: CancellationToken) { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum BackgroundShutdownStep { - DataScanner, - Ahm, -} - -fn background_shutdown_steps(enable_scanner: bool, enable_heal: bool) -> Vec { - let mut steps = Vec::with_capacity(2); - if enable_scanner { - steps.push(BackgroundShutdownStep::DataScanner); - } - if enable_heal || enable_scanner { - steps.push(BackgroundShutdownStep::Ahm); - } - steps -} - -async fn handle_shutdown( - state_manager: &ServiceStateManager, - shutdown_signal: ShutdownSignal, - s3_shutdown_handle: Option, - console_shutdown_handle: Option, - optional_runtimes: OptionalRuntimeServices, - ctx: CancellationToken, -) { - ctx.cancel(); - - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_SHUTDOWN_SIGNAL_RECEIVED, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - signal = shutdown_signal.log_label(), - "Shutdown signal received" - ); - state_manager.update(ServiceState::Stopping); - - let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true); - let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true); - - let background_steps = background_shutdown_steps(enable_scanner, enable_heal); - for step in &background_steps { - match step { - BackgroundShutdownStep::DataScanner => { - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_BACKGROUND_SERVICE_SHUTDOWN, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - service = "data_scanner", - state = "stopping", - "Background service shutdown started" - ); - shutdown_background_services(); - } - BackgroundShutdownStep::Ahm => { - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_BACKGROUND_SERVICE_SHUTDOWN, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - service = "ahm", - state = "stopping", - "Background service shutdown started" - ); - shutdown_ahm_services(); - } - } - } - - if background_steps.is_empty() { - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_BACKGROUND_SERVICE_SHUTDOWN, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - service = "ahm", - state = "skipped", - reason = "disabled", - "Background service shutdown skipped" - ); - } - - let optional_runtime_shutdowns = prepare_optional_runtime_shutdowns(optional_runtimes); - - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_EVENT_NOTIFIER_SHUTDOWN, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - state = "stopping", - "Event notifier shutdown started" - ); - shutdown_event_notifier().await; - - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_AUDIT_SYSTEM_STATE, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - state = "stopping", - "Audit runtime stopping" - ); - match stop_audit_system().await { - Ok(_) => info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_AUDIT_SYSTEM_STATE, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - state = "stopped", - "Audit runtime stopped" - ), - Err(e) => error!( - target: "rustfs::main::handle_shutdown", - event = EVENT_AUDIT_SYSTEM_STATE, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - state = "stop_failed", - error = %e, - "Audit runtime failed to stop" - ), - } - - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_PROFILING_SHUTDOWN, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - state = "stopping", - "Profiling shutdown started" - ); - crate::startup_profiling::shutdown_profiling_runtime(); - - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_SERVER_SHUTDOWN_STATE, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - state = "stopping", - "RustFS server stopping" - ); - if let Some(s3_shutdown_handle) = s3_shutdown_handle { - s3_shutdown_handle.shutdown().await; - } - if let Some(console_shutdown_handle) = console_shutdown_handle { - console_shutdown_handle.shutdown().await; - } - shutdown_optional_runtime_services(optional_runtime_shutdowns).await; - state_manager.update(ServiceState::Stopped); - info!( - target: "rustfs::main::handle_shutdown", - event = EVENT_SERVER_SHUTDOWN_STATE, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - state = "stopped", - "RustFS server stopped" - ); -} - pub async fn init_event_notifier_and_audit() -> AuditResult<()> { init_event_notifier_and_audit_with(init_event_notifier, start_audit_system).await } @@ -547,9 +377,7 @@ where #[cfg(test)] mod tests { - use super::{ - BackgroundShutdownStep, background_shutdown_steps, init_event_notifier_and_audit_with, init_notification_system_with, - }; + use super::{init_event_notifier_and_audit_with, init_notification_system_with}; use rustfs_audit::AuditError; use std::sync::{Arc, Mutex}; @@ -602,18 +430,4 @@ mod tests { assert!(result.is_err()); } - - #[test] - fn background_shutdown_plan_keeps_scanner_before_ahm() { - assert_eq!( - background_shutdown_steps(true, true), - vec![BackgroundShutdownStep::DataScanner, BackgroundShutdownStep::Ahm] - ); - assert_eq!( - background_shutdown_steps(true, false), - vec![BackgroundShutdownStep::DataScanner, BackgroundShutdownStep::Ahm] - ); - assert_eq!(background_shutdown_steps(false, true), vec![BackgroundShutdownStep::Ahm]); - assert!(background_shutdown_steps(false, false).is_empty()); - } } diff --git a/rustfs/src/startup_shutdown.rs b/rustfs/src/startup_shutdown.rs new file mode 100644 index 000000000..62494e5d8 --- /dev/null +++ b/rustfs/src/startup_shutdown.rs @@ -0,0 +1,216 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + server::{ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier, stop_audit_system}, + startup_optional_runtimes::{ + OptionalRuntimeServices, prepare_optional_runtime_shutdowns, shutdown_optional_runtime_services, + }, + storage_compat::shutdown_background_services, +}; +use rustfs_heal::shutdown_ahm_services; +use rustfs_utils::get_env_bool_with_aliases; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED"; +const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER"; +const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED"; +const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL"; +const LOG_COMPONENT_MAIN: &str = "main"; +const LOG_SUBSYSTEM_STARTUP: &str = "startup"; +const EVENT_AUDIT_SYSTEM_STATE: &str = "audit_system_state"; +const EVENT_SHUTDOWN_SIGNAL_RECEIVED: &str = "shutdown_signal_received"; +const EVENT_BACKGROUND_SERVICE_SHUTDOWN: &str = "background_service_shutdown"; +const EVENT_EVENT_NOTIFIER_SHUTDOWN: &str = "event_notifier_shutdown"; +const EVENT_PROFILING_SHUTDOWN: &str = "profiling_shutdown"; +const EVENT_SERVER_SHUTDOWN_STATE: &str = "server_shutdown_state"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BackgroundShutdownStep { + DataScanner, + Ahm, +} + +fn background_shutdown_steps(enable_scanner: bool, enable_heal: bool) -> Vec { + let mut steps = Vec::with_capacity(2); + if enable_scanner { + steps.push(BackgroundShutdownStep::DataScanner); + } + if enable_heal || enable_scanner { + steps.push(BackgroundShutdownStep::Ahm); + } + steps +} + +pub async fn run_startup_shutdown_sequence( + state_manager: &ServiceStateManager, + shutdown_signal: ShutdownSignal, + s3_shutdown_handle: Option, + console_shutdown_handle: Option, + optional_runtimes: OptionalRuntimeServices, + ctx: CancellationToken, +) { + ctx.cancel(); + + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_SHUTDOWN_SIGNAL_RECEIVED, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + signal = shutdown_signal.log_label(), + "Shutdown signal received" + ); + state_manager.update(ServiceState::Stopping); + + let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true); + let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true); + + let background_steps = background_shutdown_steps(enable_scanner, enable_heal); + for step in &background_steps { + match step { + BackgroundShutdownStep::DataScanner => { + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_BACKGROUND_SERVICE_SHUTDOWN, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + service = "data_scanner", + state = "stopping", + "Background service shutdown started" + ); + shutdown_background_services(); + } + BackgroundShutdownStep::Ahm => { + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_BACKGROUND_SERVICE_SHUTDOWN, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + service = "ahm", + state = "stopping", + "Background service shutdown started" + ); + shutdown_ahm_services(); + } + } + } + + if background_steps.is_empty() { + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_BACKGROUND_SERVICE_SHUTDOWN, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + service = "ahm", + state = "skipped", + reason = "disabled", + "Background service shutdown skipped" + ); + } + + let optional_runtime_shutdowns = prepare_optional_runtime_shutdowns(optional_runtimes); + + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_EVENT_NOTIFIER_SHUTDOWN, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stopping", + "Event notifier shutdown started" + ); + shutdown_event_notifier().await; + + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_AUDIT_SYSTEM_STATE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stopping", + "Audit runtime stopping" + ); + match stop_audit_system().await { + Ok(_) => info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_AUDIT_SYSTEM_STATE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stopped", + "Audit runtime stopped" + ), + Err(e) => error!( + target: "rustfs::main::handle_shutdown", + event = EVENT_AUDIT_SYSTEM_STATE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stop_failed", + error = %e, + "Audit runtime failed to stop" + ), + } + + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_PROFILING_SHUTDOWN, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stopping", + "Profiling shutdown started" + ); + crate::startup_profiling::shutdown_profiling_runtime(); + + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_SERVER_SHUTDOWN_STATE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stopping", + "RustFS server stopping" + ); + if let Some(s3_shutdown_handle) = s3_shutdown_handle { + s3_shutdown_handle.shutdown().await; + } + if let Some(console_shutdown_handle) = console_shutdown_handle { + console_shutdown_handle.shutdown().await; + } + shutdown_optional_runtime_services(optional_runtime_shutdowns).await; + state_manager.update(ServiceState::Stopped); + info!( + target: "rustfs::main::handle_shutdown", + event = EVENT_SERVER_SHUTDOWN_STATE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stopped", + "RustFS server stopped" + ); +} + +#[cfg(test)] +mod tests { + use super::{BackgroundShutdownStep, background_shutdown_steps}; + + #[test] + fn background_shutdown_plan_keeps_scanner_before_ahm() { + assert_eq!( + background_shutdown_steps(true, true), + vec![BackgroundShutdownStep::DataScanner, BackgroundShutdownStep::Ahm] + ); + assert_eq!( + background_shutdown_steps(true, false), + vec![BackgroundShutdownStep::DataScanner, BackgroundShutdownStep::Ahm] + ); + assert_eq!(background_shutdown_steps(false, true), vec![BackgroundShutdownStep::Ahm]); + assert!(background_shutdown_steps(false, false).is_empty()); + } +}