From 513f06f2683116d7519b72ab69f249febd684a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Mon, 15 Jun 2026 08:32:44 +0800 Subject: [PATCH] refactor: centralize startup server preflight (#3459) --- crates/obs/src/logging.rs | 2 +- docs/architecture/migration-progress.md | 71 ++++++---- rustfs/src/lib.rs | 1 + rustfs/src/main.rs | 118 ++-------------- rustfs/src/startup_preflight.rs | 174 ++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 137 deletions(-) create mode 100644 rustfs/src/startup_preflight.rs diff --git a/crates/obs/src/logging.rs b/crates/obs/src/logging.rs index 6845c2f12..8786d1e5c 100644 --- a/crates/obs/src/logging.rs +++ b/crates/obs/src/logging.rs @@ -178,7 +178,7 @@ mod tests { "fn emit_fatal_stderr(context: &str, error: impl std::fmt::Display)", "emit_fatal_stderr(\"Server runtime failed\", e)", "emit_fatal_stderr(\"Command parse failed\", e)", - "emit_fatal_stderr(\"Observability initialization failed\", &e)", + "emit_fatal_stderr(\"Observability initialization failed\", err)", ], ); } diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 09e99e824..c54066b85 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,18 +5,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-startup-core-runtime-bootstrap` -- Baseline: `origin/main` at `73e534682e93d1ed2c7b0e752ad550d969ca96b2` +- Branch: `overtrue/arch-startup-server-preflight` +- Baseline: `origin/main` at `2288b1bb634bda8a06bf7c19780b0318473dbe33` - PR type for this branch: `pure-move` -- Runtime behavior changes: no external behavior change expected; startup - runtime foundation still logs dial9/license state, prints the logo, initializes - profiling and trusted proxies, installs the rustls provider, and publishes - outbound TLS material in the same order and with the same fatal boundary. -- Rust code changes: centralize startup runtime foundation bootstrap in - `startup_runtime::init_startup_runtime_foundation` and use it from binary +- Runtime behavior changes: no external behavior change expected; external + prefix compatibility reporting, config snapshot initialization, license + initialization, observability guard initialization/storage, and startup + runtime foundation bootstrap still run in the same order and with the same + fatal boundaries. +- Rust code changes: centralize server startup preflight bootstrap in + `startup_preflight::init_startup_server_preflight` and use it from binary startup. - CI/script changes: none. -- Docs changes: record `R-012` startup runtime foundation bootstrap progress and +- Docs changes: record `R-013` startup server preflight bootstrap progress and verification. ## Phase 0 Tasks @@ -609,6 +610,23 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block formatting, migration guards, Rust risk scan, branch freshness check, and pre-commit quality gate. +- [x] `R-013` Centralize startup server preflight bootstrap. + - Do: move external-prefix compatibility reporting, config snapshot + initialization, runtime license initialization, observability guard + initialization/storage, and startup runtime foundation bootstrap behind + `startup_preflight::init_startup_server_preflight`. + - Acceptance: env compatibility is applied before command parsing and reported + after observability starts, config snapshot and license init happen before + runtime foundation, observability init failure still emits the dedicated + fatal stderr and sentinel, guard storage failure still returns the original + error, and runtime foundation ordering/fatal boundaries stay unchanged. + - Must preserve: env compat conflict/applied events, observability guard + set/failure events, startup order, fatal stderr suppression sentinel, and + existing command/subcommand behavior. + - Verification: focused startup preflight tests, binary/lib compile checks, + formatting, migration guards, Rust risk scan, branch freshness check, and + pre-commit quality gate. + ## Next PRs 1. `pure-move`: continue extracting startup boot wrappers in larger slices while @@ -620,38 +638,39 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | Pure-move slice removes startup runtime foundation details from binary startup without changing ownership or adding abstraction beyond the existing startup module pattern. | -| Migration preservation | passed | BOOT-006 order, dial9/license log fields, profiling/proxy/provider non-fatal setup, TLS fatal boundary, generation saturation, and TLS metric condition are preserved. | -| Testing/verification | passed | Focused startup runtime tests, compile checks, formatting, migration/layer guards, Rust risk scan, branch freshness check, and full `make pre-commit` passed. | +| Quality/architecture | passed | Pure-move slice removes server preflight details from binary startup behind the existing startup module pattern; the logging governance test was updated only to match the new fatal-stderr call shape. | +| Migration preservation | passed | Env compatibility apply/report order, config snapshot, license init, observability init/store, fatal stderr sentinel, guard-store error return, and runtime foundation order are preserved. | +| Testing/verification | passed | Focused startup preflight and logging-governance tests, compile checks, formatting, migration/layer guards, Rust risk scan, branch freshness check, and full `make pre-commit` passed. | ## Verification Notes -Passed on `73e534682e93d1ed2c7b0e752ad550d969ca96b2`: +Passed on `2288b1bb634bda8a06bf7c19780b0318473dbe33`: -- `cargo test -p rustfs startup_runtime --no-fail-fast`. -- `cargo check -p rustfs --lib`. -- `cargo check -p rustfs --bin rustfs`. -- `cargo fmt --all --check`. -- `git diff --check`. -- `./scripts/check_architecture_migration_rules.sh`. -- `./scripts/check_layer_dependencies.sh`. +- `cargo test -p rustfs startup_preflight --no-fail-fast`: passed. +- `cargo test -p rustfs-obs startup_fatal_stderr_uses_single_formatter_for_pre_observability_failures --no-fail-fast`: passed. +- `cargo check -p rustfs --lib`: passed. +- `cargo check -p rustfs --bin rustfs`: passed. +- `cargo fmt --all --check`: passed. +- `git diff --check`: passed. +- `./scripts/check_architecture_migration_rules.sh`: passed. +- `./scripts/check_layer_dependencies.sh`: passed. - `git rev-list --left-right --count HEAD...origin/main` returned `0 0`. - Added-line Rust risk scan for changed Rust files: no matches. - Full-file risk scan for changed Rust files: matches are existing docs example - output and existing binary startup stderr/expect usage. -- `make pre-commit`: all checks passed, including nextest with 6002 passed and + output and existing binary startup alias/stderr/expect usage. +- `make pre-commit`: all checks passed, including nextest with 6003 passed and 111 skipped, plus doctests. Notes: -- This slice centralizes startup runtime foundation bootstrap without changing +- This slice centralizes server startup preflight bootstrap without changing startup ordering or fatal boundaries. -- Observability initialization remains in binary startup so early fatal stderr - handling stays unchanged. +- Observability initialization failure still maps to the dedicated fatal stderr + sentinel in binary startup. ## Handoff Notes -- R-012 is complete. +- R-013 is complete. - Next startup slices can keep using larger pure moves, but must keep startup ordering, fatal/non-fatal boundaries, shutdown ownership, and readiness ownership explicit in tests. diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index ea253bfc9..669bbbe6d 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -69,6 +69,7 @@ pub mod protocols; pub mod server; pub mod startup_fs_guard; pub mod startup_iam; +pub mod startup_preflight; pub mod startup_protocols; pub mod startup_runtime; pub mod startup_services; diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index a7660fbaa..75e15d914 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -19,15 +19,14 @@ use rustfs::init::{ use futures_util::future::join_all; use rustfs::capacity::capacity_integration::init_capacity_management; -use rustfs::license::init_license; use rustfs::server::{ ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier, start_http_server, stop_audit_system, wait_for_shutdown, }; use rustfs::startup_fs_guard::enforce_unsupported_fs_policy; use rustfs::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap}; +use rustfs::startup_preflight::{StartupServerPreflightError, bootstrap_external_prefix_compat, init_startup_server_preflight}; use rustfs::startup_protocols::{ProtocolShutdownSenders, init_protocol_shutdown_senders}; -use rustfs::startup_runtime::init_startup_runtime_foundation; use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr}; use rustfs_credentials::init_global_action_credentials; use rustfs_ecstore::store::init_lock_clients; @@ -49,12 +48,10 @@ use rustfs_heal::{ create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services, }; use rustfs_iam::init_oidc_sys; -use rustfs_obs::{init_metrics_runtime, init_obs, set_global_guard}; +use rustfs_obs::init_metrics_runtime; use rustfs_scanner::init_data_scanner; use rustfs_storage_api::BucketOptions; -use rustfs_utils::{ - ExternalEnvCompatReport, apply_external_env_compat, get_env_bool_with_aliases, net::parse_and_resolve_address, -}; +use rustfs_utils::{get_env_bool_with_aliases, net::parse_and_resolve_address}; use std::io::{Error, Result}; use std::sync::Arc; use tokio_util::sync::CancellationToken; @@ -68,16 +65,12 @@ const LOG_COMPONENT_MAIN: &str = "main"; const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const LOG_SUBSYSTEM_AUTH: &str = "auth"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; -const EVENT_EXTERNAL_ENV_COMPAT_CONFLICT: &str = "external_env_compat_conflict"; -const EVENT_EXTERNAL_ENV_COMPAT_APPLIED: &str = "external_env_compat_applied"; -const EVENT_OBSERVABILITY_GUARD_SET_FAILED: &str = "observability_guard_set_failed"; const EVENT_DEFAULT_CREDENTIALS_DETECTED: &str = "default_credentials_detected"; const EVENT_SERVER_CONFIG_SANITIZED: &str = "server_config_sanitized"; const EVENT_SERVER_STARTING: &str = "server_starting"; const EVENT_SERVER_RUNTIME_FAILED: &str = "server_runtime_failed"; const EVENT_ACTION_CREDENTIALS_INITIALIZED: &str = "action_credentials_initialized"; const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed"; -const EVENT_OBSERVABILITY_GUARD_SET: &str = "observability_guard_set"; const EVENT_ENDPOINT_PARSING_STARTED: &str = "endpoint_parsing_started"; const EVENT_STARTUP_STORAGE_STAGE: &str = "startup_storage_stage"; const EVENT_STORAGE_POOL_FORMATTING: &str = "storage_pool_formatting"; @@ -126,20 +119,6 @@ fn main() { } } -fn bootstrap_external_prefix_compat() -> Result { - let env_compat_report = apply_external_env_compat(); - Ok(env_compat_report) -} - -fn format_external_prefix_mappings(report: &ExternalEnvCompatReport) -> String { - report - .mapped_pairs - .iter() - .map(|(source_key, rustfs_key)| format!("{source_key}->{rustfs_key}")) - .collect::>() - .join(", ") -} - fn is_using_default_credentials(config: &rustfs::config::Config) -> bool { config.is_using_default_credentials() } @@ -169,50 +148,16 @@ async fn async_main() -> Result<()> { rustfs::config::CommandResult::Server(config) => config, }; - // Initialize the global config snapshot for info command - rustfs::config::init_config_snapshot(&config); - - // Initialize the configuration - init_license(config.license.clone()); - - // Initialize Observability - let guard = match init_obs(Some(config.clone().obs_endpoint)).await { - Ok(g) => g, - Err(e) => { + match init_startup_server_preflight(&config, &env_compat_report).await { + Ok(()) => {} + Err(StartupServerPreflightError::ObservabilityInit(err)) => { // Structured logging is unavailable until observability initializes. - emit_fatal_stderr("Observability initialization failed", &e); + emit_fatal_stderr("Observability initialization failed", err); return Err(Error::other(OBSERVABILITY_INIT_FATAL_ALREADY_REPORTED)); } - }; - - // Store in global storage - match set_global_guard(guard).map_err(Error::other) { - Ok(_) => { - debug!( - target: "rustfs::main", - event = EVENT_OBSERVABILITY_GUARD_SET, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - result = "ok", - "Stored global observability guard" - ); - } - Err(e) => { - error!( - target: "rustfs::main", - event = EVENT_OBSERVABILITY_GUARD_SET_FAILED, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - error = %e, - "Failed to store global observability guard" - ); - return Err(e); - } + Err(StartupServerPreflightError::Other(err)) => return Err(err), } - log_external_prefix_compat_report(&env_compat_report); - init_startup_runtime_foundation(&config).await?; - // Run parameters match run(*config).await { Ok(_) => Ok(()), @@ -724,32 +669,6 @@ async fn run(config: rustfs::config::Config) -> Result<()> { Ok(()) } -fn log_external_prefix_compat_report(report: &ExternalEnvCompatReport) { - if report.conflict_count() > 0 { - warn!( - target: "rustfs::main", - event = EVENT_EXTERNAL_ENV_COMPAT_CONFLICT, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - conflict_count = report.conflict_count(), - conflict_keys = %report.conflict_keys.join(", "), - "Detected external-prefix compatibility conflicts; keeping RUSTFS_ values" - ); - } - - if report.mapped_count() > 0 { - info!( - target: "rustfs::main", - event = EVENT_EXTERNAL_ENV_COMPAT_APPLIED, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - mapped_count = report.mapped_count(), - mapped_pairs = %format_external_prefix_mappings(report), - "Applied external-prefix compatibility mappings" - ); - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BackgroundShutdownStep { DataScanner, @@ -982,27 +901,6 @@ async fn handle_shutdown( mod tests { use super::*; - #[test] - fn format_external_prefix_mappings_lists_mapped_pairs() { - let report = ExternalEnvCompatReport { - mapped_pairs: vec![ - ("MINIO_ROOT_USER".to_string(), "RUSTFS_ROOT_USER".to_string()), - ( - "MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), - "RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), - ), - ], - conflict_keys: Vec::new(), - }; - - let formatted = format_external_prefix_mappings(&report); - - assert_eq!( - formatted, - "MINIO_ROOT_USER->RUSTFS_ROOT_USER, MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY->RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY" - ); - } - #[test] fn fatal_stderr_message_uses_consistent_prefix_and_context() { assert_eq!( diff --git a/rustfs/src/startup_preflight.rs b/rustfs/src/startup_preflight.rs new file mode 100644 index 000000000..506dc3b3f --- /dev/null +++ b/rustfs/src/startup_preflight.rs @@ -0,0 +1,174 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{config::Config, license::init_license, startup_runtime::init_startup_runtime_foundation}; +use rustfs_obs::{init_obs, set_global_guard}; +use rustfs_utils::{ExternalEnvCompatReport, apply_external_env_compat}; +use std::{ + fmt, + io::{Error, Result}, +}; +use tracing::{debug, error, info, warn}; + +const LOG_COMPONENT_MAIN: &str = "main"; +const LOG_SUBSYSTEM_STARTUP: &str = "startup"; +const EVENT_EXTERNAL_ENV_COMPAT_CONFLICT: &str = "external_env_compat_conflict"; +const EVENT_EXTERNAL_ENV_COMPAT_APPLIED: &str = "external_env_compat_applied"; +const EVENT_OBSERVABILITY_GUARD_SET: &str = "observability_guard_set"; +const EVENT_OBSERVABILITY_GUARD_SET_FAILED: &str = "observability_guard_set_failed"; + +#[derive(Debug)] +pub enum StartupServerPreflightError { + ObservabilityInit(Error), + Other(Error), +} + +impl StartupServerPreflightError { + fn as_io_error(&self) -> &Error { + match self { + Self::ObservabilityInit(err) | Self::Other(err) => err, + } + } +} + +impl fmt::Display for StartupServerPreflightError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.as_io_error().fmt(formatter) + } +} + +impl std::error::Error for StartupServerPreflightError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.as_io_error()) + } +} + +pub fn bootstrap_external_prefix_compat() -> Result { + let env_compat_report = apply_external_env_compat(); + Ok(env_compat_report) +} + +pub async fn init_startup_server_preflight( + config: &Config, + env_compat_report: &ExternalEnvCompatReport, +) -> std::result::Result<(), StartupServerPreflightError> { + crate::config::init_config_snapshot(config); + init_license(config.license.clone()); + init_startup_observability(config.obs_endpoint.clone()).await?; + log_external_prefix_compat_report(env_compat_report); + init_startup_runtime_foundation(config) + .await + .map_err(StartupServerPreflightError::Other) +} + +async fn init_startup_observability(obs_endpoint: String) -> std::result::Result<(), StartupServerPreflightError> { + let guard = init_obs(Some(obs_endpoint)) + .await + .map_err(|err| StartupServerPreflightError::ObservabilityInit(Error::other(err)))?; + + match set_global_guard(guard).map_err(Error::other) { + Ok(_) => { + debug!( + target: "rustfs::main", + event = EVENT_OBSERVABILITY_GUARD_SET, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + result = "ok", + "Stored global observability guard" + ); + Ok(()) + } + Err(err) => { + error!( + target: "rustfs::main", + event = EVENT_OBSERVABILITY_GUARD_SET_FAILED, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + error = %err, + "Failed to store global observability guard" + ); + Err(StartupServerPreflightError::Other(err)) + } + } +} + +fn format_external_prefix_mappings(report: &ExternalEnvCompatReport) -> String { + report + .mapped_pairs + .iter() + .map(|(source_key, rustfs_key)| format!("{source_key}->{rustfs_key}")) + .collect::>() + .join(", ") +} + +fn log_external_prefix_compat_report(report: &ExternalEnvCompatReport) { + if report.conflict_count() > 0 { + warn!( + target: "rustfs::main", + event = EVENT_EXTERNAL_ENV_COMPAT_CONFLICT, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + conflict_count = report.conflict_count(), + conflict_keys = %report.conflict_keys.join(", "), + "Detected external-prefix compatibility conflicts; keeping RUSTFS_ values" + ); + } + + if report.mapped_count() > 0 { + info!( + target: "rustfs::main", + event = EVENT_EXTERNAL_ENV_COMPAT_APPLIED, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + mapped_count = report.mapped_count(), + mapped_pairs = %format_external_prefix_mappings(report), + "Applied external-prefix compatibility mappings" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_external_prefix_mappings_lists_mapped_pairs() { + let report = ExternalEnvCompatReport { + mapped_pairs: vec![ + ("MINIO_ROOT_USER".to_string(), "RUSTFS_ROOT_USER".to_string()), + ( + "MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), + "RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), + ), + ], + conflict_keys: Vec::new(), + }; + + let formatted = format_external_prefix_mappings(&report); + + assert_eq!( + formatted, + "MINIO_ROOT_USER->RUSTFS_ROOT_USER, MINIO_NOTIFY_WEBHOOK_ENABLE_PRIMARY->RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY" + ); + } + + #[test] + fn preflight_error_preserves_observability_init_classification() { + let err = StartupServerPreflightError::ObservabilityInit(Error::other("collector unavailable")); + + assert!(matches!(&err, StartupServerPreflightError::ObservabilityInit(_))); + assert_eq!(err.to_string(), "collector unavailable"); + assert!(std::error::Error::source(&err).is_some()); + } +}