mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor: centralize startup runtime services (#3467)
This commit is contained in:
安正超
@@ -5,17 +5,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-startup-storage-runtime`
|
||||
- Baseline: `origin/main` at `a19560da419c6d354a805fc127951cc9b56ad887`
|
||||
- Branch: `overtrue/arch-startup-runtime-services`
|
||||
- Baseline: `origin/main` at `3e723a2476692051cac1eab2cc31984c62063130`
|
||||
- PR type for this branch: `pure-move`
|
||||
- Runtime behavior changes: no external behavior change expected; ECStore
|
||||
creation, ECStore config initialization, global config retry policy,
|
||||
`StorageReady` publication, and background replication startup still run in
|
||||
the same relative order after HTTP server startup and before KMS startup.
|
||||
- Rust code changes: add `startup_storage::init_startup_storage_runtime` and
|
||||
- Runtime behavior changes: no external behavior change expected; KMS, optional
|
||||
protocols, diagnostics services, bucket metadata migration, IAM bootstrap,
|
||||
auth integrations, notification runtime, heal/scanner setup, and metrics
|
||||
initialization still run in the same relative order after storage runtime and
|
||||
before the server-ready log.
|
||||
- Rust code changes: add `startup_services::init_startup_runtime_services` and
|
||||
use it from binary startup.
|
||||
- CI/script changes: none.
|
||||
- Docs changes: record `R-016` startup storage runtime bootstrap progress and
|
||||
- Docs changes: record `R-017` startup runtime service bootstrap progress and
|
||||
verification.
|
||||
|
||||
## Phase 0 Tasks
|
||||
@@ -680,10 +681,33 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
formatting, migration guards, Rust risk scan, branch freshness check, and
|
||||
pre-commit quality gate.
|
||||
|
||||
- [x] `R-017` Centralize startup runtime service bootstrap.
|
||||
- Do: move KMS startup, optional protocol shutdown collection, buffer
|
||||
profiling, event notifier/audit startup, deadlock detector startup, bucket
|
||||
metadata migration, replication resync, IAM bootstrap, Keystone/OIDC auth
|
||||
integration startup, notification runtime setup, AHM/heal setup, server info,
|
||||
update check, allocator reclaim, metrics runtime, memory observability, and
|
||||
auto-tuner startup behind the `startup_services` boundary.
|
||||
- Acceptance: startup service initialization still runs after storage runtime
|
||||
initialization and before the server-ready log; `main.rs` keeps ownership of
|
||||
shutdown handling, server-ready publication, global init time, and scanner
|
||||
start; `startup_services` returns protocol shutdown handles, IAM bootstrap
|
||||
disposition, and scanner enablement.
|
||||
- Must preserve: KMS fatal behavior, protocol fatal/disabled behavior, audit
|
||||
non-fatal behavior, deadlock detector logging, bucket list and replication
|
||||
resync fatal behavior, bucket/IAM metadata migration non-fatal behavior, IAM
|
||||
deferred recovery semantics, Keystone parse fatal and runtime non-fatal
|
||||
behavior, OIDC non-fatal behavior, notification init fatal behavior,
|
||||
scanner-implies-heal behavior, metric-enabled guard, and shutdown token
|
||||
ownership.
|
||||
- Verification: focused startup services tests, binary/lib compile checks,
|
||||
formatting, migration guards, Rust risk scan, branch freshness check, and
|
||||
pre-commit quality gate.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `pure-move`: continue extracting startup boot wrappers in larger slices while
|
||||
preserving startup order and readiness ownership.
|
||||
1. `pure-move`: continue extracting startup ready/scanner/shutdown wrappers while
|
||||
preserving startup order, readiness ownership, and shutdown ownership.
|
||||
2. `ci-gate`: finish `G-006` public re-export and storage trait coverage checks
|
||||
before the remaining cleanup slices.
|
||||
|
||||
@@ -691,38 +715,39 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | passed | Pure-move slice removes storage runtime details from binary startup behind the existing startup storage boundary. |
|
||||
| Migration preservation | passed | HTTP server startup, ECStore creation, config init/migration, global config retry, `StorageReady`, replication startup, and KMS startup order are preserved. |
|
||||
| Testing/verification | passed | Focused startup storage tests, compile checks, formatting, migration/layer guards, Rust risk scan, branch freshness check, and full `make pre-commit` passed. |
|
||||
| Quality/architecture | passed | Pure-move slice removes runtime service details from binary startup behind the existing startup services boundary. |
|
||||
| Migration preservation | passed | Storage runtime, KMS, optional protocols, diagnostics, bucket metadata, IAM, auth, notification, heal/scanner, metrics, and server-ready order are preserved. |
|
||||
| Testing/verification | passed | Focused startup services tests, compile checks, formatting, migration/layer guards, Rust risk scan, branch freshness check, and full `make pre-commit` passed. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
Passed on `a19560da419c6d354a805fc127951cc9b56ad887`:
|
||||
Passed on `3e723a2476692051cac1eab2cc31984c62063130`:
|
||||
|
||||
- `cargo test -p rustfs startup_storage --no-fail-fast`: passed.
|
||||
- `cargo test -p rustfs startup_services --no-fail-fast`: passed.
|
||||
- `cargo check -p rustfs --lib`: passed.
|
||||
- `cargo check -p rustfs --bin rustfs`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `git rev-list --left-right --count HEAD...origin/main` returned `1 0`
|
||||
after commit and rebase.
|
||||
- Added-line Rust risk scan for changed Rust files: no matches.
|
||||
- Full-file risk scan for changed Rust files: matches are existing binary
|
||||
startup stderr/expect usage.
|
||||
- `make pre-commit`: all checks passed, including nextest with 6016 passed
|
||||
and 111 skipped, plus doctests.
|
||||
- Added-line Rust risk scan for changed Rust files: passed.
|
||||
- Full-file risk scan for changed Rust files: existing `main.rs` process setup
|
||||
entries only.
|
||||
- `make pre-commit`: passed, including nextest `6027` passed / `111` skipped
|
||||
and doctests.
|
||||
- `git rev-list --left-right --count HEAD...origin/main`: returned `1 0`
|
||||
after commit.
|
||||
|
||||
Notes:
|
||||
|
||||
- This slice centralizes startup storage runtime without changing startup
|
||||
ordering, shutdown token ownership, or endpoint pool ownership.
|
||||
- Storage runtime remains after HTTP server startup and before KMS startup.
|
||||
- This slice centralizes startup runtime services without changing startup
|
||||
ordering, shutdown token ownership, readiness ownership, or endpoint pool
|
||||
ownership.
|
||||
- Runtime services remain after storage runtime and before the server-ready log.
|
||||
|
||||
## Handoff Notes
|
||||
|
||||
- R-016 is complete.
|
||||
- R-017 is implemented, locally verified, and current with `origin/main`.
|
||||
- Next startup slices can keep using larger pure moves, but must keep startup
|
||||
ordering, fatal/non-fatal boundaries, shutdown ownership, and readiness
|
||||
ownership explicit in tests.
|
||||
ownership explicit in tests and review notes.
|
||||
|
||||
@@ -35,15 +35,15 @@ new startup semantics.
|
||||
| `RUN-010` | `rustfs/src/main.rs:356` | Start S3 HTTP listener and optional console listener before storage is ready. | Starts HTTP servers with readiness gates; console listener starts only when enabled and configured. | Fatal if a configured listener cannot start. | Requests remain gated until full readiness except probe/admin/console/rpc/tonic/table-catalog exempt paths |
|
||||
| `RUN-011` | `rustfs/src/startup_storage.rs` | Create cancellation token and initialize `ECStore`. | Creates the runtime cancellation token and storage engine. | Fatal if `ECStore::new` fails. | None |
|
||||
| `RUN-012` | `rustfs/src/startup_storage.rs` | Initialize ECStore config and global config system. | Initializes ECStore config, attempts server-config migration, then retries global config init up to 15 times. | Migration attempt is non-fatal in this path; global config init becomes fatal after retries. | Marks the `GlobalReadiness` `StorageReady` stage after global config init succeeds; later runtime readiness still rechecks storage, IAM, and lock quorum before `FullReady` |
|
||||
| `RUN-013` | `rustfs/src/startup_storage.rs` and `rustfs/src/main.rs` | Start replication and KMS systems. | Starts background replication pool, then initializes KMS from main startup. | Replication init is non-fatal in this path; KMS init is fatal on error. | `StorageReady` stage is already marked; dynamic runtime storage readiness is still checked before `FullReady` |
|
||||
| `RUN-014` | `rustfs/src/main.rs:402` | Initialize optional protocol servers. | Starts FTP/FTPS/WebDAV/SFTP when feature-enabled and configured, collecting shutdown handles. | Feature-enabled protocol init is fatal on error; disabled protocols are non-fatal. | None |
|
||||
| `RUN-015` | `rustfs/src/main.rs:482` | Initialize buffer profiling, event notifier, audit, and deadlock detector. | Starts buffer profile system, event notifier, audit system, and optional deadlock detector. | Audit startup failure is logged and non-fatal; the others are non-fatal in this path. | None |
|
||||
| `RUN-016` | `rustfs/src/main.rs:503` | List buckets and run bucket/replication/IAM metadata migrations. | Reads bucket names, migrates bucket metadata, initializes replication resync, migrates IAM config, and initializes bucket metadata system. | Bucket list and replication resync are fatal on error; metadata migration calls are non-fatal in this path. | Storage remains ready; IAM not yet ready |
|
||||
| `RUN-017` | `rustfs/src/main.rs:523` | Bootstrap IAM inline or defer recovery. | Initializes IAM when possible; otherwise starts the deferred IAM recovery path through `startup_iam`. | Fatal only when `bootstrap_or_defer_iam_init` returns an unrecoverable error. | Inline success marks `IamReady`; deferred mode publishes `IamReady` later from the recovery task |
|
||||
| `RUN-018` | `rustfs/src/main.rs:535` | Initialize Keystone and OIDC auth integrations. | Loads Keystone env config and initializes OIDC providers. | Keystone config parse is fatal; Keystone runtime init failure is non-fatal; OIDC init failure is non-fatal. | None |
|
||||
| `RUN-019` | `rustfs/src/main.rs:552` | Add bucket notification config and initialize notification system. | Adds bucket notification configuration and publishes the global notification system. | Notification config add is non-fatal in this path; global notification init is fatal on error. | None |
|
||||
| `RUN-020` | `rustfs/src/main.rs:560` | Create AHM cancellation token and initialize heal manager when scanner or heal is enabled. | Creates AHM cancellation token and starts heal manager for heal/scanner workflows. | Heal manager init is fatal when enabled. | None |
|
||||
| `RUN-021` | `rustfs/src/main.rs:584` | Print server info, init update check, allocator reclaim, metrics, memory observability, and auto-tuner. | Starts informational/update/memory/metrics background tasks when enabled. | Non-fatal in this path. | None |
|
||||
| `RUN-013` | `rustfs/src/startup_storage.rs` and `rustfs/src/startup_services.rs` | Start replication and KMS systems. | Starts background replication pool, then initializes KMS from startup services. | Replication init is non-fatal in this path; KMS init is fatal on error. | `StorageReady` stage is already marked; dynamic runtime storage readiness is still checked before `FullReady` |
|
||||
| `RUN-014` | `rustfs/src/startup_services.rs` | Initialize optional protocol servers. | Starts FTP/FTPS/WebDAV/SFTP when feature-enabled and configured, collecting shutdown handles. | Feature-enabled protocol init is fatal on error; disabled protocols are non-fatal. | None |
|
||||
| `RUN-015` | `rustfs/src/startup_services.rs` | Initialize buffer profiling, event notifier, audit, and deadlock detector. | Starts buffer profile system, event notifier, audit system, and optional deadlock detector. | Audit startup failure is logged and non-fatal; the others are non-fatal in this path. | None |
|
||||
| `RUN-016` | `rustfs/src/startup_services.rs` | List buckets and run bucket/replication/IAM metadata migrations. | Reads bucket names, migrates bucket metadata, initializes replication resync, migrates IAM config, and initializes bucket metadata system. | Bucket list and replication resync are fatal on error; metadata migration calls are non-fatal in this path. | Storage remains ready; IAM not yet ready |
|
||||
| `RUN-017` | `rustfs/src/startup_services.rs` and `rustfs/src/startup_iam.rs` | Bootstrap IAM inline or defer recovery. | Initializes IAM when possible; otherwise starts the deferred IAM recovery path through `startup_iam`. | Fatal only when `bootstrap_or_defer_iam_init` returns an unrecoverable error. | Inline success marks `IamReady`; deferred mode publishes `IamReady` later from the recovery task |
|
||||
| `RUN-018` | `rustfs/src/startup_services.rs` | Initialize Keystone and OIDC auth integrations. | Loads Keystone env config and initializes OIDC providers. | Keystone config parse is fatal; Keystone runtime init failure is non-fatal; OIDC init failure is non-fatal. | None |
|
||||
| `RUN-019` | `rustfs/src/startup_services.rs` | Add bucket notification config and initialize notification system. | Adds bucket notification configuration and publishes the global notification system. | Notification config add is non-fatal in this path; global notification init is fatal on error. | None |
|
||||
| `RUN-020` | `rustfs/src/startup_services.rs` | Create AHM cancellation token and initialize heal manager when scanner or heal is enabled. | Creates AHM cancellation token and starts heal manager for heal/scanner workflows. | Heal manager init is fatal when enabled. | None |
|
||||
| `RUN-021` | `rustfs/src/startup_services.rs` | Print server info, init update check, allocator reclaim, metrics, memory observability, and auto-tuner. | Starts informational/update/memory/metrics background tasks when enabled. | Non-fatal in this path. | None |
|
||||
| `RUN-022` | `rustfs/src/main.rs:599` | Log successful startup and publish full readiness for inline IAM. | Logs version/address, checks runtime readiness, marks `FullReady`, and sets service state to `Ready` when IAM was ready inline. | Fatal if runtime readiness is not reached within the startup wait. | Marks `FullReady` only for inline IAM here |
|
||||
| `RUN-023` | `rustfs/src/main.rs:609` | Publish global init time and start data scanner when enabled. | Sets global init time and starts scanner after the successful-startup log. | Scanner start is non-fatal in this path. | Full readiness may already be published or may await deferred IAM recovery |
|
||||
| `RUN-024` | `rustfs/src/main.rs:616` | Wait for shutdown signal. | Blocks the main task until a shutdown signal is received. | Non-fatal. | Runtime remains in its current readiness state |
|
||||
|
||||
+15
-207
@@ -13,39 +13,24 @@
|
||||
// limitations under the License.
|
||||
|
||||
// Ensure the correct path for parse_license is imported
|
||||
use rustfs::init::{
|
||||
add_bucket_notification_configuration, init_buffer_profile_system, init_kms_system, init_update_check, print_server_info,
|
||||
};
|
||||
|
||||
use futures_util::future::join_all;
|
||||
use rustfs::server::{
|
||||
ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier, stop_audit_system,
|
||||
wait_for_shutdown,
|
||||
};
|
||||
use rustfs::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap};
|
||||
use rustfs::startup_iam::publish_ready_for_iam_bootstrap;
|
||||
use rustfs::startup_preflight::{StartupServerPreflightError, bootstrap_external_prefix_compat, init_startup_server_preflight};
|
||||
use rustfs::startup_protocols::{ProtocolShutdownSenders, init_protocol_shutdown_senders};
|
||||
use rustfs::startup_protocols::ProtocolShutdownSenders;
|
||||
use rustfs::startup_server::{StartupHttpServers, StartupListenContext, init_startup_http_servers, init_startup_listen_context};
|
||||
use rustfs::startup_services::{StartupServiceRuntime, init_startup_runtime_services};
|
||||
use rustfs::startup_storage::{StartupStorageRuntime, init_startup_storage_foundation, init_startup_storage_runtime};
|
||||
use rustfs_ecstore::{
|
||||
bucket::metadata_sys::init_bucket_metadata_sys,
|
||||
bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
|
||||
bucket::replication::get_global_replication_pool,
|
||||
global::shutdown_background_services,
|
||||
store_api::BucketOperations,
|
||||
};
|
||||
use rustfs_heal::{
|
||||
create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services,
|
||||
};
|
||||
use rustfs_iam::init_oidc_sys;
|
||||
use rustfs_obs::init_metrics_runtime;
|
||||
use rustfs_ecstore::global::shutdown_background_services;
|
||||
use rustfs_heal::shutdown_ahm_services;
|
||||
use rustfs_scanner::init_data_scanner;
|
||||
use rustfs_storage_api::BucketOptions;
|
||||
use rustfs_utils::get_env_bool_with_aliases;
|
||||
use std::io::{Error, Result};
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use tracing::{error, info, instrument};
|
||||
|
||||
const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED";
|
||||
const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER";
|
||||
@@ -53,16 +38,9 @@ const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED";
|
||||
const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL";
|
||||
const LOG_COMPONENT_MAIN: &str = "main";
|
||||
const LOG_SUBSYSTEM_STARTUP: &str = "startup";
|
||||
const LOG_SUBSYSTEM_AUTH: &str = "auth";
|
||||
const EVENT_SERVER_RUNTIME_FAILED: &str = "server_runtime_failed";
|
||||
const EVENT_PROTOCOL_SYSTEM_STATE: &str = "protocol_system_state";
|
||||
const EVENT_AUDIT_SYSTEM_STATE: &str = "audit_system_state";
|
||||
const EVENT_DEADLOCK_DETECTOR_STATE: &str = "deadlock_detector_state";
|
||||
const EVENT_KEYSTONE_AUTH_INITIALIZED: &str = "keystone_auth_initialized";
|
||||
const EVENT_KEYSTONE_AUTH_INITIALIZATION_FAILED: &str = "keystone_auth_initialization_failed";
|
||||
const EVENT_OIDC_INITIALIZATION_FAILED: &str = "oidc_initialization_failed";
|
||||
const EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED: &str = "notification_system_initialization_failed";
|
||||
const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured";
|
||||
const EVENT_SERVER_READY: &str = "server_ready";
|
||||
const EVENT_SHUTDOWN_SIGNAL_RECEIVED: &str = "shutdown_signal_received";
|
||||
const EVENT_BACKGROUND_SERVICE_SHUTDOWN: &str = "background_service_shutdown";
|
||||
@@ -169,190 +147,20 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
|
||||
shutdown_token: ctx,
|
||||
} = init_startup_storage_runtime(server_addr, &endpoint_pools, readiness.clone()).await?;
|
||||
|
||||
// Initialize KMS system if enabled
|
||||
init_kms_system(&config).await?;
|
||||
|
||||
let protocol_shutdowns = init_protocol_shutdown_senders().await?;
|
||||
|
||||
// Initialize buffer profiling system
|
||||
init_buffer_profile_system(&config);
|
||||
|
||||
match rustfs::startup_services::init_event_notifier_and_audit().await {
|
||||
Ok(()) => info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_AUDIT_SYSTEM_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "started",
|
||||
"Audit runtime started"
|
||||
),
|
||||
Err(e) => error!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_AUDIT_SYSTEM_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"Audit runtime failed to start"
|
||||
),
|
||||
}
|
||||
|
||||
// Initialize deadlock detector if enabled
|
||||
let detector = rustfs::storage::deadlock_detector::get_deadlock_detector();
|
||||
if detector.is_enabled() {
|
||||
detector.start();
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_DEADLOCK_DETECTOR_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "started",
|
||||
"Deadlock detector started"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_DEADLOCK_DETECTOR_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "disabled",
|
||||
"Deadlock detector disabled"
|
||||
);
|
||||
}
|
||||
|
||||
let buckets_list = store
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
|
||||
// Collect bucket names into a vector
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
|
||||
if let Some(pool) = get_global_replication_pool() {
|
||||
pool.init_resync(ctx.clone(), buckets.clone()).await?;
|
||||
}
|
||||
|
||||
try_migrate_iam_config(store.clone()).await;
|
||||
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
|
||||
|
||||
// 3. Initialize IAM System (Blocking load)
|
||||
// This ensures data is in memory before moving forward
|
||||
let kms_interface = rustfs_kms::get_global_kms_service_manager().unwrap_or_else(rustfs_kms::init_global_kms_service_manager);
|
||||
let iam_bootstrap = bootstrap_or_defer_iam_init(
|
||||
let StartupServiceRuntime {
|
||||
protocol_shutdowns,
|
||||
iam_bootstrap,
|
||||
enable_scanner,
|
||||
} = init_startup_runtime_services(
|
||||
&config,
|
||||
endpoint_pools,
|
||||
store.clone(),
|
||||
kms_interface,
|
||||
ctx.clone(),
|
||||
readiness.clone(),
|
||||
Some(state_manager.clone()),
|
||||
Some(ctx.clone()),
|
||||
state_manager.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// 3a. Initialize Keystone authentication if enabled
|
||||
let keystone_config = rustfs_keystone::KeystoneConfig::from_env().map_err(Error::other)?;
|
||||
if keystone_config.enable {
|
||||
match rustfs::auth_keystone::init_keystone_auth(keystone_config).await {
|
||||
Ok(_) => info!(
|
||||
event = EVENT_KEYSTONE_AUTH_INITIALIZED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_AUTH,
|
||||
"Initialized Keystone authentication"
|
||||
),
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_KEYSTONE_AUTH_INITIALIZATION_FAILED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_AUTH,
|
||||
error = %e,
|
||||
"Failed to initialize Keystone authentication"
|
||||
);
|
||||
// Continue without Keystone - fall back to standard auth
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3b. Initialize OIDC System (non-fatal if no providers configured)
|
||||
if let Err(e) = init_oidc_sys().await {
|
||||
warn!(
|
||||
event = EVENT_OIDC_INITIALIZATION_FAILED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_AUTH,
|
||||
error = %e,
|
||||
"OIDC initialization failed; continuing without OIDC providers"
|
||||
);
|
||||
}
|
||||
|
||||
add_bucket_notification_configuration(buckets.clone()).await;
|
||||
|
||||
rustfs::startup_services::init_notification_system(endpoint_pools.clone())
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
error = ?err,
|
||||
"Failed to initialize notification system"
|
||||
);
|
||||
Error::other(err)
|
||||
})?;
|
||||
|
||||
// Create a cancellation token for AHM services
|
||||
let _ = create_ahm_services_cancel_token();
|
||||
|
||||
// Check environment variables to determine if scanner and heal should be enabled
|
||||
let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true);
|
||||
let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true);
|
||||
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_BACKGROUND_SERVICES_CONFIGURED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
enable_scanner = enable_scanner,
|
||||
enable_heal = enable_heal,
|
||||
"Background services configured"
|
||||
);
|
||||
|
||||
// Scanner depends on the heal channel/manager, so scanner implies heal.
|
||||
if enable_heal || enable_scanner {
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
|
||||
init_heal_manager(heal_storage, None).await?;
|
||||
}
|
||||
|
||||
if !enable_heal && !enable_scanner {
|
||||
debug!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_BACKGROUND_SERVICES_CONFIGURED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
enable_scanner = false,
|
||||
enable_heal = false,
|
||||
ahm_state = "skipped",
|
||||
reason = "disabled",
|
||||
"Background services disabled"
|
||||
);
|
||||
}
|
||||
|
||||
// print server info
|
||||
print_server_info();
|
||||
|
||||
init_update_check();
|
||||
rustfs::allocator_reclaim::init_allocator_reclaim(ctx.clone());
|
||||
|
||||
if rustfs_obs::observability_metric_enabled() {
|
||||
// Initialize metrics system
|
||||
init_metrics_runtime(ctx.clone());
|
||||
rustfs::memory_observability::init_memory_observability(ctx.clone());
|
||||
|
||||
// Initialize auto-tuner for performance optimization (optional)
|
||||
rustfs::init::init_auto_tuner(ctx.clone()).await;
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_SERVER_READY,
|
||||
|
||||
@@ -12,11 +12,271 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::server::{init_event_notifier, start_audit_system};
|
||||
use crate::{
|
||||
config::Config,
|
||||
init::{
|
||||
add_bucket_notification_configuration, init_auto_tuner, init_buffer_profile_system, init_kms_system, init_update_check,
|
||||
print_server_info,
|
||||
},
|
||||
server::{ServiceStateManager, init_event_notifier, start_audit_system},
|
||||
startup_iam::{IamBootstrapDisposition, bootstrap_or_defer_iam_init},
|
||||
startup_protocols::{ProtocolShutdownSenders, init_protocol_shutdown_senders},
|
||||
};
|
||||
use rustfs_audit::AuditResult;
|
||||
use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
use rustfs_ecstore::notification_sys::new_global_notification_sys;
|
||||
use std::future::Future;
|
||||
use rustfs_common::GlobalReadiness;
|
||||
use rustfs_ecstore::{
|
||||
bucket::{
|
||||
metadata_sys::init_bucket_metadata_sys,
|
||||
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
|
||||
replication::get_global_replication_pool,
|
||||
},
|
||||
endpoints::EndpointServerPools,
|
||||
notification_sys::new_global_notification_sys,
|
||||
store::ECStore,
|
||||
store_api::BucketOperations,
|
||||
};
|
||||
use rustfs_heal::{create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager};
|
||||
use rustfs_iam::init_oidc_sys;
|
||||
use rustfs_obs::init_metrics_runtime;
|
||||
use rustfs_storage_api::BucketOptions;
|
||||
use rustfs_utils::get_env_bool_with_aliases;
|
||||
use std::{
|
||||
future::Future,
|
||||
io::{Error, Result},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED";
|
||||
const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER";
|
||||
const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED";
|
||||
const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL";
|
||||
const LOG_COMPONENT_MAIN: &str = "main";
|
||||
const LOG_SUBSYSTEM_STARTUP: &str = "startup";
|
||||
const LOG_SUBSYSTEM_AUTH: &str = "auth";
|
||||
const EVENT_AUDIT_SYSTEM_STATE: &str = "audit_system_state";
|
||||
const EVENT_DEADLOCK_DETECTOR_STATE: &str = "deadlock_detector_state";
|
||||
const EVENT_KEYSTONE_AUTH_INITIALIZED: &str = "keystone_auth_initialized";
|
||||
const EVENT_KEYSTONE_AUTH_INITIALIZATION_FAILED: &str = "keystone_auth_initialization_failed";
|
||||
const EVENT_OIDC_INITIALIZATION_FAILED: &str = "oidc_initialization_failed";
|
||||
const EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED: &str = "notification_system_initialization_failed";
|
||||
const EVENT_BACKGROUND_SERVICES_CONFIGURED: &str = "background_services_configured";
|
||||
|
||||
pub struct StartupServiceRuntime {
|
||||
pub protocol_shutdowns: ProtocolShutdownSenders,
|
||||
pub iam_bootstrap: IamBootstrapDisposition,
|
||||
pub enable_scanner: bool,
|
||||
}
|
||||
|
||||
pub async fn init_startup_runtime_services(
|
||||
config: &Config,
|
||||
endpoint_pools: EndpointServerPools,
|
||||
store: Arc<ECStore>,
|
||||
ctx: CancellationToken,
|
||||
readiness: Arc<GlobalReadiness>,
|
||||
state_manager: Arc<ServiceStateManager>,
|
||||
) -> Result<StartupServiceRuntime> {
|
||||
init_kms_system(config).await?;
|
||||
|
||||
let protocol_shutdowns = init_protocol_shutdown_senders().await?;
|
||||
|
||||
init_buffer_profile_system(config);
|
||||
init_audit_runtime().await;
|
||||
init_deadlock_detector_runtime();
|
||||
|
||||
let buckets = init_bucket_metadata_runtime(store.clone(), ctx.clone()).await?;
|
||||
let iam_bootstrap = init_iam_runtime(store.clone(), ctx.clone(), readiness, state_manager).await?;
|
||||
init_auth_integrations().await?;
|
||||
init_notification_runtime(endpoint_pools, buckets).await?;
|
||||
let enable_scanner = init_background_service_runtime(store.clone()).await?;
|
||||
init_observability_runtime(ctx.clone()).await;
|
||||
|
||||
Ok(StartupServiceRuntime {
|
||||
protocol_shutdowns,
|
||||
iam_bootstrap,
|
||||
enable_scanner,
|
||||
})
|
||||
}
|
||||
|
||||
async fn init_audit_runtime() {
|
||||
match init_event_notifier_and_audit().await {
|
||||
Ok(()) => info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_AUDIT_SYSTEM_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "started",
|
||||
"Audit runtime started"
|
||||
),
|
||||
Err(e) => error!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_AUDIT_SYSTEM_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"Audit runtime failed to start"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn init_deadlock_detector_runtime() {
|
||||
let detector = crate::storage::deadlock_detector::get_deadlock_detector();
|
||||
if detector.is_enabled() {
|
||||
detector.start();
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_DEADLOCK_DETECTOR_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "started",
|
||||
"Deadlock detector started"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_DEADLOCK_DETECTOR_STATE,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
state = "disabled",
|
||||
"Deadlock detector disabled"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: CancellationToken) -> Result<Vec<String>> {
|
||||
let buckets_list = store
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
|
||||
if let Some(pool) = get_global_replication_pool() {
|
||||
pool.init_resync(ctx, buckets.clone()).await?;
|
||||
}
|
||||
|
||||
try_migrate_iam_config(store.clone()).await;
|
||||
init_bucket_metadata_sys(store, buckets.clone()).await;
|
||||
|
||||
Ok(buckets)
|
||||
}
|
||||
|
||||
async fn init_iam_runtime(
|
||||
store: Arc<ECStore>,
|
||||
ctx: CancellationToken,
|
||||
readiness: Arc<GlobalReadiness>,
|
||||
state_manager: Arc<ServiceStateManager>,
|
||||
) -> Result<IamBootstrapDisposition> {
|
||||
let kms_interface = rustfs_kms::get_global_kms_service_manager().unwrap_or_else(rustfs_kms::init_global_kms_service_manager);
|
||||
bootstrap_or_defer_iam_init(store, kms_interface, readiness, Some(state_manager), Some(ctx)).await
|
||||
}
|
||||
|
||||
async fn init_auth_integrations() -> Result<()> {
|
||||
let keystone_config = rustfs_keystone::KeystoneConfig::from_env().map_err(Error::other)?;
|
||||
if keystone_config.enable {
|
||||
match crate::auth_keystone::init_keystone_auth(keystone_config).await {
|
||||
Ok(_) => info!(
|
||||
event = EVENT_KEYSTONE_AUTH_INITIALIZED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_AUTH,
|
||||
"Initialized Keystone authentication"
|
||||
),
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_KEYSTONE_AUTH_INITIALIZATION_FAILED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_AUTH,
|
||||
error = %e,
|
||||
"Failed to initialize Keystone authentication"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = init_oidc_sys().await {
|
||||
warn!(
|
||||
event = EVENT_OIDC_INITIALIZATION_FAILED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_AUTH,
|
||||
error = %e,
|
||||
"OIDC initialization failed; continuing without OIDC providers"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn init_notification_runtime(endpoint_pools: EndpointServerPools, buckets: Vec<String>) -> Result<()> {
|
||||
add_bucket_notification_configuration(buckets).await;
|
||||
|
||||
init_notification_system(endpoint_pools).await.map_err(|err| {
|
||||
error!(
|
||||
event = EVENT_NOTIFICATION_SYSTEM_INITIALIZATION_FAILED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
error = ?err,
|
||||
"Failed to initialize notification system"
|
||||
);
|
||||
Error::other(err)
|
||||
})
|
||||
}
|
||||
|
||||
async fn init_background_service_runtime(store: Arc<ECStore>) -> Result<bool> {
|
||||
let _ = create_ahm_services_cancel_token();
|
||||
|
||||
let enable_scanner = get_env_bool_with_aliases(ENV_SCANNER_ENABLED, &[ENV_SCANNER_ENABLED_DEPRECATED], true);
|
||||
let enable_heal = get_env_bool_with_aliases(ENV_HEAL_ENABLED, &[ENV_HEAL_ENABLED_DEPRECATED], true);
|
||||
|
||||
info!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_BACKGROUND_SERVICES_CONFIGURED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
enable_scanner = enable_scanner,
|
||||
enable_heal = enable_heal,
|
||||
"Background services configured"
|
||||
);
|
||||
|
||||
if enable_heal || enable_scanner {
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(store));
|
||||
init_heal_manager(heal_storage, None).await?;
|
||||
}
|
||||
|
||||
if !enable_heal && !enable_scanner {
|
||||
debug!(
|
||||
target: "rustfs::main::run",
|
||||
event = EVENT_BACKGROUND_SERVICES_CONFIGURED,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
enable_scanner = false,
|
||||
enable_heal = false,
|
||||
ahm_state = "skipped",
|
||||
reason = "disabled",
|
||||
"Background services disabled"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(enable_scanner)
|
||||
}
|
||||
|
||||
async fn init_observability_runtime(ctx: CancellationToken) {
|
||||
print_server_info();
|
||||
init_update_check();
|
||||
crate::allocator_reclaim::init_allocator_reclaim(ctx.clone());
|
||||
|
||||
if rustfs_obs::observability_metric_enabled() {
|
||||
init_metrics_runtime(ctx.clone());
|
||||
crate::memory_observability::init_memory_observability(ctx.clone());
|
||||
init_auto_tuner(ctx).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_event_notifier_and_audit() -> AuditResult<()> {
|
||||
init_event_notifier_and_audit_with(init_event_notifier, start_audit_system).await
|
||||
|
||||
Reference in New Issue
Block a user