fix(startup): survive slow multi-node cold starts (#4357)

* fix(server): make startup readiness wait configurable and raise default (#4264)

The startup runtime-readiness wait was a hardcoded 30s constant with no env
override. On slow multi-node cold starts (Docker/K8s/Synology NAS) this window
is shorter than the internal startup budgets it depends on — the endpoint
DNS-retry window (~90s) and the format-load retry loop (~100s worst case) — so
readiness times out and the node exits with
`startup readiness timed out after 30s: storage_ready=false, lock_quorum_ready=false`
before storage/lock quorum can converge, feeding the restart storm reported in
the issue.

- Add `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` (default 120s), documented in
  rustfs-config health constants.
- Resolve the wait at runtime via `startup_runtime_readiness_max_wait()`; a
  value of `0` falls back to the default instead of timing out instantly.
- Repoint `STARTUP_RUNTIME_READINESS_MAX_WAIT` at the shared config default so
  there is a single source of truth, and cover the getter with unit tests.

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(ecstore): stop peer/disk background monitors on graceful shutdown (#4264)

Long-lived peer health/recovery and remote-disk monitors are detached
`tokio::spawn` tasks that each hold a `tracing::Span` via `.instrument(..)` for
their whole lifetime. Nothing cancelled them at shutdown, so on the normal
return path the Tokio runtime was dropped while they were still alive and their
`Span`s were dropped during worker-thread thread-local-storage (TLS)
destruction. At that point `tracing-subscriber`'s fmt `on_close` can touch an
already-destroyed TLS slot and panic with
`cannot access a Thread Local Storage value during or after destruction`, which
escalates to a panic-during-panic abort (SIGILL / exit 132) — the crash
reported on Synology in issue #4264, amplified by the restart storm.

- Add `cluster::rpc::background_monitor` with a process-global shutdown token,
  `spawn_background_monitor()` (races the monitor future against that token so
  its span drops while the runtime is alive), and public
  `shutdown_background_monitors()`.
- Route every span-holding peer_s3 / peer_rest / remote_disk monitor spawn
  through `spawn_background_monitor` instead of `tokio::spawn(..).instrument()`.
- Expose `rustfs_ecstore::shutdown_background_monitors()` and call it from the
  graceful shutdown sequence (right after `ctx.cancel()`, before runtime
  teardown) via the `storage_api` compatibility boundary.

Existing recovery-probe span-context tests still pass, confirming log
correlation is preserved.

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/server/readiness.rs

@@ -43,8 +43,34 @@ use tokio::sync::Mutex;
 use tower::{Layer, Service};
 use tracing::{debug, info};
 
-pub const STARTUP_RUNTIME_READINESS_MAX_WAIT: Duration = Duration::from_secs(30);
+/// Default upper bound for the startup runtime-readiness wait.
+///
+/// This is the compile-time fallback used when
+/// [`rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS`] is unset or `0`. The
+/// effective value is resolved at runtime by [`startup_runtime_readiness_max_wait`],
+/// which lets slow multi-node cold starts (Docker/K8s/NAS) extend the budget past
+/// the internal DNS-retry and format-load windows without a rebuild.
+pub const STARTUP_RUNTIME_READINESS_MAX_WAIT: Duration =
+    Duration::from_secs(rustfs_config::DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS);
 pub const STARTUP_RUNTIME_READINESS_POLL_INTERVAL: Duration = Duration::from_secs(1);
+
+/// Resolve the effective startup runtime-readiness wait.
+///
+/// Reads `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS`, falling back to
+/// [`STARTUP_RUNTIME_READINESS_MAX_WAIT`] when unset. A configured value of `0`
+/// is treated as "use the default" rather than an instant timeout, so an empty or
+/// misconfigured env var can never make the server give up on readiness immediately.
+fn startup_runtime_readiness_max_wait() -> Duration {
+    let secs = rustfs_utils::get_env_u64(
+        rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS,
+        rustfs_config::DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS,
+    );
+    if secs == 0 {
+        STARTUP_RUNTIME_READINESS_MAX_WAIT
+    } else {
+        Duration::from_secs(secs)
+    }
+}
 const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
 const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";
 
@@ -227,7 +253,7 @@ pub async fn publish_ready_when_runtime_ready(
     state_manager: Option<&ServiceStateManager>,
 ) -> Result<(), std::io::Error> {
     wait_for_runtime_readiness_with(
-        STARTUP_RUNTIME_READINESS_MAX_WAIT,
+        startup_runtime_readiness_max_wait(),
         STARTUP_RUNTIME_READINESS_POLL_INTERVAL,
         collect_node_readiness,
         |dependency_readiness| {
@@ -914,10 +940,32 @@ mod tests {
     #[test]
     fn startup_runtime_readiness_wait_constants_are_ordered() {
         assert!(STARTUP_RUNTIME_READINESS_MAX_WAIT > STARTUP_RUNTIME_READINESS_POLL_INTERVAL);
-        assert_eq!(STARTUP_RUNTIME_READINESS_MAX_WAIT.as_secs(), 30);
+        assert_eq!(
+            STARTUP_RUNTIME_READINESS_MAX_WAIT.as_secs(),
+            rustfs_config::DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS
+        );
         assert_eq!(STARTUP_RUNTIME_READINESS_POLL_INTERVAL.as_secs(), 1);
     }
 
+    #[test]
+    #[serial]
+    fn startup_runtime_readiness_max_wait_reads_env_override() {
+        // An explicit override extends (or shortens) the budget.
+        with_var(rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS, Some("240"), || {
+            assert_eq!(startup_runtime_readiness_max_wait(), Duration::from_secs(240));
+        });
+
+        // Unset falls back to the compile-time default.
+        with_var(rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS, None::<&str>, || {
+            assert_eq!(startup_runtime_readiness_max_wait(), STARTUP_RUNTIME_READINESS_MAX_WAIT);
+        });
+
+        // Zero is treated as "use default", never an instant timeout.
+        with_var(rustfs_config::ENV_STARTUP_READINESS_MAX_WAIT_SECS, Some("0"), || {
+            assert_eq!(startup_runtime_readiness_max_wait(), STARTUP_RUNTIME_READINESS_MAX_WAIT);
+        });
+    }
+
     fn peer_health_test_pools() -> EndpointServerPools {
         EndpointServerPools::from(vec![PoolEndpoints {
             legacy: false,

rustfs/src/startup_shutdown.rs

@@ -69,6 +69,12 @@ pub(crate) async fn run_startup_shutdown_sequence(
 ) {
     ctx.cancel();
 
+    // Stop long-lived peer/disk background monitors while the runtime and tracing
+    // subscriber are still alive, so the `tracing::Span` each monitor holds is
+    // dropped here instead of during worker-thread TLS destruction at runtime
+    // teardown (which can panic in the fmt layer's `on_close`; see issue #4264).
+    crate::storage_api::startup::shutdown::shutdown_background_monitors();
+
     info!(
         target: "rustfs::main::handle_shutdown",
         event = EVENT_SHUTDOWN_SIGNAL_RECEIVED,

rustfs/src/storage/storage_api.rs

@@ -782,6 +782,10 @@ pub(crate) fn shutdown_background_services() {
     ecstore_global::shutdown_background_services();
 }
 
+pub(crate) fn shutdown_background_monitors() {
+    rustfs_ecstore::shutdown_background_monitors();
+}
+
 pub(crate) fn set_global_endpoints(endpoints: Vec<PoolEndpoints>) {
     ecstore_global::set_global_endpoints(endpoints);
 }

rustfs/src/storage_api.rs

@@ -219,7 +219,9 @@ pub(crate) mod startup {
     }
 
     pub(crate) mod shutdown {
-        pub(crate) use crate::storage::storage_api::{shutdown_background_services, store_compression_total_in_backend};
+        pub(crate) use crate::storage::storage_api::{
+            shutdown_background_monitors, shutdown_background_services, store_compression_total_in_backend,
+        };
     }
 
     pub(crate) mod storage {