feat: improve degraded readiness reporting and shutdown handling (#3089)

* fix(server): unify runtime readiness and shutdown flow

* refactor(server): decouple readiness shared types

* refactor(server): keep readiness types crate-private

* refactor(server): await protocol shutdown handles

* fix(server): bypass cached startup readiness

* feat(health): expose degraded readiness reasons

* fix(health): preserve raw IAM readiness in degraded reports

rustfs/src/admin/console.rs

@@ -523,7 +523,9 @@ async fn health_check(method: Method, uri: Uri) -> Response {
     } else {
         HealthProbe::Liveness
     };
-    let (storage_ready, iam_ready) = collect_dependency_readiness().await;
+    let readiness_report = collect_dependency_readiness().await;
+    let storage_ready = readiness_report.readiness.storage_ready;
+    let iam_ready = readiness_report.readiness.iam_ready;
     let health = health_check_state(storage_ready, iam_ready, probe);
 
     let builder = Response::builder()
@@ -537,7 +539,14 @@ async fn health_check(method: Method, uri: Uri) -> Response {
                 .duration_since(std::time::UNIX_EPOCH)
                 .unwrap_or_default()
                 .as_secs();
-            let body_json = build_health_payload(health, storage_ready, iam_ready, "rustfs-console", Some(uptime));
+            let body_json = build_health_payload(
+                health,
+                storage_ready,
+                iam_ready,
+                &readiness_report.degraded_reasons,
+                "rustfs-console",
+                Some(uptime),
+            );
 
             // Return a minimal JSON when serialization fails to avoid panic
             let body_str = serde_json::to_string(&body_json).unwrap_or_else(|e| {

rustfs/src/admin/handlers/health.rs

@@ -16,7 +16,7 @@ use super::profile::{TriggerProfileCPU, TriggerProfileMemory};
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::server::{
     HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
-    collect_dependency_readiness as collect_runtime_dependency_readiness,
+    collect_dependency_readiness_report as collect_runtime_dependency_readiness_report,
 };
 use http::{HeaderMap, HeaderValue};
 use hyper::{Method, StatusCode};
@@ -57,9 +57,8 @@ pub(crate) enum HealthProbe {
     Readiness,
 }
 
-pub(crate) async fn collect_dependency_readiness() -> (bool, bool) {
-    let readiness = collect_runtime_dependency_readiness().await;
-    (readiness.storage_ready, readiness.iam_ready)
+pub(crate) async fn collect_dependency_readiness() -> crate::server::DependencyReadinessReport {
+    collect_runtime_dependency_readiness_report().await
 }
 
 pub(crate) fn health_check_state(storage_ready: bool, iam_ready: bool, probe: HealthProbe) -> HealthCheckState {
@@ -99,6 +98,15 @@ pub(crate) fn build_component_details(storage_ready: bool, iam_ready: bool) -> V
     })
 }
 
+pub(crate) fn build_degraded_reasons(reasons: &[crate::server::ReadinessDegradedReason]) -> Value {
+    Value::Array(
+        reasons
+            .iter()
+            .map(|reason| Value::String(reason.as_str().to_string()))
+            .collect(),
+    )
+}
+
 pub(crate) fn probe_from_path(path: &str) -> HealthProbe {
     if path == HEALTH_READY_PATH {
         HealthProbe::Readiness
@@ -111,6 +119,7 @@ pub(crate) fn build_health_payload(
     health: HealthCheckState,
     storage_ready: bool,
     iam_ready: bool,
+    degraded_reasons: &[crate::server::ReadinessDegradedReason],
     service: &str,
     uptime: Option<u64>,
 ) -> Value {
@@ -128,6 +137,7 @@ pub(crate) fn build_health_payload(
         "timestamp": jiff::Zoned::now().to_string(),
         "version": env!("CARGO_PKG_VERSION"),
         "details": build_component_details(storage_ready, iam_ready),
+        "degradedReasons": build_degraded_reasons(degraded_reasons),
     });
 
     if let Some(uptime) = uptime {
@@ -142,9 +152,10 @@ pub(crate) fn build_health_response(
     probe: HealthProbe,
     storage_ready: bool,
     iam_ready: bool,
+    degraded_reasons: &[crate::server::ReadinessDegradedReason],
 ) -> S3Response<(StatusCode, Body)> {
     let health = health_check_state(storage_ready, iam_ready, probe);
-    let health_info = build_health_payload(health, storage_ready, iam_ready, "rustfs-endpoint", None);
+    let health_info = build_health_payload(health, storage_ready, iam_ready, degraded_reasons, "rustfs-endpoint", None);
 
     let mut headers = HeaderMap::new();
     headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
@@ -177,9 +188,15 @@ impl Operation for HealthCheckHandler {
         }
 
         let probe = probe_from_path(req.uri.path());
-        let (storage_ready, iam_ready) = collect_dependency_readiness().await;
+        let readiness_report = collect_dependency_readiness().await;
 
-        Ok(build_health_response(method, probe, storage_ready, iam_ready))
+        Ok(build_health_response(
+            method,
+            probe,
+            readiness_report.readiness.storage_ready,
+            readiness_report.readiness.iam_ready,
+            &readiness_report.degraded_reasons,
+        ))
     }
 }
 
@@ -243,25 +260,43 @@ mod tests {
 
     #[test]
     fn test_build_health_response_readiness_returns_503_when_deps_not_ready() {
-        let resp = build_health_response(Method::GET, HealthProbe::Readiness, false, true);
+        let resp = build_health_response(
+            Method::GET,
+            HealthProbe::Readiness,
+            false,
+            true,
+            &[crate::server::ReadinessDegradedReason::StorageQuorumUnavailable],
+        );
         assert_eq!(resp.output.0, StatusCode::SERVICE_UNAVAILABLE);
     }
 
     #[test]
     fn test_build_health_response_readiness_returns_200_when_deps_ready() {
-        let resp = build_health_response(Method::GET, HealthProbe::Readiness, true, true);
+        let resp = build_health_response(Method::GET, HealthProbe::Readiness, true, true, &[]);
         assert_eq!(resp.output.0, StatusCode::OK);
     }
 
     #[test]
     fn test_build_health_response_liveness_returns_200_when_deps_not_ready() {
-        let resp = build_health_response(Method::GET, HealthProbe::Liveness, false, false);
+        let resp = build_health_response(
+            Method::GET,
+            HealthProbe::Liveness,
+            false,
+            false,
+            &[crate::server::ReadinessDegradedReason::StorageAndIamUnavailable],
+        );
         assert_eq!(resp.output.0, StatusCode::OK);
     }
 
     #[test]
     fn test_build_health_response_head_returns_empty_body() {
-        let resp = build_health_response(Method::HEAD, HealthProbe::Readiness, false, false);
+        let resp = build_health_response(
+            Method::HEAD,
+            HealthProbe::Readiness,
+            false,
+            false,
+            &[crate::server::ReadinessDegradedReason::StorageAndIamUnavailable],
+        );
         assert_eq!(resp.output.0, StatusCode::SERVICE_UNAVAILABLE);
     }
 
@@ -269,7 +304,14 @@ mod tests {
     fn test_build_health_payload_minimal_mode_returns_status_and_ready_only() {
         let health = health_check_state(true, false, HealthProbe::Readiness);
         with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("true"), || {
-            let payload = build_health_payload(health, true, false, "rustfs-endpoint", Some(123));
+            let payload = build_health_payload(
+                health,
+                true,
+                false,
+                &[crate::server::ReadinessDegradedReason::IamNotReady],
+                "rustfs-endpoint",
+                Some(123),
+            );
             assert_eq!(payload["status"], "degraded");
             assert_eq!(payload["ready"], false);
             assert!(payload.get("version").is_none());
@@ -278,4 +320,18 @@ mod tests {
             assert!(payload.get("uptime").is_none());
         });
     }
+
+    #[test]
+    fn test_build_health_payload_includes_degraded_reasons() {
+        let health = health_check_state(false, false, HealthProbe::Readiness);
+        let payload = build_health_payload(
+            health,
+            false,
+            false,
+            &[crate::server::ReadinessDegradedReason::StorageAndIamUnavailable],
+            "rustfs-endpoint",
+            None,
+        );
+        assert_eq!(payload["degradedReasons"][0], "storage_and_iam_unavailable");
+    }
 }

rustfs/src/server/layer.rs

@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use crate::admin::console::is_console_path;
-use crate::admin::handlers::health::{build_health_payload, collect_dependency_readiness, health_check_state, probe_from_path};
+use crate::admin::handlers::health::{build_health_payload, health_check_state, probe_from_path};
 use crate::error::ApiError;
 use crate::server::cors;
 use crate::server::hybrid::HybridBody;
 use crate::server::{
     ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, MINIO_ADMIN_V3_PREFIX, RPC_PREFIX,
-    RUSTFS_ADMIN_PREFIX,
+    RUSTFS_ADMIN_PREFIX, collect_dependency_readiness_report,
 };
 use crate::storage::apply_cors_headers;
 use crate::storage::request_context::{RequestContext, extract_request_id_from_headers};
@@ -639,12 +639,21 @@ where
     RestBody: From<Bytes>,
 {
     let probe = probe_from_path(&path);
-    let (storage_ready, iam_ready) = collect_dependency_readiness().await;
+    let readiness_report = collect_dependency_readiness_report().await;
+    let storage_ready = readiness_report.readiness.storage_ready;
+    let iam_ready = readiness_report.readiness.iam_ready;
     let health = health_check_state(storage_ready, iam_ready, probe);
     let body = if method == Method::HEAD {
         Bytes::new()
     } else {
-        let payload = build_health_payload(health, storage_ready, iam_ready, "rustfs-endpoint", None);
+        let payload = build_health_payload(
+            health,
+            storage_ready,
+            iam_ready,
+            &readiness_report.degraded_reasons,
+            "rustfs-endpoint",
+            None,
+        );
         Bytes::from(serde_json::to_vec(&payload).unwrap_or_else(|_| b"{}".to_vec()))
     };
 

rustfs/src/server/mod.rs

@@ -51,8 +51,11 @@ pub(crate) use prefix::{
     MINIO_ADMIN_V3_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, TONIC_PREFIX, VERSION,
 };
 pub(crate) use readiness::DependencyReadiness;
+pub(crate) use readiness::DependencyReadinessReport;
+pub(crate) use readiness::ReadinessDegradedReason;
 pub(crate) use readiness::ReadinessGateLayer;
 pub(crate) use readiness::collect_dependency_readiness;
+pub(crate) use readiness::collect_dependency_readiness_report;
 pub use readiness::publish_ready_when_runtime_ready;
 
 #[derive(Clone, Copy, Debug)]

rustfs/src/server/readiness.rs

@@ -18,6 +18,7 @@ use http::{Request as HttpRequest, Response, StatusCode};
 use http_body::Body;
 use http_body_util::{BodyExt, Full};
 use hyper::body::Incoming;
+use metrics::{counter, gauge};
 use rustfs_common::GlobalReadiness;
 use rustfs_ecstore::new_object_layer_fn;
 use rustfs_ecstore::store_api::StorageAPI;
@@ -39,6 +40,8 @@ use tracing::{debug, info};
 
 pub const STARTUP_RUNTIME_READINESS_MAX_WAIT: Duration = Duration::from_secs(30);
 pub const STARTUP_RUNTIME_READINESS_POLL_INTERVAL: Duration = Duration::from_secs(1);
+const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
+const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";
 
 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
 pub struct DependencyReadiness {
@@ -46,6 +49,29 @@ pub struct DependencyReadiness {
     pub iam_ready: bool,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReadinessDegradedReason {
+    StorageQuorumUnavailable,
+    IamNotReady,
+    StorageAndIamUnavailable,
+}
+
+impl ReadinessDegradedReason {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ReadinessDegradedReason::StorageQuorumUnavailable => "storage_quorum_unavailable",
+            ReadinessDegradedReason::IamNotReady => "iam_not_ready",
+            ReadinessDegradedReason::StorageAndIamUnavailable => "storage_and_iam_unavailable",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct DependencyReadinessReport {
+    pub readiness: DependencyReadiness,
+    pub degraded_reasons: Vec<ReadinessDegradedReason>,
+}
+
 /// ReadinessGateLayer ensures that the system components (IAM, Storage)
 /// are fully initialized before allowing any request to proceed.
 #[derive(Clone)]
@@ -320,8 +346,29 @@ fn storage_ready_from_runtime_state(info: &StorageInfo) -> bool {
     })
 }
 
+fn degraded_reasons(storage_ready: bool, iam_ready_raw: bool) -> Vec<ReadinessDegradedReason> {
+    match (storage_ready, iam_ready_raw) {
+        (true, true) => Vec::new(),
+        (false, false) => vec![ReadinessDegradedReason::StorageAndIamUnavailable],
+        (false, true) => vec![ReadinessDegradedReason::StorageQuorumUnavailable],
+        (true, false) => vec![ReadinessDegradedReason::IamNotReady],
+    }
+}
+
+fn record_readiness_report(report: &DependencyReadinessReport) {
+    let ready = report.readiness.storage_ready && report.readiness.iam_ready;
+    gauge!(METRIC_RUNTIME_READINESS_READY).set(if ready { 1.0 } else { 0.0 });
+    for reason in &report.degraded_reasons {
+        counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
+    }
+}
+
 pub async fn collect_dependency_readiness() -> DependencyReadiness {
-    let iam_ready = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
+    collect_dependency_readiness_report().await.readiness
+}
+
+pub async fn collect_dependency_readiness_report() -> DependencyReadinessReport {
+    let iam_ready_raw = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
     let storage_ready = if let Some(cached) = load_cached_storage_readiness().await {
         cached
     } else {
@@ -330,20 +377,32 @@ pub async fn collect_dependency_readiness() -> DependencyReadiness {
         computed
     };
 
-    DependencyReadiness {
+    let readiness = DependencyReadiness {
         storage_ready,
-        iam_ready: iam_ready && storage_ready,
-    }
+        iam_ready: iam_ready_raw,
+    };
+    let report = DependencyReadinessReport {
+        degraded_reasons: degraded_reasons(readiness.storage_ready, iam_ready_raw),
+        readiness,
+    };
+    record_readiness_report(&report);
+    report
 }
 
 async fn collect_dependency_readiness_uncached() -> DependencyReadiness {
-    let iam_ready = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
+    let iam_ready_raw = get_global_iam_sys().is_some_and(|sys| sys.is_ready());
     let storage_ready = collect_storage_readiness_uncached().await;
 
-    DependencyReadiness {
+    let readiness = DependencyReadiness {
         storage_ready,
-        iam_ready: iam_ready && storage_ready,
-    }
+        iam_ready: iam_ready_raw,
+    };
+    let report = DependencyReadinessReport {
+        degraded_reasons: degraded_reasons(readiness.storage_ready, iam_ready_raw),
+        readiness,
+    };
+    record_readiness_report(&report);
+    report.readiness
 }
 
 async fn collect_storage_readiness_uncached() -> bool {