fix(health): keep liveness peer independent (#6576)

Keep liveness probes local by avoiding readiness collection and omitting readiness-only fields from liveness payloads. Readiness and MinIO cluster probes continue to report dependency and quorum state.

Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/admin/console.rs

@@ -961,6 +961,44 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn console_liveness_omits_readiness_state() {
+        temp_env::async_with_vars([(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"))], async {
+            let object_traffic_health =
+                Arc::new(crate::app::object_traffic_health::ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+            let _stalled = object_traffic_health
+                .track_write_storage()
+                .expect("write tracking must be enabled");
+            let app_context =
+                crate::app::gating_test_env::app_context_with_object_traffic_health(Arc::clone(&object_traffic_health)).await;
+            let server_ctx = crate::runtime_sources::ServerContextSlot::new();
+            assert!(server_ctx.install(app_context));
+
+            let response = health_check(
+                Method::GET,
+                format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}")
+                    .parse()
+                    .expect("console liveness URI"),
+                Some(Extension(server_ctx)),
+            )
+            .await;
+
+            assert_eq!(response.status(), StatusCode::OK);
+            let body = response
+                .into_body()
+                .collect()
+                .await
+                .expect("console liveness body")
+                .to_bytes();
+            let payload: serde_json::Value = serde_json::from_slice(&body).expect("console liveness JSON");
+            assert_eq!(payload["status"], "ok");
+            assert!(payload.get("ready").is_none());
+            assert!(payload.get("degradedReasons").is_none());
+        })
+        .await;
+    }
+
     // setup_console_middleware_stack reads ENV_HEALTH_ENDPOINT_ENABLE (see above).
     #[tokio::test]
     #[serial]

rustfs/src/admin/handlers/health.rs

@@ -113,8 +113,8 @@ mod tests {
     fn test_liveness_state_iam_not_ready() {
         let state = health_check_state(true, false, true, true, HealthProbe::Liveness);
         assert_eq!(state.status_code, StatusCode::OK);
-        assert_eq!(state.status, "degraded");
-        assert!(!state.ready);
+        assert_eq!(state.status, "ok");
+        assert!(state.ready);
     }
 
     #[test]
@@ -137,8 +137,8 @@ mod tests {
     fn test_liveness_state_lock_not_ready() {
         let state = health_check_state(true, true, false, true, HealthProbe::Liveness);
         assert_eq!(state.status_code, StatusCode::OK);
-        assert_eq!(state.status, "degraded");
-        assert!(!state.ready);
+        assert_eq!(state.status, "ok");
+        assert!(state.ready);
     }
 
     #[test]
@@ -180,7 +180,7 @@ mod tests {
     #[test]
     fn test_readiness_probe_uses_node_collector_only() {
         assert_eq!(readiness_source_for_probe(HealthProbe::Readiness), Some(HealthReadinessSource::Node));
-        assert_eq!(readiness_source_for_probe(HealthProbe::Liveness), Some(HealthReadinessSource::Node));
+        assert_eq!(readiness_source_for_probe(HealthProbe::Liveness), None);
     }
 
     #[test]
@@ -246,7 +246,7 @@ mod tests {
     }
 
     #[test]
-    fn test_build_health_response_liveness_returns_200_when_deps_not_ready() {
+    fn test_build_health_response_liveness_omits_readiness_state_when_deps_not_ready() {
         let readiness_report = crate::shared_types::DependencyReadinessReport {
             readiness: crate::shared_types::DependencyReadiness {
                 storage_ready: false,
@@ -264,20 +264,17 @@ mod tests {
             None,
             None,
         );
-        // Liveness HTTP status remains 200 (process is alive).
         assert_eq!(parts.status_code, StatusCode::OK);
         let payload = parts.payload.expect("GET should include payload");
-        // But `ready` now reflects actual readiness state.
-        assert_eq!(payload["status"], "degraded");
-        assert_eq!(payload["ready"], false);
-        // Dependency details are included when readiness report is present.
-        assert!(payload.get("details").is_some());
-        assert!(payload.get("degradedReasons").is_some());
+        assert_eq!(payload["status"], "ok");
+        assert!(payload.get("ready").is_none());
+        assert!(payload.get("details").is_none());
+        assert!(payload.get("degradedReasons").is_none());
     }
 
     #[test]
     #[serial]
-    fn test_health_and_readiness_body_agree_when_only_lock_quorum_is_unavailable() {
+    fn test_liveness_body_stays_peer_independent_when_only_lock_quorum_is_unavailable() {
         with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
             let readiness_report = crate::shared_types::DependencyReadinessReport {
                 readiness: crate::shared_types::DependencyReadiness {
@@ -310,13 +307,13 @@ mod tests {
             assert_eq!(readiness.status_code, StatusCode::SERVICE_UNAVAILABLE);
             let liveness_payload = liveness.payload.expect("GET should include liveness payload");
             let readiness_payload = readiness.payload.expect("GET should include readiness payload");
-            assert_eq!(liveness_payload["status"], "degraded");
+            assert_eq!(liveness_payload["status"], "ok");
             assert_eq!(readiness_payload["status"], "degraded");
-            assert_eq!(liveness_payload["ready"], false);
+            assert!(liveness_payload.get("ready").is_none());
             assert_eq!(readiness_payload["ready"], false);
-            assert_eq!(liveness_payload["details"]["lock"]["ready"], false);
+            assert!(liveness_payload.get("details").is_none());
             assert_eq!(readiness_payload["details"]["lock"]["ready"], false);
-            assert_eq!(liveness_payload["degradedReasons"][0], "lock_quorum_unavailable");
+            assert!(liveness_payload.get("degradedReasons").is_none());
             assert_eq!(readiness_payload["degradedReasons"][0], "lock_quorum_unavailable");
         });
     }
@@ -350,6 +347,7 @@ mod tests {
         let health = health_check_state(true, false, true, true, HealthProbe::Readiness);
         with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("true"), || {
             let payload = build_health_payload(HealthPayloadContext {
+                probe: HealthProbe::Readiness,
                 health,
                 storage_ready: true,
                 iam_ready: false,
@@ -375,6 +373,7 @@ mod tests {
         with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
             let health = health_check_state(false, false, false, true, HealthProbe::Readiness);
             let payload = build_health_payload(HealthPayloadContext {
+                probe: HealthProbe::Readiness,
                 health,
                 storage_ready: false,
                 iam_ready: false,

rustfs/src/server/health.rs

@@ -54,6 +54,7 @@ pub(crate) struct HealthResponseParts {
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) struct HealthPayloadContext<'a> {
+    pub(crate) probe: HealthProbe,
     pub(crate) health: HealthCheckState,
     pub(crate) storage_ready: bool,
     pub(crate) iam_ready: bool,
@@ -97,7 +98,8 @@ fn apply_object_traffic_snapshot(report: &mut DependencyReadinessReport, snapsho
 
 pub(crate) fn readiness_source_for_probe(probe: HealthProbe) -> Option<HealthReadinessSource> {
     match probe {
-        HealthProbe::Liveness | HealthProbe::Readiness => Some(HealthReadinessSource::Node),
+        HealthProbe::Liveness => None,
+        HealthProbe::Readiness => Some(HealthReadinessSource::Node),
         HealthProbe::ClusterWrite => Some(HealthReadinessSource::ClusterWrite),
         HealthProbe::ClusterRead => Some(HealthReadinessSource::ClusterRead),
     }
@@ -113,13 +115,12 @@ pub(crate) fn health_check_state(
     let ready = storage_ready && iam_ready && lock_quorum_ready && peer_health_ready;
 
     if probe == HealthProbe::Liveness {
-        // Liveness always returns HTTP 200 (process is alive), but the `ready`
-        // field now reflects actual node readiness so that callers who inspect
-        // the body get a truthful signal instead of a hardcoded `true`.
+        // Liveness is intentionally local and peer-independent. Dependency
+        // readiness belongs to `/health/ready` and the MinIO cluster probes.
         return HealthCheckState {
             status_code: StatusCode::OK,
-            status: if ready { "ok" } else { "degraded" },
-            ready,
+            status: "ok",
+            ready: true,
         };
     }
 
@@ -295,7 +296,7 @@ pub(crate) fn build_health_response_parts(
                     lock_quorum_ready,
                     health_check_state(storage_ready, iam_ready, lock_quorum_ready, peer_health_ready, probe),
                     readiness_report.degraded_reasons.clone(),
-                    true,
+                    probe != HealthProbe::Liveness,
                 )
             }
             (HealthProbe::Readiness | HealthProbe::ClusterWrite | HealthProbe::ClusterRead, None) => (
@@ -341,6 +342,7 @@ pub(crate) fn build_health_response_parts(
         None
     } else {
         Some(build_health_payload(HealthPayloadContext {
+            probe,
             health,
             storage_ready,
             iam_ready,
@@ -361,20 +363,29 @@ pub(crate) fn build_health_response_parts(
 
 pub(crate) fn build_health_payload(ctx: HealthPayloadContext<'_>) -> Value {
     if health_minimal_response_enabled() {
-        return json!({
-            "status": ctx.health.status,
-            "ready": ctx.health.ready,
-        });
+        return if ctx.probe == HealthProbe::Liveness {
+            json!({
+                "status": ctx.health.status,
+            })
+        } else {
+            json!({
+                "status": ctx.health.status,
+                "ready": ctx.health.ready,
+            })
+        };
     }
 
     let mut payload = json!({
         "status": ctx.health.status,
-        "ready": ctx.health.ready,
         "service": ctx.service,
         "timestamp": jiff::Zoned::now().to_string(),
         "version": env!("CARGO_PKG_VERSION"),
     });
 
+    if ctx.probe != HealthProbe::Liveness {
+        payload["ready"] = json!(ctx.health.ready);
+    }
+
     if ctx.include_dependency_details {
         payload["details"] = build_component_details(ctx.storage_ready, ctx.iam_ready, ctx.lock_quorum_ready, ctx.kms_ready);
         payload["degradedReasons"] = build_degraded_reasons(ctx.degraded_reasons);
@@ -491,7 +502,7 @@ mod tests {
 
     #[test]
     #[serial]
-    fn liveness_and_readiness_payloads_share_lock_quorum_readiness() {
+    fn liveness_payload_omits_lock_quorum_readiness() {
         with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
             let mut report = ready_report();
             report.readiness.lock_quorum_ready = false;
@@ -506,11 +517,12 @@ mod tests {
             assert_eq!(readiness.status_code, StatusCode::SERVICE_UNAVAILABLE);
             let liveness_payload = liveness.payload.expect("liveness GET should include payload");
             let readiness_payload = readiness.payload.expect("readiness GET should include payload");
-            assert_eq!(liveness_payload["ready"], false);
+            assert_eq!(liveness_payload["status"], "ok");
+            assert!(liveness_payload.get("ready").is_none());
+            assert!(liveness_payload.get("details").is_none());
+            assert!(liveness_payload.get("degradedReasons").is_none());
             assert_eq!(readiness_payload["ready"], false);
-            assert_eq!(liveness_payload["details"]["lock"]["ready"], false);
             assert_eq!(readiness_payload["details"]["lock"]["ready"], false);
-            assert_eq!(liveness_payload["degradedReasons"], json!(["lock_quorum_unavailable"]));
             assert_eq!(readiness_payload["degradedReasons"], json!(["lock_quorum_unavailable"]));
         });
     }

rustfs/src/server/layer.rs

@@ -2718,13 +2718,10 @@ mod tests {
                 let body = BodyExt::collect(response.into_body()).await.expect("body").to_bytes();
                 let payload: serde_json::Value =
                     serde_json::from_slice(&body).expect("public liveness health response should be valid JSON");
-                assert!(matches!(payload["status"].as_str(), Some("ok" | "degraded")));
-                assert!(payload["ready"].is_boolean());
-                assert!(payload["details"].is_object());
-                assert!(payload["details"]["storage"]["ready"].is_boolean());
-                assert!(payload["details"]["iam"]["ready"].is_boolean());
-                assert!(payload["details"]["lock"]["ready"].is_boolean());
-                assert!(payload["degradedReasons"].is_array());
+                assert_eq!(payload["status"], "ok");
+                assert!(payload.get("ready").is_none());
+                assert!(payload.get("details").is_none());
+                assert!(payload.get("degradedReasons").is_none());
             },
         )
         .await;
@@ -3058,6 +3055,14 @@ mod tests {
                     .await
                     .expect("liveness response");
                 assert_eq!(response.status(), StatusCode::OK);
+                let body = BodyExt::collect(response.into_body())
+                    .await
+                    .expect("liveness body")
+                    .to_bytes();
+                let payload: serde_json::Value = serde_json::from_slice(&body).expect("liveness JSON");
+                assert_eq!(payload["status"], "ok");
+                assert!(payload.get("ready").is_none());
+                assert!(payload.get("degradedReasons").is_none());
                 assert_eq!(calls.load(Ordering::SeqCst), 0);
 
                 drop(stalled);