fix(health): keep liveness peer independent

Keep liveness probes local by avoiding readiness collection and omitting readiness-only fields from liveness payloads. Readiness and MinIO cluster probes continue to report dependency and quorum state.

Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/admin/console.rs

@@ -961,6 +961,44 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial]
+    async fn console_liveness_omits_readiness_state() {
+        temp_env::async_with_vars([(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"))], async {
+            let object_traffic_health =
+                Arc::new(crate::app::object_traffic_health::ObjectTrafficHealth::enabled_for_test(Duration::ZERO));
+            let _stalled = object_traffic_health
+                .track_write_storage()
+                .expect("write tracking must be enabled");
+            let app_context =
+                crate::app::gating_test_env::app_context_with_object_traffic_health(Arc::clone(&object_traffic_health)).await;
+            let server_ctx = crate::runtime_sources::ServerContextSlot::new();
+            assert!(server_ctx.install(app_context));
+
+            let response = health_check(
+                Method::GET,
+                format!("{CONSOLE_PREFIX}{HEALTH_PREFIX}")
+                    .parse()
+                    .expect("console liveness URI"),
+                Some(Extension(server_ctx)),
+            )
+            .await;
+
+            assert_eq!(response.status(), StatusCode::OK);
+            let body = response
+                .into_body()
+                .collect()
+                .await
+                .expect("console liveness body")
+                .to_bytes();
+            let payload: serde_json::Value = serde_json::from_slice(&body).expect("console liveness JSON");
+            assert_eq!(payload["status"], "ok");
+            assert!(payload.get("ready").is_none());
+            assert!(payload.get("degradedReasons").is_none());
+        })
+        .await;
+    }
+
     // setup_console_middleware_stack reads ENV_HEALTH_ENDPOINT_ENABLE (see above).
     #[tokio::test]
     #[serial]

rustfs/src/admin/handlers/health.rs

@@ -113,8 +113,8 @@ mod tests {
     fn test_liveness_state_iam_not_ready() {
         let state = health_check_state(true, false, true, true, HealthProbe::Liveness);
         assert_eq!(state.status_code, StatusCode::OK);
-        assert_eq!(state.status, "degraded");
-        assert!(!state.ready);
+        assert_eq!(state.status, "ok");
+        assert!(state.ready);
     }
 
     #[test]
@@ -128,6 +128,14 @@ mod tests {
     #[test]
     fn test_readiness_state_lock_not_ready() {
         let state = health_check_state(true, true, false, true, HealthProbe::Readiness);
+        assert_eq!(state.status_code, StatusCode::SERVICE_UNAVAILABLE);
+        assert_eq!(state.status, "degraded");
+        assert!(!state.ready);
+    }
+
+    #[test]
+    fn test_liveness_state_lock_not_ready() {
+        let state = health_check_state(true, true, false, true, HealthProbe::Liveness);
         assert_eq!(state.status_code, StatusCode::OK);
         assert_eq!(state.status, "ok");
         assert!(state.ready);
@@ -172,7 +180,7 @@ mod tests {
     #[test]
     fn test_readiness_probe_uses_node_collector_only() {
         assert_eq!(readiness_source_for_probe(HealthProbe::Readiness), Some(HealthReadinessSource::Node));
-        assert_eq!(readiness_source_for_probe(HealthProbe::Liveness), Some(HealthReadinessSource::Node));
+        assert_eq!(readiness_source_for_probe(HealthProbe::Liveness), None);
     }
 
     #[test]
@@ -238,7 +246,7 @@ mod tests {
     }
 
     #[test]
-    fn test_build_health_response_liveness_returns_200_when_deps_not_ready() {
+    fn test_build_health_response_liveness_omits_readiness_state_when_deps_not_ready() {
         let readiness_report = crate::shared_types::DependencyReadinessReport {
             readiness: crate::shared_types::DependencyReadiness {
                 storage_ready: false,
@@ -256,15 +264,58 @@ mod tests {
             None,
             None,
         );
-        // Liveness HTTP status remains 200 (process is alive).
         assert_eq!(parts.status_code, StatusCode::OK);
         let payload = parts.payload.expect("GET should include payload");
-        // But `ready` now reflects actual readiness state.
-        assert_eq!(payload["status"], "degraded");
-        assert_eq!(payload["ready"], false);
-        // Dependency details are included when readiness report is present.
-        assert!(payload.get("details").is_some());
-        assert!(payload.get("degradedReasons").is_some());
+        assert_eq!(payload["status"], "ok");
+        assert!(payload.get("ready").is_none());
+        assert!(payload.get("details").is_none());
+        assert!(payload.get("degradedReasons").is_none());
+    }
+
+    #[test]
+    #[serial]
+    fn test_liveness_body_stays_peer_independent_when_only_lock_quorum_is_unavailable() {
+        with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
+            let readiness_report = crate::shared_types::DependencyReadinessReport {
+                readiness: crate::shared_types::DependencyReadiness {
+                    storage_ready: true,
+                    iam_ready: true,
+                    lock_quorum_ready: false,
+                    peer_health_ready: true,
+                },
+                degraded_reasons: vec![crate::shared_types::ReadinessDegradedReason::LockQuorumUnavailable],
+            };
+
+            let liveness = build_health_response_parts(
+                Method::GET,
+                HealthProbe::Liveness,
+                Some(&readiness_report),
+                "rustfs-endpoint",
+                None,
+                None,
+            );
+            let readiness = build_health_response_parts(
+                Method::GET,
+                HealthProbe::Readiness,
+                Some(&readiness_report),
+                "rustfs-endpoint",
+                None,
+                None,
+            );
+
+            assert_eq!(liveness.status_code, StatusCode::OK);
+            assert_eq!(readiness.status_code, StatusCode::SERVICE_UNAVAILABLE);
+            let liveness_payload = liveness.payload.expect("GET should include liveness payload");
+            let readiness_payload = readiness.payload.expect("GET should include readiness payload");
+            assert_eq!(liveness_payload["status"], "ok");
+            assert_eq!(readiness_payload["status"], "degraded");
+            assert!(liveness_payload.get("ready").is_none());
+            assert_eq!(readiness_payload["ready"], false);
+            assert!(liveness_payload.get("details").is_none());
+            assert_eq!(readiness_payload["details"]["lock"]["ready"], false);
+            assert!(liveness_payload.get("degradedReasons").is_none());
+            assert_eq!(readiness_payload["degradedReasons"][0], "lock_quorum_unavailable");
+        });
     }
 
     #[test]
@@ -296,6 +347,7 @@ mod tests {
         let health = health_check_state(true, false, true, true, HealthProbe::Readiness);
         with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("true"), || {
             let payload = build_health_payload(HealthPayloadContext {
+                probe: HealthProbe::Readiness,
                 health,
                 storage_ready: true,
                 iam_ready: false,
@@ -321,6 +373,7 @@ mod tests {
         with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
             let health = health_check_state(false, false, false, true, HealthProbe::Readiness);
             let payload = build_health_payload(HealthPayloadContext {
+                probe: HealthProbe::Readiness,
                 health,
                 storage_ready: false,
                 iam_ready: false,

rustfs/src/server/health.rs

@@ -39,12 +39,6 @@ pub(crate) enum HealthProbe {
     ClusterRead,
 }
 
-impl HealthProbe {
-    const fn requires_lock_quorum(self) -> bool {
-        matches!(self, Self::ClusterWrite | Self::ClusterRead)
-    }
-}
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum HealthReadinessSource {
     Node,
@@ -60,6 +54,7 @@ pub(crate) struct HealthResponseParts {
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) struct HealthPayloadContext<'a> {
+    pub(crate) probe: HealthProbe,
     pub(crate) health: HealthCheckState,
     pub(crate) storage_ready: bool,
     pub(crate) iam_ready: bool,
@@ -103,7 +98,8 @@ fn apply_object_traffic_snapshot(report: &mut DependencyReadinessReport, snapsho
 
 pub(crate) fn readiness_source_for_probe(probe: HealthProbe) -> Option<HealthReadinessSource> {
     match probe {
-        HealthProbe::Liveness | HealthProbe::Readiness => Some(HealthReadinessSource::Node),
+        HealthProbe::Liveness => None,
+        HealthProbe::Readiness => Some(HealthReadinessSource::Node),
         HealthProbe::ClusterWrite => Some(HealthReadinessSource::ClusterWrite),
         HealthProbe::ClusterRead => Some(HealthReadinessSource::ClusterRead),
     }
@@ -116,16 +112,15 @@ pub(crate) fn health_check_state(
     peer_health_ready: bool,
     probe: HealthProbe,
 ) -> HealthCheckState {
-    let ready = storage_ready && iam_ready && peer_health_ready && (!probe.requires_lock_quorum() || lock_quorum_ready);
+    let ready = storage_ready && iam_ready && lock_quorum_ready && peer_health_ready;
 
     if probe == HealthProbe::Liveness {
-        // Liveness always returns HTTP 200 (process is alive), but the `ready`
-        // field now reflects actual node readiness so that callers who inspect
-        // the body get a truthful signal instead of a hardcoded `true`.
+        // Liveness is intentionally local and peer-independent. Dependency
+        // readiness belongs to `/health/ready` and the MinIO cluster probes.
         return HealthCheckState {
             status_code: StatusCode::OK,
-            status: if ready { "ok" } else { "degraded" },
-            ready,
+            status: "ok",
+            ready: true,
         };
     }
 
@@ -301,7 +296,7 @@ pub(crate) fn build_health_response_parts(
                     lock_quorum_ready,
                     health_check_state(storage_ready, iam_ready, lock_quorum_ready, peer_health_ready, probe),
                     readiness_report.degraded_reasons.clone(),
-                    true,
+                    probe != HealthProbe::Liveness,
                 )
             }
             (HealthProbe::Readiness | HealthProbe::ClusterWrite | HealthProbe::ClusterRead, None) => (
@@ -347,6 +342,7 @@ pub(crate) fn build_health_response_parts(
         None
     } else {
         Some(build_health_payload(HealthPayloadContext {
+            probe,
             health,
             storage_ready,
             iam_ready,
@@ -367,20 +363,29 @@ pub(crate) fn build_health_response_parts(
 
 pub(crate) fn build_health_payload(ctx: HealthPayloadContext<'_>) -> Value {
     if health_minimal_response_enabled() {
-        return json!({
-            "status": ctx.health.status,
-            "ready": ctx.health.ready,
-        });
+        return if ctx.probe == HealthProbe::Liveness {
+            json!({
+                "status": ctx.health.status,
+            })
+        } else {
+            json!({
+                "status": ctx.health.status,
+                "ready": ctx.health.ready,
+            })
+        };
     }
 
     let mut payload = json!({
         "status": ctx.health.status,
-        "ready": ctx.health.ready,
         "service": ctx.service,
         "timestamp": jiff::Zoned::now().to_string(),
         "version": env!("CARGO_PKG_VERSION"),
     });
 
+    if ctx.probe != HealthProbe::Liveness {
+        payload["ready"] = json!(ctx.health.ready);
+    }
+
     if ctx.include_dependency_details {
         payload["details"] = build_component_details(ctx.storage_ready, ctx.iam_ready, ctx.lock_quorum_ready, ctx.kms_ready);
         payload["degradedReasons"] = build_degraded_reasons(ctx.degraded_reasons);
@@ -495,6 +500,33 @@ mod tests {
         assert!(parts.payload.is_none());
     }
 
+    #[test]
+    #[serial]
+    fn liveness_payload_omits_lock_quorum_readiness() {
+        with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
+            let mut report = ready_report();
+            report.readiness.lock_quorum_ready = false;
+            report.degraded_reasons.push(ReadinessDegradedReason::LockQuorumUnavailable);
+
+            let liveness =
+                build_health_response_parts(Method::GET, HealthProbe::Liveness, Some(&report), "rustfs-endpoint", None, None);
+            let readiness =
+                build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs-endpoint", None, None);
+
+            assert_eq!(liveness.status_code, StatusCode::OK);
+            assert_eq!(readiness.status_code, StatusCode::SERVICE_UNAVAILABLE);
+            let liveness_payload = liveness.payload.expect("liveness GET should include payload");
+            let readiness_payload = readiness.payload.expect("readiness GET should include payload");
+            assert_eq!(liveness_payload["status"], "ok");
+            assert!(liveness_payload.get("ready").is_none());
+            assert!(liveness_payload.get("details").is_none());
+            assert!(liveness_payload.get("degradedReasons").is_none());
+            assert_eq!(readiness_payload["ready"], false);
+            assert_eq!(readiness_payload["details"]["lock"]["ready"], false);
+            assert_eq!(readiness_payload["degradedReasons"], json!(["lock_quorum_unavailable"]));
+        });
+    }
+
     #[test]
     fn object_stall_overlay_records_the_final_readiness_metrics() {
         let recorder = DebuggingRecorder::new();

rustfs/src/server/layer.rs

@@ -2683,48 +2683,47 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn public_health_endpoint_layer_handles_health_before_inner_service() {
-        async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async {
-            let inner = CountingHybridService::default();
-            let calls = inner.calls();
-            let mut service = public_health_layer().layer(inner);
+        async_with_vars(
+            [
+                (rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true")),
+                (rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false")),
+            ],
+            async {
+                let inner = CountingHybridService::default();
+                let calls = inner.calls();
+                let mut service = public_health_layer().layer(inner);
 
-            let response = service
-                .call(
-                    Request::builder()
-                        .method(Method::GET)
-                        .uri(HEALTH_PREFIX)
-                        .header(http::header::HOST, "localhost:9000")
-                        .body(Full::<Bytes>::from(Bytes::new()))
-                        .expect("request"),
-                )
-                .await
-                .expect("health response");
+                let response = service
+                    .call(
+                        Request::builder()
+                            .method(Method::GET)
+                            .uri(HEALTH_PREFIX)
+                            .header(http::header::HOST, "localhost:9000")
+                            .body(Full::<Bytes>::from(Bytes::new()))
+                            .expect("request"),
+                    )
+                    .await
+                    .expect("health response");
 
-            assert_eq!(response.status(), StatusCode::OK);
-            assert_eq!(calls.load(Ordering::SeqCst), 0);
-            assert_eq!(
-                response
-                    .headers()
-                    .get(http::header::CONTENT_TYPE)
-                    .and_then(|value| value.to_str().ok()),
-                Some("application/json")
-            );
+                assert_eq!(response.status(), StatusCode::OK);
+                assert_eq!(calls.load(Ordering::SeqCst), 0);
+                assert_eq!(
+                    response
+                        .headers()
+                        .get(http::header::CONTENT_TYPE)
+                        .and_then(|value| value.to_str().ok()),
+                    Some("application/json")
+                );
 
-            let body = BodyExt::collect(response.into_body()).await.expect("body").to_bytes();
-            let payload: serde_json::Value =
-                serde_json::from_slice(&body).expect("public liveness health response should be valid JSON");
-            assert_eq!(payload["status"], "degraded");
-            assert_eq!(payload["ready"], false);
-            assert_eq!(
-                payload["details"],
-                serde_json::json!({
-                    "storage": { "status": "disconnected", "ready": false },
-                    "iam": { "status": "disconnected", "ready": false },
-                    "lock": { "status": "connected", "ready": true },
-                })
-            );
-            assert_eq!(payload["degradedReasons"], serde_json::json!(["storage_and_iam_unavailable"]));
-        })
+                let body = BodyExt::collect(response.into_body()).await.expect("body").to_bytes();
+                let payload: serde_json::Value =
+                    serde_json::from_slice(&body).expect("public liveness health response should be valid JSON");
+                assert_eq!(payload["status"], "ok");
+                assert!(payload.get("ready").is_none());
+                assert!(payload.get("details").is_none());
+                assert!(payload.get("degradedReasons").is_none());
+            },
+        )
         .await;
     }
 
@@ -3056,6 +3055,14 @@ mod tests {
                     .await
                     .expect("liveness response");
                 assert_eq!(response.status(), StatusCode::OK);
+                let body = BodyExt::collect(response.into_body())
+                    .await
+                    .expect("liveness body")
+                    .to_bytes();
+                let payload: serde_json::Value = serde_json::from_slice(&body).expect("liveness JSON");
+                assert_eq!(payload["status"], "ok");
+                assert!(payload.get("ready").is_none());
+                assert!(payload.get("degradedReasons").is_none());
                 assert_eq!(calls.load(Ordering::SeqCst), 0);
 
                 drop(stalled);