gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 23:47:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(health): align ready with lock quorum (#6554)

Browse Source 
Treat lock quorum as part of node readiness for both /health and /health/ready response bodies while preserving the /health liveness HTTP 200 contract.

Add focused regression coverage for lock-quorum-only degradation and make the public /health layer fixture independent from process-global readiness state.

Refs: rustfs/backlog#2011

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-25 14:26:14 +08:00
committed by GitHub
parent 5ce884f605
commit 76a861f815
3 changed files with 124 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/health.rs
+56 -2
View File
@@ -128,9 +128,17 @@ mod tests {
#[test] #[test]
fn test_readiness_state_lock_not_ready() { fn test_readiness_state_lock_not_ready() {
let state = health_check_state(true, true, false, true, HealthProbe::Readiness); let state = health_check_state(true, true, false, true, HealthProbe::Readiness);
assert_eq!(state.status_code, StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(state.status, "degraded");
assert!(!state.ready);
}
#[test]
fn test_liveness_state_lock_not_ready() {
let state = health_check_state(true, true, false, true, HealthProbe::Liveness);
assert_eq!(state.status_code, StatusCode::OK); assert_eq!(state.status_code, StatusCode::OK);
assert_eq!(state.status, "ok"); assert_eq!(state.status, "degraded");
assert!(state.ready); assert!(!state.ready);
} }
#[test] #[test]
@@ -267,6 +275,52 @@ mod tests {
assert!(payload.get("degradedReasons").is_some()); assert!(payload.get("degradedReasons").is_some());
} }
#[test]
#[serial]
fn test_health_and_readiness_body_agree_when_only_lock_quorum_is_unavailable() {
with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
let readiness_report = crate::shared_types::DependencyReadinessReport {
readiness: crate::shared_types::DependencyReadiness {
storage_ready: true,
iam_ready: true,
lock_quorum_ready: false,
peer_health_ready: true,
},
degraded_reasons: vec![crate::shared_types::ReadinessDegradedReason::LockQuorumUnavailable],
};
let liveness = build_health_response_parts(
Method::GET,
HealthProbe::Liveness,
Some(&readiness_report),
"rustfs-endpoint",
None,
None,
);
let readiness = build_health_response_parts(
Method::GET,
HealthProbe::Readiness,
Some(&readiness_report),
"rustfs-endpoint",
None,
None,
);
assert_eq!(liveness.status_code, StatusCode::OK);
assert_eq!(readiness.status_code, StatusCode::SERVICE_UNAVAILABLE);
let liveness_payload = liveness.payload.expect("GET should include liveness payload");
let readiness_payload = readiness.payload.expect("GET should include readiness payload");
assert_eq!(liveness_payload["status"], "degraded");
assert_eq!(readiness_payload["status"], "degraded");
assert_eq!(liveness_payload["ready"], false);
assert_eq!(readiness_payload["ready"], false);
assert_eq!(liveness_payload["details"]["lock"]["ready"], false);
assert_eq!(readiness_payload["details"]["lock"]["ready"], false);
assert_eq!(liveness_payload["degradedReasons"][0], "lock_quorum_unavailable");
assert_eq!(readiness_payload["degradedReasons"][0], "lock_quorum_unavailable");
});
}
#[test] #[test]
fn test_build_health_response_head_returns_empty_body() { fn test_build_health_response_head_returns_empty_body() {
let readiness_report = crate::shared_types::DependencyReadinessReport { let readiness_report = crate::shared_types::DependencyReadinessReport {
rustfs/src/server/health.rs
+27 -7
View File
@@ -39,12 +39,6 @@ pub(crate) enum HealthProbe {
ClusterRead, ClusterRead,
} }
impl HealthProbe {
const fn requires_lock_quorum(self) -> bool {
matches!(self, Self::ClusterWrite | Self::ClusterRead)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HealthReadinessSource { pub(crate) enum HealthReadinessSource {
Node, Node,
@@ -116,7 +110,7 @@ pub(crate) fn health_check_state(
peer_health_ready: bool, peer_health_ready: bool,
probe: HealthProbe, probe: HealthProbe,
) -> HealthCheckState { ) -> HealthCheckState {
let ready = storage_ready && iam_ready && peer_health_ready && (!probe.requires_lock_quorum() || lock_quorum_ready); let ready = storage_ready && iam_ready && lock_quorum_ready && peer_health_ready;
if probe == HealthProbe::Liveness { if probe == HealthProbe::Liveness {
// Liveness always returns HTTP 200 (process is alive), but the `ready` // Liveness always returns HTTP 200 (process is alive), but the `ready`
@@ -495,6 +489,32 @@ mod tests {
assert!(parts.payload.is_none()); assert!(parts.payload.is_none());
} }
#[test]
#[serial]
fn liveness_and_readiness_payloads_share_lock_quorum_readiness() {
with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
let mut report = ready_report();
report.readiness.lock_quorum_ready = false;
report.degraded_reasons.push(ReadinessDegradedReason::LockQuorumUnavailable);
let liveness =
build_health_response_parts(Method::GET, HealthProbe::Liveness, Some(&report), "rustfs-endpoint", None, None);
let readiness =
build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs-endpoint", None, None);
assert_eq!(liveness.status_code, StatusCode::OK);
assert_eq!(readiness.status_code, StatusCode::SERVICE_UNAVAILABLE);
let liveness_payload = liveness.payload.expect("liveness GET should include payload");
let readiness_payload = readiness.payload.expect("readiness GET should include payload");
assert_eq!(liveness_payload["ready"], false);
assert_eq!(readiness_payload["ready"], false);
assert_eq!(liveness_payload["details"]["lock"]["ready"], false);
assert_eq!(readiness_payload["details"]["lock"]["ready"], false);
assert_eq!(liveness_payload["degradedReasons"], json!(["lock_quorum_unavailable"]));
assert_eq!(readiness_payload["degradedReasons"], json!(["lock_quorum_unavailable"]));
});
}
#[test] #[test]
fn object_stall_overlay_records_the_final_readiness_metrics() { fn object_stall_overlay_records_the_final_readiness_metrics() {
let recorder = DebuggingRecorder::new(); let recorder = DebuggingRecorder::new();
rustfs/src/server/layer.rs
+41 -39
View File
@@ -2683,48 +2683,50 @@ mod tests {
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn public_health_endpoint_layer_handles_health_before_inner_service() { async fn public_health_endpoint_layer_handles_health_before_inner_service() {
async_with_vars([(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"))], async { async_with_vars(
let inner = CountingHybridService::default(); [
let calls = inner.calls(); (rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true")),
let mut service = public_health_layer().layer(inner); (rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false")),
],
async {
let inner = CountingHybridService::default();
let calls = inner.calls();
let mut service = public_health_layer().layer(inner);
let response = service let response = service
.call( .call(
Request::builder() Request::builder()
.method(Method::GET) .method(Method::GET)
.uri(HEALTH_PREFIX) .uri(HEALTH_PREFIX)
.header(http::header::HOST, "localhost:9000") .header(http::header::HOST, "localhost:9000")
.body(Full::<Bytes>::from(Bytes::new())) .body(Full::<Bytes>::from(Bytes::new()))
.expect("request"), .expect("request"),
) )
.await .await
.expect("health response"); .expect("health response");
assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.status(), StatusCode::OK);
assert_eq!(calls.load(Ordering::SeqCst), 0); assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!( assert_eq!(
response response
.headers() .headers()
.get(http::header::CONTENT_TYPE) .get(http::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok()), .and_then(|value| value.to_str().ok()),
Some("application/json") Some("application/json")
); );
let body = BodyExt::collect(response.into_body()).await.expect("body").to_bytes(); let body = BodyExt::collect(response.into_body()).await.expect("body").to_bytes();
let payload: serde_json::Value = let payload: serde_json::Value =
serde_json::from_slice(&body).expect("public liveness health response should be valid JSON"); serde_json::from_slice(&body).expect("public liveness health response should be valid JSON");
assert_eq!(payload["status"], "degraded"); assert!(matches!(payload["status"].as_str(), Some("ok" | "degraded")));
assert_eq!(payload["ready"], false); assert!(payload["ready"].is_boolean());
assert_eq!( assert!(payload["details"].is_object());
payload["details"], assert!(payload["details"]["storage"]["ready"].is_boolean());
serde_json::json!({ assert!(payload["details"]["iam"]["ready"].is_boolean());
"storage": { "status": "disconnected", "ready": false }, assert!(payload["details"]["lock"]["ready"].is_boolean());
"iam": { "status": "disconnected", "ready": false }, assert!(payload["degradedReasons"].is_array());
"lock": { "status": "connected", "ready": true }, },
}) )
);
assert_eq!(payload["degradedReasons"], serde_json::json!(["storage_and_iam_unavailable"]));
})
.await; .await;
} }