fix: harden lock timeouts, health, and heal paths (#3815)

This commit is contained in:
cxymds
2026-06-24 15:48:06 +08:00
committed by GitHub
parent 94034ef406
commit eef8f43251
13 changed files with 400 additions and 179 deletions
+5 -2
View File
@@ -19,7 +19,7 @@ use crate::license::has_valid_license;
use crate::server::has_path_prefix;
use crate::server::{
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier, LICENSE, RUSTFS_ADMIN_PREFIX,
RequestContextLayer, VERSION,
RequestContextLayer, VERSION, liveness_dependency_readiness_report,
};
use crate::storage::request_context::RequestContext;
use crate::version::build;
@@ -597,7 +597,10 @@ async fn health_check(method: Method, uri: Uri) -> Response {
} else {
HealthProbe::Liveness
};
let readiness_report = collect_dependency_readiness().await;
let readiness_report = match probe {
HealthProbe::Liveness => liveness_dependency_readiness_report(),
HealthProbe::Readiness => collect_dependency_readiness().await,
};
let uptime = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
+14 -2
View File
@@ -16,7 +16,7 @@ use super::profile::{TriggerProfileCPU, TriggerProfileMemory};
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::server::{
HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
collect_dependency_readiness_report as collect_runtime_dependency_readiness_report,
collect_dependency_readiness_report as collect_runtime_dependency_readiness_report, liveness_dependency_readiness_report,
};
use http::{HeaderMap, HeaderValue};
use hyper::{Method, StatusCode};
@@ -245,7 +245,10 @@ impl Operation for HealthCheckHandler {
}
let probe = probe_from_path(req.uri.path());
let readiness_report = collect_dependency_readiness().await;
let readiness_report = match probe {
HealthProbe::Liveness => liveness_dependency_readiness_report(),
HealthProbe::Readiness => collect_dependency_readiness().await,
};
let response_parts = build_health_response_parts(method.clone(), probe, &readiness_report, "rustfs-endpoint", None, None);
@@ -292,6 +295,15 @@ mod tests {
assert!(!state.ready);
}
#[test]
fn test_liveness_dependency_readiness_report_is_lightweight_ready() {
let report = liveness_dependency_readiness_report();
assert!(report.readiness.storage_ready);
assert!(report.readiness.iam_ready);
assert!(report.readiness.lock_quorum_ready);
assert!(report.degraded_reasons.is_empty());
}
#[test]
fn test_readiness_state_iam_not_ready() {
let state = health_check_state(true, false, true, HealthProbe::Readiness);
+5 -1
View File
@@ -23,6 +23,7 @@ use crate::server::{
ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX,
MINIO_ADMIN_V3_PREFIX, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX,
active_http_requests, collect_dependency_readiness_report, has_path_prefix, is_admin_path, is_table_catalog_path,
liveness_dependency_readiness_report,
};
use crate::storage::apply_cors_headers;
use crate::storage::request_context::{
@@ -938,7 +939,10 @@ where
.expect("failed to build health busy response");
}
let readiness_report = collect_dependency_readiness_report().await;
let readiness_report = match probe {
HealthProbe::Liveness => liveness_dependency_readiness_report(),
HealthProbe::Readiness => collect_dependency_readiness_report().await,
};
let kms_ready = if probe == HealthProbe::Readiness && health_compat_kms_ready_check_enabled() {
Some(health_kms_ready().await)
} else {
+1
View File
@@ -61,6 +61,7 @@ pub(crate) use readiness::ReadinessDegradedReason;
pub(crate) use readiness::ReadinessGateLayer;
pub(crate) use readiness::collect_dependency_readiness;
pub(crate) use readiness::collect_dependency_readiness_report;
pub(crate) use readiness::liveness_dependency_readiness_report;
pub use readiness::publish_ready_when_runtime_ready;
pub(crate) use readiness::snapshot_dependency_readiness_report;
+11
View File
@@ -442,6 +442,17 @@ fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) ->
}
}
pub(crate) fn liveness_dependency_readiness_report() -> DependencyReadinessReport {
DependencyReadinessReport {
readiness: DependencyReadiness {
storage_ready: true,
iam_ready: true,
lock_quorum_ready: true,
},
degraded_reasons: Vec::new(),
}
}
pub async fn collect_dependency_readiness() -> DependencyReadiness {
collect_dependency_readiness_report().await.readiness
}