mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-06 21:33:14 +00:00
fix: harden lock timeouts, health, and heal paths (#3815)
This commit is contained in:
@@ -19,7 +19,7 @@ use crate::license::has_valid_license;
|
||||
use crate::server::has_path_prefix;
|
||||
use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier, LICENSE, RUSTFS_ADMIN_PREFIX,
|
||||
RequestContextLayer, VERSION,
|
||||
RequestContextLayer, VERSION, liveness_dependency_readiness_report,
|
||||
};
|
||||
use crate::storage::request_context::RequestContext;
|
||||
use crate::version::build;
|
||||
@@ -597,7 +597,10 @@ async fn health_check(method: Method, uri: Uri) -> Response {
|
||||
} else {
|
||||
HealthProbe::Liveness
|
||||
};
|
||||
let readiness_report = collect_dependency_readiness().await;
|
||||
let readiness_report = match probe {
|
||||
HealthProbe::Liveness => liveness_dependency_readiness_report(),
|
||||
HealthProbe::Readiness => collect_dependency_readiness().await,
|
||||
};
|
||||
let uptime = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::profile::{TriggerProfileCPU, TriggerProfileMemory};
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::server::{
|
||||
HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
|
||||
collect_dependency_readiness_report as collect_runtime_dependency_readiness_report,
|
||||
collect_dependency_readiness_report as collect_runtime_dependency_readiness_report, liveness_dependency_readiness_report,
|
||||
};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use hyper::{Method, StatusCode};
|
||||
@@ -245,7 +245,10 @@ impl Operation for HealthCheckHandler {
|
||||
}
|
||||
|
||||
let probe = probe_from_path(req.uri.path());
|
||||
let readiness_report = collect_dependency_readiness().await;
|
||||
let readiness_report = match probe {
|
||||
HealthProbe::Liveness => liveness_dependency_readiness_report(),
|
||||
HealthProbe::Readiness => collect_dependency_readiness().await,
|
||||
};
|
||||
|
||||
let response_parts = build_health_response_parts(method.clone(), probe, &readiness_report, "rustfs-endpoint", None, None);
|
||||
|
||||
@@ -292,6 +295,15 @@ mod tests {
|
||||
assert!(!state.ready);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_liveness_dependency_readiness_report_is_lightweight_ready() {
|
||||
let report = liveness_dependency_readiness_report();
|
||||
assert!(report.readiness.storage_ready);
|
||||
assert!(report.readiness.iam_ready);
|
||||
assert!(report.readiness.lock_quorum_ready);
|
||||
assert!(report.degraded_reasons.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_readiness_state_iam_not_ready() {
|
||||
let state = health_check_state(true, false, true, HealthProbe::Readiness);
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::server::{
|
||||
ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX,
|
||||
MINIO_ADMIN_V3_PREFIX, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX,
|
||||
active_http_requests, collect_dependency_readiness_report, has_path_prefix, is_admin_path, is_table_catalog_path,
|
||||
liveness_dependency_readiness_report,
|
||||
};
|
||||
use crate::storage::apply_cors_headers;
|
||||
use crate::storage::request_context::{
|
||||
@@ -938,7 +939,10 @@ where
|
||||
.expect("failed to build health busy response");
|
||||
}
|
||||
|
||||
let readiness_report = collect_dependency_readiness_report().await;
|
||||
let readiness_report = match probe {
|
||||
HealthProbe::Liveness => liveness_dependency_readiness_report(),
|
||||
HealthProbe::Readiness => collect_dependency_readiness_report().await,
|
||||
};
|
||||
let kms_ready = if probe == HealthProbe::Readiness && health_compat_kms_ready_check_enabled() {
|
||||
Some(health_kms_ready().await)
|
||||
} else {
|
||||
|
||||
@@ -61,6 +61,7 @@ pub(crate) use readiness::ReadinessDegradedReason;
|
||||
pub(crate) use readiness::ReadinessGateLayer;
|
||||
pub(crate) use readiness::collect_dependency_readiness;
|
||||
pub(crate) use readiness::collect_dependency_readiness_report;
|
||||
pub(crate) use readiness::liveness_dependency_readiness_report;
|
||||
pub use readiness::publish_ready_when_runtime_ready;
|
||||
pub(crate) use readiness::snapshot_dependency_readiness_report;
|
||||
|
||||
|
||||
@@ -442,6 +442,17 @@ fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) ->
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn liveness_dependency_readiness_report() -> DependencyReadinessReport {
|
||||
DependencyReadinessReport {
|
||||
readiness: DependencyReadiness {
|
||||
storage_ready: true,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
},
|
||||
degraded_reasons: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn collect_dependency_readiness() -> DependencyReadiness {
|
||||
collect_dependency_readiness_report().await.readiness
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user