fix: report stalled object traffic as unready (#5936)

* fix: report stalled object traffic as unready

* fix: track fully received PUT storage progress
This commit is contained in:
cxymds
2026-08-11 10:55:22 +08:00
committed by GitHub
parent 3289d40ce9
commit 2aa0148454
14 changed files with 1164 additions and 60 deletions
+8
View File
@@ -97,6 +97,14 @@ Current guidance:
- enables minimal payload mode for GET health responses (`status`, `ready` only).
- `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS`
- TTL for readiness cache evaluation.
- `RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE`
- withdraws readiness when bounded object read/write stages stop completing while requests remain active.
- default is `true`.
- `RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS`
- maximum time without completion in a bounded object stage before readiness is withdrawn.
- default is `30000`; `0` uses the default.
- the effective value is at least 5 seconds longer than `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`.
- this readiness SLO is independent of disk read/write failure deadlines and may withdraw traffic before those deadlines expire.
- `RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE`
- enables busy protection behavior for health probes.
- default is `false`.
+13
View File
@@ -22,6 +22,19 @@ pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
/// Enable readiness withdrawal when bounded object read/write stages stop
/// completing while requests remain active.
pub const ENV_HEALTH_OBJECT_PROGRESS_ENABLE: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE";
pub const DEFAULT_HEALTH_OBJECT_PROGRESS_ENABLE: bool = true;
/// Requested time without completion in a bounded object stage before local
/// readiness is withdrawn (milliseconds). A value of `0` uses the default;
/// runtime adds a safety floor based on the object-lock acquisition timeout.
pub const ENV_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS";
pub const DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: u64 = 30_000;
/// Additional time beyond the configured object-lock acquisition deadline.
pub const HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS: u64 = 5_000;
/// Timeout for cluster health readiness collectors (milliseconds).
/// This bounds expensive storage and lock quorum checks used by cluster probes.
pub const ENV_HEALTH_CLUSTER_TIMEOUT_MS: &str = "RUSTFS_HEALTH_CLUSTER_TIMEOUT_MS";