fix(ecstore): send valid ping body in remote locker (#3112)

* fix(ecstore): send valid ping body in remote locker

Build ping requests with a flatbuffer payload so health checks remain compatible with the ping response parser after restart.

* fix(bench): use multi-host warp target during failover

Normalize comma-separated warp host lists in run_object_batch_bench and let four-node failover bench pass BENCH_WARP_HOSTS so rolling restart does not pin load to a single restarting node.

* feat(health): add compat health probes with busy/KMS checks

  - Add /health/live liveness probe endpoint
  - Add busy protection (429) for readiness probes, gated by RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE
  - Add KMS readiness check for /health/ready, gated by RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE
  - Add lock quorum status caching with TTL to reduce RPC pressure
  - Consolidate health response building into build_health_response_parts
  - Register /health/live in console router and readiness gate
  - Remove MinIO references from newly added health code

* fix(health): decouple kms readiness from lock quorum

crates/config/README.md

@@ -71,6 +71,24 @@ Current guidance:
 - `RUSTFS_SCANNER_IDLE_MODE` (canonical)
 - `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` (canonical)
 
+## Health compatibility switches
+
+- `RUSTFS_HEALTH_ENDPOINT_ENABLE`
+  - controls canonical `/health`, `/health/live`, and `/health/ready` endpoint exposure.
+- `RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE`
+  - enables minimal payload mode for GET health responses (`status`, `ready` only).
+- `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS`
+  - TTL for readiness cache evaluation.
+- `RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE`
+  - enables busy protection behavior for health probes.
+  - default is `false`.
+- `RUSTFS_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS`
+  - max active HTTP requests; health probes return `429` when active requests reach or exceed this value.
+  - `0` disables thresholding even if busy protection is enabled.
+- `RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE`
+  - enables KMS readiness enforcement for `/health/ready`.
+  - default is `false`.
+
 ## Drive timeout environment variables
 
 - `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`

crates/config/src/constants/health.rs

@@ -26,3 +26,20 @@ pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
 /// When enabled, only `status` and `ready` fields are returned.
 pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
 pub const DEFAULT_HEALTH_MINIMAL_RESPONSE_ENABLE: bool = false;
+
+/// Enable busy protection for health probes.
+/// When enabled with a positive request threshold, alias health probes may
+/// return 429 when active HTTP requests exceed the threshold.
+pub const ENV_HEALTH_COMPAT_BUSY_CHECK_ENABLE: &str = "RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE";
+pub const DEFAULT_HEALTH_COMPAT_BUSY_CHECK_ENABLE: bool = false;
+
+/// Max active HTTP requests; alias health probes report busy (429) when active requests reach or exceed this value.
+/// Set to 0 to disable thresholding even when busy protection is enabled.
+pub const ENV_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS: &str = "RUSTFS_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS";
+pub const DEFAULT_HEALTH_COMPAT_BUSY_MAX_ACTIVE_REQUESTS: usize = 0;
+
+/// Enable KMS readiness check for alias readiness probes.
+/// When enabled, `/health/ready` additionally requires KMS service to be
+/// in running state if a global KMS manager exists.
+pub const ENV_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE: &str = "RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE";
+pub const DEFAULT_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE: bool = false;

crates/e2e_test/src/kms/README.md

@@ -191,7 +191,7 @@ RUST_LOG=debug cargo test test_local_kms_end_to_end -- --nocapture
 4. **Monitor ports**
    ```bash
    netstat -an | grep 9050
-   curl http://127.0.0.1:9050/minio/health/ready
+   curl http://127.0.0.1:9050/health/ready
    ```
 
 ## 📊 Coverage

crates/ecstore/src/rpc/remote_disk.rs

@@ -450,7 +450,7 @@ impl RemoteDisk {
 
     async fn mark_faulty_and_evict(&self, reason: &'static str) {
         let previous_state = self.runtime_state();
-        let became_offline = self.mark_suspect_or_offline(reason);
+        let transitioned_to_offline = self.mark_suspect_or_offline(reason);
         let state = self.runtime_state();
 
         if state != previous_state {
@@ -461,7 +461,7 @@ impl RemoteDisk {
                 "reason" => reason.to_string()
             )
             .increment(1);
-            if became_offline || state == RuntimeDriveHealthState::Offline {
+            if transitioned_to_offline {
                 warn!(
                     "Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
                     self.endpoint, self.addr, reason

crates/ecstore/src/rpc/remote_locker.rs

@@ -14,12 +14,15 @@
 
 use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
 use async_trait::async_trait;
+use bytes::Bytes;
 use rustfs_lock::{
     LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
 };
 use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
-use rustfs_protos::{evict_failed_connection, proto_gen::node_service::node_service_client::NodeServiceClient};
+use rustfs_protos::{
+    evict_failed_connection, models::PingBodyBuilder, proto_gen::node_service::node_service_client::NodeServiceClient,
+};
 use std::time::Duration;
 use tokio::time::timeout;
 use tonic::Request;
@@ -42,6 +45,20 @@ impl RemoteClient {
         Self { addr: url.to_string() }
     }
 
+    fn build_ping_request() -> PingRequest {
+        let mut fbb = flatbuffers::FlatBufferBuilder::new();
+        let payload = fbb.create_vector(b"health-check");
+        let mut builder = PingBodyBuilder::new(&mut fbb);
+        builder.add_payload(payload);
+        let root = builder.finish();
+        fbb.finish(root, None);
+
+        PingRequest {
+            version: 1,
+            body: Bytes::copy_from_slice(fbb.finished_data()),
+        }
+    }
+
     /// Create a minimal LockRequest for unlock operations using only lock_id
     fn create_unlock_request(lock_id: &LockId) -> LockRequest {
         LockRequest {
@@ -510,10 +527,7 @@ impl LockClient for RemoteClient {
             }
         };
 
-        let ping_req = Request::new(PingRequest {
-            version: 1,
-            body: bytes::Bytes::new(),
-        });
+        let ping_req = Request::new(Self::build_ping_request());
 
         match client.ping(ping_req).await {
             Ok(_) => {