From 9959c828a17a9a6844c6f4d67806ffac185675f9 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 5 Jul 2026 19:27:23 +0800 Subject: [PATCH] fix(internode): relax keepalive/RPC timeouts to fix large-object GET EOF (#4284) fix(internode): relax aggressive HTTP/2 keepalive & RPC timeouts to stop large-object GET truncation The internode gRPC channel used a 3s HTTP/2 keepalive timeout and a 10s overall RPC timeout. Under high-concurrency large-object reads a saturated peer's PING ACK is legitimately delayed past 3s, so the whole channel (and every RPC/stream on it) is torn down as a 'dead peer'. In-flight peer shard reads then fail mid-object and large GETs truncate after headers+Content-Length are already sent, surfacing to clients as 'download error: unexpected EOF'. Reproduced on a 4-node erasure cluster with pure GET-only warp (no concurrent writes) at 64 concurrency: 10MiB GET ~31 unexpected-EOF / 3min. Raising the internode keepalive timeout (3s->30s via env) alone cut that to ~7; also raising the RPC timeout cut it to ~3. - Raise DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS 3 -> 20 - Raise DEFAULT_INTERNODE_RPC_TIMEOUT_SECS 10 -> 30 - Wire the data-plane rio HttpReader keepalive timeout (was hardcoded 3s) to the same env/default so control- and data-plane stay consistent. Refs backlog#832. --- crates/config/src/constants/internode.rs | 15 +++++++++++---- crates/rio/src/http_reader.rs | 8 +++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/config/src/constants/internode.rs b/crates/config/src/constants/internode.rs index 14b570da2..14ca76c79 100644 --- a/crates/config/src/constants/internode.rs +++ b/crates/config/src/constants/internode.rs @@ -25,12 +25,19 @@ pub const ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: &str = "RUSTFS_INTERNODE_ pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5; /// HTTP/2 keepalive timeout for internode gRPC channels. +/// +/// This is the time a peer has to ACK a keepalive PING before the whole channel +/// (and every RPC/stream multiplexed on it) is torn down. A very aggressive value +/// misfires under load: on a saturated node the PING ACK is legitimately delayed, +/// the channel is wrongly declared dead, in-flight peer reads fail, and large-object +/// GETs truncate mid-stream (client "unexpected EOF"). See backlog#832. Keep this +/// generous enough to tolerate transient load while still detecting truly dead peers. pub const ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"; -pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3; +pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 20; /// Overall timeout for a single internode gRPC request. pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS"; -pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10; +pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 30; /// Profile selector for conservative internode HTTP data-plane client tuning. pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE"; @@ -73,8 +80,8 @@ mod tests { assert_eq!(DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, 3); assert_eq!(DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, 10); assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5); - assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3); - assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10); + assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 20); + assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 30); assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy"); } diff --git a/crates/rio/src/http_reader.rs b/crates/rio/src/http_reader.rs index 6c52c48e1..d985737d0 100644 --- a/crates/rio/src/http_reader.rs +++ b/crates/rio/src/http_reader.rs @@ -410,11 +410,17 @@ async fn build_http_client( tuning: InternodeHttpClientTuning, outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState, ) -> io::Result { + // Keep the data-plane HTTP/2 keepalive timeout consistent with the control-plane + // gRPC channel and env-configurable. A too-aggressive value (e.g. 3s) tears down a + // busy data connection when a PING ACK is delayed under load, aborting in-flight + // shard read streams and truncating large-object GETs mid-stream (backlog#832). + let http2_keepalive_timeout_secs = get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS) + .unwrap_or(rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS); let mut builder = Client::builder() .connect_timeout(std::time::Duration::from_secs(5)) .tcp_keepalive(std::time::Duration::from_secs(10)) .http2_keep_alive_interval(std::time::Duration::from_secs(5)) - .http2_keep_alive_timeout(std::time::Duration::from_secs(3)) + .http2_keep_alive_timeout(std::time::Duration::from_secs(http2_keepalive_timeout_secs)) .http2_keep_alive_while_idle(true); builder = apply_http_client_tuning(builder, tuning);