fix(internode): relax keepalive/RPC timeouts to fix large-object GET EOF (#4284)

fix(internode): relax aggressive HTTP/2 keepalive & RPC timeouts to stop large-object GET truncation

The internode gRPC channel used a 3s HTTP/2 keepalive timeout and a 10s
overall RPC timeout. Under high-concurrency large-object reads a saturated
peer's PING ACK is legitimately delayed past 3s, so the whole channel (and
every RPC/stream on it) is torn down as a 'dead peer'. In-flight peer shard
reads then fail mid-object and large GETs truncate after headers+Content-Length
are already sent, surfacing to clients as 'download error: unexpected EOF'.

Reproduced on a 4-node erasure cluster with pure GET-only warp (no concurrent
writes) at 64 concurrency: 10MiB GET ~31 unexpected-EOF / 3min. Raising the
internode keepalive timeout (3s->30s via env) alone cut that to ~7; also raising
the RPC timeout cut it to ~3.

- Raise DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS 3 -> 20
- Raise DEFAULT_INTERNODE_RPC_TIMEOUT_SECS 10 -> 30
- Wire the data-plane rio HttpReader keepalive timeout (was hardcoded 3s) to the
  same env/default so control- and data-plane stay consistent.

Refs backlog#832.

crates/config/src/constants/internode.rs

@@ -25,12 +25,19 @@ pub const ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: &str = "RUSTFS_INTERNODE_
 pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5;
 
 /// HTTP/2 keepalive timeout for internode gRPC channels.
+///
+/// This is the time a peer has to ACK a keepalive PING before the whole channel
+/// (and every RPC/stream multiplexed on it) is torn down. A very aggressive value
+/// misfires under load: on a saturated node the PING ACK is legitimately delayed,
+/// the channel is wrongly declared dead, in-flight peer reads fail, and large-object
+/// GETs truncate mid-stream (client "unexpected EOF"). See backlog#832. Keep this
+/// generous enough to tolerate transient load while still detecting truly dead peers.
 pub const ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS";
-pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
+pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 20;
 
 /// Overall timeout for a single internode gRPC request.
 pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
-pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
+pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 30;
 
 /// Profile selector for conservative internode HTTP data-plane client tuning.
 pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
@@ -73,8 +80,8 @@ mod tests {
         assert_eq!(DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, 3);
         assert_eq!(DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, 10);
         assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
-        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
-        assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
+        assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 20);
+        assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 30);
         assert_eq!(DEFAULT_INTERNODE_HTTP_TUNING_PROFILE, "legacy");
     }
 

crates/rio/src/http_reader.rs

@@ -410,11 +410,17 @@ async fn build_http_client(
     tuning: InternodeHttpClientTuning,
     outbound_tls: &rustfs_tls_runtime::GlobalPublishedOutboundTlsState,
 ) -> io::Result<Client> {
+    // Keep the data-plane HTTP/2 keepalive timeout consistent with the control-plane
+    // gRPC channel and env-configurable. A too-aggressive value (e.g. 3s) tears down a
+    // busy data connection when a PING ACK is delayed under load, aborting in-flight
+    // shard read streams and truncating large-object GETs mid-stream (backlog#832).
+    let http2_keepalive_timeout_secs = get_env_opt_u64(rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS)
+        .unwrap_or(rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS);
     let mut builder = Client::builder()
         .connect_timeout(std::time::Duration::from_secs(5))
         .tcp_keepalive(std::time::Duration::from_secs(10))
         .http2_keep_alive_interval(std::time::Duration::from_secs(5))
-        .http2_keep_alive_timeout(std::time::Duration::from_secs(3))
+        .http2_keep_alive_timeout(std::time::Duration::from_secs(http2_keepalive_timeout_secs))
         .http2_keep_alive_while_idle(true);
     builder = apply_http_client_tuning(builder, tuning);