From 69cd230568729f877c4f641811ca8eb1e3e61cca Mon Sep 17 00:00:00 2001 From: rajsinghtech Date: Fri, 20 Feb 2026 21:28:29 +0000 Subject: [PATCH] fix: enable TCP keepalive on RPC connections (#1348) Garage RPC connections have no TCP keepalive enabled. When a connection dies silently (proxy pod restart, NAT timeout, network partition), it's only detected by application-level pings after ~60s (4 failed pings x 15s interval). During this window, the node appears connected but all RPC calls to it fail. Enable TCP keepalive on both outgoing and incoming RPC connections via socket2: - Idle time before first probe: 30s (TCP_KEEPALIVE_TIME) - Probe interval after first: 10s (TCP_KEEPALIVE_INTERVAL) A helper set_keepalive() function avoids duplicating the socket2 setup. Incoming connection keepalive failures are logged as warnings but don't reject the connection. Companion to #1345 (stale address pruning + connect timeout). Together they address both halves of the reconnection problem: faster detection (this PR) and faster recovery. Co-authored-by: Raj Singh Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1348 Reviewed-by: maximilien Co-authored-by: rajsinghtech Co-committed-by: rajsinghtech --- Cargo.lock | 1 + Cargo.toml | 1 + src/net/Cargo.toml | 1 + src/net/netapp.rs | 24 ++++++++++++++++++++++++ 4 files changed, 27 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 2eff66aa..482db25c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1713,6 +1713,7 @@ dependencies = [ "rand 0.9.2", "rmp-serde", "serde", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/Cargo.toml b/Cargo.toml index 5aac1826..c6da24c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,6 +144,7 @@ tokio = { version = "1.49", default-features = false, features = [ ] } tokio-util = { version = "0.7", features = ["compat", "io"] } tokio-stream = { version = "0.1", features = ["net"] } +socket2 = { version = "0.6", features = ["all"] } opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics", "trace"] } opentelemetry-prometheus = "0.10" diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml index 314f748a..01e89db3 100644 --- a/src/net/Cargo.toml +++ b/src/net/Cargo.toml @@ -27,6 +27,7 @@ rmp-serde.workspace = true hex.workspace = true rand.workspace = true +socket2.workspace = true log.workspace = true arc-swap.workspace = true diff --git a/src/net/netapp.rs b/src/net/netapp.rs index 14d0b53b..f712a30c 100644 --- a/src/net/netapp.rs +++ b/src/net/netapp.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::net::{IpAddr, SocketAddr}; use std::sync::{Arc, RwLock}; +use std::time::Duration; use log::{debug, error, info, trace, warn}; @@ -39,6 +40,19 @@ pub(crate) type VersionTag = [u8; 16]; /// Since Garage v1.0, we have replaced the prefix by `grgnet` (shorthand for `garage_net`). pub(crate) const NETAPP_VERSION_TAG: u64 = 0x6772676e65740010; // grgnet 0x0010 (1.0) +/// Time a connection must be idle before the first keepalive probe is sent. +const TCP_KEEPALIVE_TIME: Duration = Duration::from_secs(30); +/// Interval between keepalive probes after the first. +const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); + +fn set_keepalive(stream: &TcpStream) -> Result<(), std::io::Error> { + let sock_ref = socket2::SockRef::from(stream); + let keepalive = socket2::TcpKeepalive::new() + .with_time(TCP_KEEPALIVE_TIME) + .with_interval(TCP_KEEPALIVE_INTERVAL); + sock_ref.set_tcp_keepalive(&keepalive) +} + /// `HelloMessage` is sent by the client on a Netapp connection to indicate /// that they are also a server and ready to receive incoming connections /// at the specified address and port. If the client doesn't know their @@ -252,6 +266,13 @@ impl NetApp { _ = must_exit.changed() => continue, }; + if let Err(e) = set_keepalive(&socket) { + warn!( + "Failed to set keepalive on connection from {}: {}", + peer_addr, e + ); + } + info!( "Incoming connection from {}, negotiating handshake...", peer_addr @@ -314,6 +335,9 @@ impl NetApp { } None => TcpStream::connect(ip).await?, }; + if let Err(e) = set_keepalive(&stream) { + warn!("Failed to set keepalive on connection to {}: {}", ip, e); + } info!("Connected to {}, negotiating handshake...", ip); ClientConn::init(self, stream, id).await?; Ok(())