diff --git a/Cargo.lock b/Cargo.lock index 2eff66aa..482db25c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1713,6 +1713,7 @@ dependencies = [ "rand 0.9.2", "rmp-serde", "serde", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/Cargo.toml b/Cargo.toml index 5aac1826..c6da24c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,6 +144,7 @@ tokio = { version = "1.49", default-features = false, features = [ ] } tokio-util = { version = "0.7", features = ["compat", "io"] } tokio-stream = { version = "0.1", features = ["net"] } +socket2 = { version = "0.6", features = ["all"] } opentelemetry = { version = "0.17", features = ["rt-tokio", "metrics", "trace"] } opentelemetry-prometheus = "0.10" diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml index 314f748a..01e89db3 100644 --- a/src/net/Cargo.toml +++ b/src/net/Cargo.toml @@ -27,6 +27,7 @@ rmp-serde.workspace = true hex.workspace = true rand.workspace = true +socket2.workspace = true log.workspace = true arc-swap.workspace = true diff --git a/src/net/netapp.rs b/src/net/netapp.rs index 14d0b53b..f712a30c 100644 --- a/src/net/netapp.rs +++ b/src/net/netapp.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::net::{IpAddr, SocketAddr}; use std::sync::{Arc, RwLock}; +use std::time::Duration; use log::{debug, error, info, trace, warn}; @@ -39,6 +40,19 @@ pub(crate) type VersionTag = [u8; 16]; /// Since Garage v1.0, we have replaced the prefix by `grgnet` (shorthand for `garage_net`). pub(crate) const NETAPP_VERSION_TAG: u64 = 0x6772676e65740010; // grgnet 0x0010 (1.0) +/// Time a connection must be idle before the first keepalive probe is sent. +const TCP_KEEPALIVE_TIME: Duration = Duration::from_secs(30); +/// Interval between keepalive probes after the first. +const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(10); + +fn set_keepalive(stream: &TcpStream) -> Result<(), std::io::Error> { + let sock_ref = socket2::SockRef::from(stream); + let keepalive = socket2::TcpKeepalive::new() + .with_time(TCP_KEEPALIVE_TIME) + .with_interval(TCP_KEEPALIVE_INTERVAL); + sock_ref.set_tcp_keepalive(&keepalive) +} + /// `HelloMessage` is sent by the client on a Netapp connection to indicate /// that they are also a server and ready to receive incoming connections /// at the specified address and port. If the client doesn't know their @@ -252,6 +266,13 @@ impl NetApp { _ = must_exit.changed() => continue, }; + if let Err(e) = set_keepalive(&socket) { + warn!( + "Failed to set keepalive on connection from {}: {}", + peer_addr, e + ); + } + info!( "Incoming connection from {}, negotiating handshake...", peer_addr @@ -314,6 +335,9 @@ impl NetApp { } None => TcpStream::connect(ip).await?, }; + if let Err(e) = set_keepalive(&stream) { + warn!("Failed to set keepalive on connection to {}: {}", ip, e); + } info!("Connected to {}, negotiating handshake...", ip); ClientConn::init(self, stream, id).await?; Ok(())