fix(net): resolve 1GB upload hang and macos build (Issue #1001 regression) (#1035)

crates/rio/src/http_reader.rs

@@ -32,7 +32,16 @@ use crate::{EtagResolvable, HashReaderDetector, HashReaderMut};
 fn get_http_client() -> Client {
     // Reuse the HTTP connection pool in the global `reqwest::Client` instance
     // TODO: interact with load balancing?
-    static CLIENT: LazyLock<Client> = LazyLock::new(Client::new);
+    static CLIENT: LazyLock<Client> = LazyLock::new(|| {
+        Client::builder()
+            .connect_timeout(std::time::Duration::from_secs(5))
+            .tcp_keepalive(std::time::Duration::from_secs(10))
+            .http2_keep_alive_interval(std::time::Duration::from_secs(5))
+            .http2_keep_alive_timeout(std::time::Duration::from_secs(3))
+            .http2_keep_alive_while_idle(true)
+            .build()
+            .expect("Failed to create global HTTP client")
+    });
     CLIENT.clone()
 }
 

docs/cluster_recovery.md

@@ -25,6 +25,21 @@ To resolve this, we needed to transform the passive failure detection (waiting f
 ## 3. Implemented Solution
 We modified the internal gRPC client configuration in `crates/protos/src/lib.rs` to implement a multi-layered health check strategy.
 
+### Solution Overview
+The fix implements a multi-layered detection strategy covering both Control Plane (RPC) and Data Plane (Streaming):
+
+1.  **Control Plane (gRPC)**:
+    *   Enabled `http2_keep_alive_interval` (5s) and `keep_alive_timeout` (3s) in `tonic` clients.
+    *   Enforced `tcp_keepalive` (10s) on underlying transport.
+    *   Context: Ensures cluster metadata operations (raft, status checks) fail fast if a node dies.
+
+2.  **Data Plane (File Uploads/Downloads)**:
+    *   **Client (Rio)**: Updated `reqwest` client builder in `crates/rio` to enable TCP Keepalive (10s) and HTTP/2 Keepalive (5s). This prevents hangs during large file streaming (e.g., 1GB uploads).
+    *   **Server**: Enabled `SO_KEEPALIVE` on all incoming TCP connections in `rustfs/src/server/http.rs` to forcefully close sockets from dead clients.
+
+3.  **Cross-Platform Build Stability**:
+    *   Guarded Linux-specific profiling code (`jemalloc_pprof`) with `#[cfg(target_os = "linux")]` to fix build failures on macOS/AArch64.
+
 ### Configuration Changes
 
 ```rust

rustfs/src/profiling.rs

@@ -12,6 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#[cfg(not(target_os = "linux"))]
+pub async fn init_from_env() {}
+
+#[cfg(not(target_os = "linux"))]
+pub async fn dump_cpu_pprof_for(_duration: std::time::Duration) -> Result<std::path::PathBuf, String> {
+    Err("CPU profiling is only supported on Linux".to_string())
+}
+
+#[cfg(not(target_os = "linux"))]
+pub async fn dump_memory_pprof_now() -> Result<std::path::PathBuf, String> {
+    Err("Memory profiling is only supported on Linux".to_string())
+}
+
+#[cfg(target_os = "linux")]
+mod linux_impl {
     use chrono::Utc;
     use jemalloc_pprof::PROF_CTL;
     use pprof::protos::Message;
@@ -281,3 +296,7 @@ pub async fn init_from_env() {
             start_memory_periodic(mem_interval).await;
         }
     }
+}
+
+#[cfg(target_os = "linux")]
+pub use linux_impl::{dump_cpu_pprof_for, dump_memory_pprof_now, init_from_env};

rustfs/src/server/http.rs

@@ -33,7 +33,7 @@ use rustfs_protos::proto_gen::node_service::node_service_server::NodeServiceServ
 use rustfs_utils::net::parse_and_resolve_address;
 use rustls::ServerConfig;
 use s3s::{host::MultiDomain, service::S3Service, service::S3ServiceBuilder};
-use socket2::SockRef;
+use socket2::{SockRef, TcpKeepalive};
 use std::io::{Error, Result};
 use std::net::SocketAddr;
 use std::sync::Arc;
@@ -371,6 +371,20 @@ pub async fn start_http_server(
             };
 
             let socket_ref = SockRef::from(&socket);
+
+            // Enable TCP Keepalive to detect dead clients (e.g. power loss)
+            // Idle: 10s, Interval: 5s, Retries: 3
+            let ka = TcpKeepalive::new()
+                .with_time(Duration::from_secs(10))
+                .with_interval(Duration::from_secs(5));
+
+            #[cfg(not(any(target_os = "openbsd", target_os = "netbsd")))]
+            let ka = ka.with_retries(3);
+
+            if let Err(err) = socket_ref.set_tcp_keepalive(&ka) {
+                warn!(?err, "Failed to set TCP_KEEPALIVE");
+            }
+
             if let Err(err) = socket_ref.set_tcp_nodelay(true) {
                 warn!(?err, "Failed to set TCP_NODELAY");
             }