diff --git a/crates/config/src/constants/tls.rs b/crates/config/src/constants/tls.rs index f6d012e2f..f775f9933 100644 --- a/crates/config/src/constants/tls.rs +++ b/crates/config/src/constants/tls.rs @@ -124,15 +124,48 @@ pub const ENV_H2_KEEP_ALIVE_TIMEOUT: &str = "RUSTFS_H2_KEEP_ALIVE_TIMEOUT"; pub const DEFAULT_H2_KEEP_ALIVE_TIMEOUT: u64 = 10; /// Environment variable for HTTP/1.1 header read timeout (seconds) -/// Default: 5 +/// Default: 75 +/// +/// In hyper's HTTP/1.1 stack this timeout is armed as soon as the connection is +/// ready to read the *next* request head, so on a keep-alive connection it also +/// bounds how long an idle upstream connection is allowed to sit before RustFS +/// closes it. Reverse proxies (Caddy, Nginx) pool upstream connections and keep +/// them idle far longer than a few seconds (Caddy ≈ 2 min, Nginx = 60 s by +/// default). A short value here makes RustFS FIN pooled connections while the +/// proxy still believes they are alive; the proxy then reuses a dead socket and +/// the client sees `socket hang up` / connection reset — most visibly on +/// non-idempotent `PUT` uploads, which proxies will not transparently retry +/// (see issue #3076). +/// +/// The default is therefore set above common proxy idle-keepalive windows. +/// Operators fronting RustFS with a proxy should keep this larger than the +/// proxy's upstream idle-keepalive, or lower the proxy's keepalive below this +/// value. Environments that expose RustFS directly to untrusted slow clients and +/// want tighter slowloris protection can lower it via the env var below. pub const ENV_HTTP1_HEADER_READ_TIMEOUT: &str = "RUSTFS_HTTP1_HEADER_READ_TIMEOUT"; -pub const DEFAULT_HTTP1_HEADER_READ_TIMEOUT: u64 = 5; +pub const DEFAULT_HTTP1_HEADER_READ_TIMEOUT: u64 = 75; /// Environment variable for HTTP/1.1 max buffer size (bytes) /// Default: 65536 (64 KB) pub const ENV_HTTP1_MAX_BUF_SIZE: &str = "RUSTFS_HTTP1_MAX_BUF_SIZE"; pub const DEFAULT_HTTP1_MAX_BUF_SIZE: usize = 64 * 1024; // 64 KB +/// Environment variable for the S3 request-body inter-chunk read timeout +/// (seconds). Default: 300. Set to 0 to disable. +/// +/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk +/// while more bytes are still expected. It resets on every chunk, so it does not +/// penalize slow-but-progressing uploads — it only fires when a peer sends a +/// partial body and then goes silent *without* closing the connection (no EOF). +/// A reverse proxy or CDN that forwards a truncated body this way would +/// otherwise make RustFS wait forever for bytes that never arrive; with this +/// timeout the stalled read is aborted and logged with the received/expected +/// byte counts, turning a silent hang into an actionable diagnostic (issue +/// #3076). A short-body that arrives with a proper EOF is already rejected +/// promptly and is unaffected by this setting. +pub const ENV_HTTP_REQUEST_BODY_READ_TIMEOUT: &str = "RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT"; +pub const DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT: u64 = 300; + // ── TLS Hot Reload Parameters ── /// Environment variable to enable TLS certificate hot reload diff --git a/docs/operations/reverse-proxy.md b/docs/operations/reverse-proxy.md new file mode 100644 index 000000000..93bbfe746 --- /dev/null +++ b/docs/operations/reverse-proxy.md @@ -0,0 +1,173 @@ +# Running RustFS behind a reverse proxy + +RustFS speaks plain S3 over HTTP/1.1 and HTTP/2 and works behind reverse +proxies (Caddy, Nginx, HAProxy) and CDNs (Cloudflare). Most proxy problems are +**not** RustFS storage bugs — the same request sent directly to `:9000` +succeeds, while the proxied request fails. This page documents the request +semantics RustFS expects from the proxy layer and gives known-good +configurations. + +> Rule of thumb: if a request works against `http://:9000` directly but +> fails through the proxy, the fault is in the proxy/CDN request forwarding, not +> in RustFS object handling. Use the checklist below to find which forwarding +> behavior broke. + +## What RustFS requires from the proxy + +S3 clients sign requests with AWS SigV4. RustFS (via `s3s`) re-derives the +signature from the forwarded request, and streams the request body to storage. +For this to succeed the proxy must forward the request **byte-for-byte** with +respect to the signed material and the body: + +1. **Do not alter the body.** No transparent compression, no re-encoding, no + truncation. If the client sent `Content-Length: N`, exactly `N` body bytes + must reach RustFS. If fewer bytes arrive, RustFS waits for the rest per the + HTTP spec and the request appears to hang until the client aborts. +2. **Do not rewrite signed headers.** `Host` and any `x-amz-*` / signed headers + must reach RustFS unchanged. Rewriting `Host` is fine only if the client + signed with that same host. +3. **Preserve `Content-Length`; avoid re-chunking large bodies.** Some CDNs + drop `Content-Length` and switch to `Transfer-Encoding: chunked`, or buffer + the whole request body before forwarding — both change the timing and + framing RustFS sees. +4. **Keep upstream idle keep-alive shorter than RustFS's, or vice-versa** (see + next section) so the proxy never reuses a connection RustFS has already + closed. +5. **Do not strip `ETag`** from responses (breaks multipart completion). + +## Idle keep-alive: the #1 cause of `socket hang up` on writes + +RustFS closes **idle** upstream HTTP/1.1 keep-alive connections after +`RUSTFS_HTTP1_HEADER_READ_TIMEOUT` seconds (default **75s**; see +`crates/config/src/constants/tls.rs`). Reverse proxies keep a pool of upstream +connections and reuse them. If the proxy's upstream idle-keepalive window is +**longer** than RustFS's timeout, the proxy can pick a connection that RustFS +has already FIN'd, write a request onto the dead socket, and the client sees: + +``` +TimeoutError: socket hang up # ECONNRESET +AbortError: Request aborted +``` + +This is most visible on large `PutObject` uploads because: + +- `PUT` is non-idempotent, so proxies will **not** transparently retry it; and +- a larger body keeps the connection in use longer, widening the race window, + so small uploads on the same path often succeed. + +### Fix — make the two windows agree + +Pick **either** side; doing both is safest: + +- **RustFS side:** keep `RUSTFS_HTTP1_HEADER_READ_TIMEOUT` (default 75s) *above* + the proxy's upstream idle-keepalive. To harden slowloris protection on a + directly-exposed node instead, lower it — but then also lower the proxy + keepalive below it. +- **Proxy side:** lower the proxy's upstream idle-keepalive below RustFS's + timeout, or disable upstream keep-alive entirely. + +## Known-good Caddy configuration + +```caddy +your-domain.example.com { + reverse_proxy http://127.0.0.1:9000 { + transport http { + # Talk HTTP/1.1 to RustFS. + versions 1.1 + + # Keep the proxy's upstream idle-keepalive BELOW RustFS's + # RUSTFS_HTTP1_HEADER_READ_TIMEOUT (default 75s) so Caddy never + # reuses a connection RustFS already closed. Set to 0 to disable + # upstream keep-alive entirely (simplest, slightly less efficient). + keepalive 30s + keepalive_idle_conns_per_host 0 + + # Never let the proxy compress/transform the request body. + compression off + + # Generous timeouts for multi-MB single-request PUTs. + dial_timeout 30s + read_timeout 300s + write_timeout 300s + } + + # Forward the body untouched; do not negotiate compression upstream. + header_up Accept-Encoding identity + + # Preserve the host the client signed with. + header_up Host {upstream_hostport} + + # Stream immediately instead of buffering. + flush_interval -1 + } +} +``` + +## Nginx equivalent (essentials) + +```nginx +location / { + proxy_pass http://127.0.0.1:9000; + proxy_http_version 1.1; + + # Nginx default upstream keepalive is 60s; keep it under RustFS's 75s. + # (set `keepalive` in the matching `upstream {}` block) + proxy_set_header Connection ""; + + proxy_set_header Host $host; + proxy_set_header Accept-Encoding "identity"; + + # Do not buffer/limit large uploads. + proxy_request_buffering off; + client_max_body_size 0; + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} +``` + +## Cloudflare (orange-cloud) caveats + +Cloudflare's proxy (orange cloud) may **buffer the entire request body** before +forwarding, and can rewrite requests to `Transfer-Encoding: chunked`, dropping +the client's `Content-Length`. Symptoms match this pattern exactly: tiny uploads +succeed, larger uploads fail with `socket hang up`. + +- For large object writes, prefer **DNS-only (grey cloud)** for the S3 endpoint, + or a Cloudflare plan/tunnel configuration that does not buffer/re-chunk the + request body. +- Force `Accept-Encoding: identity` so nothing in the path negotiates + compression (see issues #609, #1492). +- Ensure `Content-Length` reaches RustFS; disable chunked re-encoding in tunnel + settings (see issue #934). + +## Diagnosis checklist + +Run each step and note where behavior diverges: + +1. **Bypass the proxy.** Send the failing request to `http://:9000` + directly. Success here confirms the fault is in the proxy/CDN path. +2. **Bypass the CDN, keep the proxy.** Point the proxy straight at the origin + (Cloudflare grey cloud / direct DNS). If it now works, the CDN was + buffering/re-chunking the body. +3. **Check idle reuse.** If failures are intermittent and correlate with upload + size, it is almost always the keep-alive mismatch above. Lower the proxy + keepalive (or disable it) and retry. + - If instead the upload **hangs indefinitely** (rather than resetting), the + proxy is likely forwarding a *partial* body and then going silent without + closing the connection. RustFS bounds this wait with + `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` (default 300s; `0` disables) and, on + timeout, logs a `put_object_body_read_stalled` event with the + received/expected byte counts — grep the server log for it to confirm a + truncated-body forwarding problem. +4. **Compare bytes.** Confirm the proxy forwards exactly `Content-Length` body + bytes with no compression/transformation. +5. **Confirm signed headers survive.** `Host` and `x-amz-*` headers must reach + RustFS unchanged; a `SignatureDoesNotMatch` (rather than a hang) points here. + +## Related issues + +- #3076 — Large single-request PutObject fails behind Caddy (this document) +- #609 — Bucket inaccessible via Cloudflare proxied DNS (`Accept-Encoding`) +- #1492 — SigV4 `SignatureDoesNotMatch` on Cloudflare tunnel (`Accept-Encoding`) +- #934 — Console fails behind Cloudflare tunnels (chunked / `Content-Length`) +- #1766 — Large multipart upload fails through Nginx (`ETag` stripping) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index c8588686a..267e58be6 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -109,7 +109,7 @@ use crate::error::ApiError; use crate::server::convert_ecstore_object_info; use crate::table_catalog; use bytes::Bytes; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use http::{HeaderMap, HeaderValue, StatusCode}; use md5::Context as Md5Context; use metrics::{counter, histogram}; @@ -145,6 +145,7 @@ use rustfs_utils::http::{ }; use rustfs_utils::path::{encode_dir_object, is_dir_object, path_join_buf}; use rustfs_zip::{ArchiveLimits, CompressionFormat}; +use s3s::StdError; use s3s::dto::{ CacheControl, Checksum, ChecksumAlgorithm, ChecksumType, ContentDisposition, ContentEncoding, ContentLanguage, ContentType, CopyObjectInput, CopyObjectOutput, CopyObjectResult, CopySource, DeleteObjectInput, DeleteObjectOutput, DeleteObjectsInput, @@ -156,7 +157,7 @@ use s3s::dto::{ StreamingBlob, TaggingHeader, Timestamp, TimestampFormat, WebsiteRedirectLocation, }; use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; -use s3s::stream::{ByteStream, RemainingLength}; +use s3s::stream::{ByteStream, DynByteStream, RemainingLength}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; const DEFAULT_PUT_LARGE_CONCURRENCY_TUNING_MIN_SIZE_BYTES: i64 = 32 * 1024 * 1024; @@ -214,6 +215,7 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object"; const EVENT_PUT_OBJECT_STORE_INFLIGHT_SLOW: &str = "put_object_store_inflight_slow"; const EVENT_PUT_OBJECT_STORE_RETURNED: &str = "put_object_store_returned"; const EVENT_GET_OBJECT_STREAM_BODY: &str = "get_object_stream_body"; +const EVENT_PUT_OBJECT_BODY_READ_STALLED: &str = "put_object_body_read_stalled"; const GET_OBJECT_STAGE_PATH_S3_HANDLER: &str = "s3_handler"; const GET_OBJECT_STAGE_REQUEST_INGRESS_TO_CONTEXT: &str = "request_ingress_to_context"; const GET_OBJECT_STAGE_OUTPUT_STRATEGY: &str = "output_strategy"; @@ -977,6 +979,154 @@ impl Drop for GetObjectStreamingReader { } } +/// Resolve the S3 request-body inter-chunk read timeout from the environment. +/// +/// Returns `Duration::ZERO` when disabled (`RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT=0`), +/// in which case [`guard_put_object_body_read_timeout`] passes the body through +/// untouched. +fn put_object_body_read_timeout() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT, + rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT, + )) +} + +/// A [`ByteStream`] decorator that aborts a request body whose peer stops +/// sending bytes without closing the connection. +/// +/// A well-behaved short body ends with EOF and is rejected promptly by the +/// eager/streaming readers. The failure this guards against is different: a +/// reverse proxy or CDN forwards a *partial* body and then goes silent while +/// holding the connection open, so the inner stream neither yields more bytes +/// nor reports EOF. Without a bound, RustFS would wait forever for bytes that +/// never arrive and the client eventually sees a hang/abort with no server-side +/// explanation (issue #3076). +/// +/// The timer resets on every chunk, so slow-but-progressing uploads are not +/// penalized; it only fires after `timeout` of complete silence. On timeout the +/// stall is logged with the received/expected byte counts and the read fails +/// with an `ErrorKind::TimedOut` error instead of hanging. +/// +/// `remaining_length` and `size_hint` are forwarded from the inner stream so +/// wrapping is transparent to length/content handling downstream. +struct RequestBodyReadTimeout { + inner: DynByteStream, + timeout: Duration, + timer: Option>>, + received: u64, + expected: Option, + bucket: String, + key: String, + request_id: String, + timed_out: bool, +} + +impl Stream for RequestBodyReadTimeout { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + // Once we have surfaced a stall error, treat the stream as terminated so + // we never poll the abandoned inner stream again. + if this.timed_out { + return Poll::Ready(None); + } + + match Pin::new(&mut this.inner).poll_next(cx) { + Poll::Ready(Some(Ok(chunk))) => { + this.timer = None; + this.received = this.received.saturating_add(chunk.len() as u64); + Poll::Ready(Some(Ok(chunk))) + } + Poll::Ready(other) => { + this.timer = None; + Poll::Ready(other) + } + Poll::Pending => { + if this.timeout.is_zero() { + return Poll::Pending; + } + + if this.timer.is_none() { + this.timer = Some(Box::pin(tokio::time::sleep(this.timeout))); + } + + if let Some(timer) = this.timer.as_mut() + && std::future::Future::poll(timer.as_mut(), cx).is_ready() + { + this.timer = None; + this.timed_out = true; + let expected_display = this.expected.map(|v| v.to_string()).unwrap_or_else(|| "unknown".to_string()); + warn!( + target: "rustfs::app::object_usecase", + event = EVENT_PUT_OBJECT_BODY_READ_STALLED, + component = LOG_COMPONENT_APP, + subsystem = LOG_SUBSYSTEM_OBJECT, + request_id = %this.request_id, + bucket = %this.bucket, + key = %this.key, + received_bytes = this.received, + expected_bytes = %expected_display, + timeout_secs = this.timeout.as_secs(), + state = "stall_timeout", + "PutObject request body read stalled; aborting. A proxy/CDN likely forwarded a partial body without closing the connection." + ); + return Poll::Ready(Some(Err(Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!( + "request body read stalled: received {} of {} bytes, no data for {}s", + this.received, + expected_display, + this.timeout.as_secs() + ), + )) as StdError))); + } + + Poll::Pending + } + } + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl ByteStream for RequestBodyReadTimeout { + fn remaining_length(&self) -> RemainingLength { + self.inner.remaining_length() + } +} + +/// Wrap an incoming request body with [`RequestBodyReadTimeout`] unless the +/// feature is disabled (`timeout == 0`), in which case the body is returned +/// untouched. `remaining_length` is preserved via [`StreamingBlob::new`]. +fn guard_put_object_body_read_timeout( + body: StreamingBlob, + bucket: &str, + key: &str, + request_id: &str, + expected: Option, + timeout: Duration, +) -> StreamingBlob { + if timeout.is_zero() { + return body; + } + + StreamingBlob::new(RequestBodyReadTimeout { + inner: body.into(), + timeout, + timer: None, + received: 0, + expected: expected.and_then(|v| u64::try_from(v).ok()), + bucket: bucket.to_string(), + key: key.to_string(), + request_id: request_id.to_string(), + timed_out: false, + }) +} + impl ExtractArchiveEtagReader { fn new(inner: R, etag: Arc>>) -> Self { Self { @@ -3157,6 +3307,18 @@ impl DefaultObjectUsecase { let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; + // Guard against a proxy/CDN that forwards a partial body then goes silent + // without closing the connection: bound the inter-chunk wait so the read + // fails (with a diagnostic log) instead of hanging forever (issue #3076). + let body = { + let request_id = req + .extensions + .get::() + .map(|ctx| ctx.request_id.clone()) + .unwrap_or_default(); + guard_put_object_body_read_timeout(body, &bucket, &key, &request_id, content_length, put_object_body_read_timeout()) + }; + let decoded_content_length = decoded_content_length_from_headers(&req.headers)?; let mut size = match (request_uses_aws_chunked(&req.headers), decoded_content_length, content_length) { (true, Some(decoded), _) => decoded, @@ -6757,6 +6919,64 @@ mod tests { assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); } + #[tokio::test] + async fn put_object_body_read_timeout_guard_aborts_on_stall() { + // Inner stream never yields and never reports EOF (a proxy that forwarded + // a partial body then went silent while holding the connection open). + let inner = StreamingBlob::wrap(futures::stream::pending::>()); + let mut guarded = guard_put_object_body_read_timeout( + inner, + "test-bucket", + "stalled-object", + "req-1", + Some(1024), + Duration::from_millis(1), + ); + + let err = guarded + .next() + .await + .expect("guard should yield a stall error") + .expect_err("stalled body should return an error"); + let io_err = err + .downcast_ref::() + .expect("stall error should wrap an io::Error"); + assert_eq!(io_err.kind(), std::io::ErrorKind::TimedOut); + + // After a stall the guard terminates the stream instead of re-polling the + // abandoned inner stream. + assert!(guarded.next().await.is_none()); + } + + #[tokio::test] + async fn put_object_body_read_timeout_guard_preserves_length_and_passes_through() { + let body = StreamingBlob::from(s3s::Body::from(Bytes::from_static(b"hello world"))); + assert_eq!(body.remaining_length().exact(), Some(11)); + + let mut guarded = + guard_put_object_body_read_timeout(body, "test-bucket", "ok-object", "req-2", Some(11), Duration::from_secs(60)); + // remaining_length must be forwarded, not reset to unknown. + assert_eq!(guarded.remaining_length().exact(), Some(11)); + + let mut collected = Vec::new(); + while let Some(chunk) = guarded.next().await { + collected.extend_from_slice(&chunk.expect("chunk should read")); + } + assert_eq!(collected, b"hello world"); + } + + #[tokio::test] + async fn put_object_body_read_timeout_guard_disabled_passthrough() { + let body = StreamingBlob::from(s3s::Body::from(Bytes::from_static(b"data"))); + let mut guarded = guard_put_object_body_read_timeout(body, "test-bucket", "ok-object", "req-3", Some(4), Duration::ZERO); + + let mut collected = Vec::new(); + while let Some(chunk) = guarded.next().await { + collected.extend_from_slice(&chunk.expect("chunk should read")); + } + assert_eq!(collected, b"data"); + } + #[tokio::test] #[serial_test::serial] async fn get_object_streaming_reader_holds_request_guard_until_eof() {