mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
fix(server): stop closing idle proxy keep-alive connections + reverse-proxy hardening (#3076) (#4360)
* fix(server): raise default HTTP/1.1 idle keep-alive timeout for proxies In hyper's HTTP/1.1 stack `header_read_timeout` is armed as soon as the connection is ready to read the next request head, so on a keep-alive connection it also bounds how long an idle upstream connection may sit before RustFS closes it. The previous 5s default meant RustFS FIN'd pooled upstream connections while reverse proxies (Caddy ~2min, Nginx 60s) still believed they were alive. The proxy then reused a dead socket and clients saw `socket hang up` / connection reset, most visibly on non-idempotent `PUT` uploads that proxies will not transparently retry (issue #3076). Raise DEFAULT_HTTP1_HEADER_READ_TIMEOUT to 75s (above common proxy idle-keepalive windows) and document the coupling. Still overridable via RUSTFS_HTTP1_HEADER_READ_TIMEOUT for deployments that expose RustFS directly to untrusted slow clients and want tighter slowloris protection. Co-Authored-By: heihutu <heihutu@gmail.com> * docs(operations): add reverse-proxy deployment guide Document the request/body semantics RustFS expects from a proxy layer, the idle keep-alive mismatch that causes `socket hang up` on large PutObject writes, and known-good Caddy/Nginx configs plus Cloudflare caveats. Closes the documentation gap called out in issue #3076 and consolidates the scattered findings from #609/#934/#1492/#1766. Co-Authored-By: heihutu <heihutu@gmail.com> * feat(object): bound stalled PutObject request-body reads with diagnostics When a reverse proxy or CDN forwards a partial request body and then goes silent without closing the connection, the inner body stream neither yields more bytes nor reports EOF, so RustFS waited forever for bytes that never arrive and the client saw a silent hang/abort (issue #3076). A short body that ends with a proper EOF was already rejected promptly; the gap was the no-EOF stall case. Add a single-point request-body guard on the PutObject path that wraps the incoming StreamingBlob with an inter-chunk read timeout. The timer resets on every chunk, so slow-but-progressing uploads are unaffected; it only fires after RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT (default 300s, 0 disables) of complete silence. On timeout it logs a structured `put_object_body_read_stalled` event with received/expected byte counts and fails the read with ErrorKind::TimedOut instead of hanging. remaining_length/size_hint are forwarded so wrapping is transparent to downstream length handling. Covered by unit tests for the stall path, length-preserving pass-through, and the disabled (timeout=0) pass-through. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
@@ -124,15 +124,48 @@ pub const ENV_H2_KEEP_ALIVE_TIMEOUT: &str = "RUSTFS_H2_KEEP_ALIVE_TIMEOUT";
|
||||
pub const DEFAULT_H2_KEEP_ALIVE_TIMEOUT: u64 = 10;
|
||||
|
||||
/// Environment variable for HTTP/1.1 header read timeout (seconds)
|
||||
/// Default: 5
|
||||
/// Default: 75
|
||||
///
|
||||
/// In hyper's HTTP/1.1 stack this timeout is armed as soon as the connection is
|
||||
/// ready to read the *next* request head, so on a keep-alive connection it also
|
||||
/// bounds how long an idle upstream connection is allowed to sit before RustFS
|
||||
/// closes it. Reverse proxies (Caddy, Nginx) pool upstream connections and keep
|
||||
/// them idle far longer than a few seconds (Caddy ≈ 2 min, Nginx = 60 s by
|
||||
/// default). A short value here makes RustFS FIN pooled connections while the
|
||||
/// proxy still believes they are alive; the proxy then reuses a dead socket and
|
||||
/// the client sees `socket hang up` / connection reset — most visibly on
|
||||
/// non-idempotent `PUT` uploads, which proxies will not transparently retry
|
||||
/// (see issue #3076).
|
||||
///
|
||||
/// The default is therefore set above common proxy idle-keepalive windows.
|
||||
/// Operators fronting RustFS with a proxy should keep this larger than the
|
||||
/// proxy's upstream idle-keepalive, or lower the proxy's keepalive below this
|
||||
/// value. Environments that expose RustFS directly to untrusted slow clients and
|
||||
/// want tighter slowloris protection can lower it via the env var below.
|
||||
pub const ENV_HTTP1_HEADER_READ_TIMEOUT: &str = "RUSTFS_HTTP1_HEADER_READ_TIMEOUT";
|
||||
pub const DEFAULT_HTTP1_HEADER_READ_TIMEOUT: u64 = 5;
|
||||
pub const DEFAULT_HTTP1_HEADER_READ_TIMEOUT: u64 = 75;
|
||||
|
||||
/// Environment variable for HTTP/1.1 max buffer size (bytes)
|
||||
/// Default: 65536 (64 KB)
|
||||
pub const ENV_HTTP1_MAX_BUF_SIZE: &str = "RUSTFS_HTTP1_MAX_BUF_SIZE";
|
||||
pub const DEFAULT_HTTP1_MAX_BUF_SIZE: usize = 64 * 1024; // 64 KB
|
||||
|
||||
/// Environment variable for the S3 request-body inter-chunk read timeout
|
||||
/// (seconds). Default: 300. Set to 0 to disable.
|
||||
///
|
||||
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
|
||||
/// while more bytes are still expected. It resets on every chunk, so it does not
|
||||
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
|
||||
/// partial body and then goes silent *without* closing the connection (no EOF).
|
||||
/// A reverse proxy or CDN that forwards a truncated body this way would
|
||||
/// otherwise make RustFS wait forever for bytes that never arrive; with this
|
||||
/// timeout the stalled read is aborted and logged with the received/expected
|
||||
/// byte counts, turning a silent hang into an actionable diagnostic (issue
|
||||
/// #3076). A short-body that arrives with a proper EOF is already rejected
|
||||
/// promptly and is unaffected by this setting.
|
||||
pub const ENV_HTTP_REQUEST_BODY_READ_TIMEOUT: &str = "RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT";
|
||||
pub const DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT: u64 = 300;
|
||||
|
||||
// ── TLS Hot Reload Parameters ──
|
||||
|
||||
/// Environment variable to enable TLS certificate hot reload
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
# Running RustFS behind a reverse proxy
|
||||
|
||||
RustFS speaks plain S3 over HTTP/1.1 and HTTP/2 and works behind reverse
|
||||
proxies (Caddy, Nginx, HAProxy) and CDNs (Cloudflare). Most proxy problems are
|
||||
**not** RustFS storage bugs — the same request sent directly to `:9000`
|
||||
succeeds, while the proxied request fails. This page documents the request
|
||||
semantics RustFS expects from the proxy layer and gives known-good
|
||||
configurations.
|
||||
|
||||
> Rule of thumb: if a request works against `http://<host>:9000` directly but
|
||||
> fails through the proxy, the fault is in the proxy/CDN request forwarding, not
|
||||
> in RustFS object handling. Use the checklist below to find which forwarding
|
||||
> behavior broke.
|
||||
|
||||
## What RustFS requires from the proxy
|
||||
|
||||
S3 clients sign requests with AWS SigV4. RustFS (via `s3s`) re-derives the
|
||||
signature from the forwarded request, and streams the request body to storage.
|
||||
For this to succeed the proxy must forward the request **byte-for-byte** with
|
||||
respect to the signed material and the body:
|
||||
|
||||
1. **Do not alter the body.** No transparent compression, no re-encoding, no
|
||||
truncation. If the client sent `Content-Length: N`, exactly `N` body bytes
|
||||
must reach RustFS. If fewer bytes arrive, RustFS waits for the rest per the
|
||||
HTTP spec and the request appears to hang until the client aborts.
|
||||
2. **Do not rewrite signed headers.** `Host` and any `x-amz-*` / signed headers
|
||||
must reach RustFS unchanged. Rewriting `Host` is fine only if the client
|
||||
signed with that same host.
|
||||
3. **Preserve `Content-Length`; avoid re-chunking large bodies.** Some CDNs
|
||||
drop `Content-Length` and switch to `Transfer-Encoding: chunked`, or buffer
|
||||
the whole request body before forwarding — both change the timing and
|
||||
framing RustFS sees.
|
||||
4. **Keep upstream idle keep-alive shorter than RustFS's, or vice-versa** (see
|
||||
next section) so the proxy never reuses a connection RustFS has already
|
||||
closed.
|
||||
5. **Do not strip `ETag`** from responses (breaks multipart completion).
|
||||
|
||||
## Idle keep-alive: the #1 cause of `socket hang up` on writes
|
||||
|
||||
RustFS closes **idle** upstream HTTP/1.1 keep-alive connections after
|
||||
`RUSTFS_HTTP1_HEADER_READ_TIMEOUT` seconds (default **75s**; see
|
||||
`crates/config/src/constants/tls.rs`). Reverse proxies keep a pool of upstream
|
||||
connections and reuse them. If the proxy's upstream idle-keepalive window is
|
||||
**longer** than RustFS's timeout, the proxy can pick a connection that RustFS
|
||||
has already FIN'd, write a request onto the dead socket, and the client sees:
|
||||
|
||||
```
|
||||
TimeoutError: socket hang up # ECONNRESET
|
||||
AbortError: Request aborted
|
||||
```
|
||||
|
||||
This is most visible on large `PutObject` uploads because:
|
||||
|
||||
- `PUT` is non-idempotent, so proxies will **not** transparently retry it; and
|
||||
- a larger body keeps the connection in use longer, widening the race window,
|
||||
so small uploads on the same path often succeed.
|
||||
|
||||
### Fix — make the two windows agree
|
||||
|
||||
Pick **either** side; doing both is safest:
|
||||
|
||||
- **RustFS side:** keep `RUSTFS_HTTP1_HEADER_READ_TIMEOUT` (default 75s) *above*
|
||||
the proxy's upstream idle-keepalive. To harden slowloris protection on a
|
||||
directly-exposed node instead, lower it — but then also lower the proxy
|
||||
keepalive below it.
|
||||
- **Proxy side:** lower the proxy's upstream idle-keepalive below RustFS's
|
||||
timeout, or disable upstream keep-alive entirely.
|
||||
|
||||
## Known-good Caddy configuration
|
||||
|
||||
```caddy
|
||||
your-domain.example.com {
|
||||
reverse_proxy http://127.0.0.1:9000 {
|
||||
transport http {
|
||||
# Talk HTTP/1.1 to RustFS.
|
||||
versions 1.1
|
||||
|
||||
# Keep the proxy's upstream idle-keepalive BELOW RustFS's
|
||||
# RUSTFS_HTTP1_HEADER_READ_TIMEOUT (default 75s) so Caddy never
|
||||
# reuses a connection RustFS already closed. Set to 0 to disable
|
||||
# upstream keep-alive entirely (simplest, slightly less efficient).
|
||||
keepalive 30s
|
||||
keepalive_idle_conns_per_host 0
|
||||
|
||||
# Never let the proxy compress/transform the request body.
|
||||
compression off
|
||||
|
||||
# Generous timeouts for multi-MB single-request PUTs.
|
||||
dial_timeout 30s
|
||||
read_timeout 300s
|
||||
write_timeout 300s
|
||||
}
|
||||
|
||||
# Forward the body untouched; do not negotiate compression upstream.
|
||||
header_up Accept-Encoding identity
|
||||
|
||||
# Preserve the host the client signed with.
|
||||
header_up Host {upstream_hostport}
|
||||
|
||||
# Stream immediately instead of buffering.
|
||||
flush_interval -1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Nginx equivalent (essentials)
|
||||
|
||||
```nginx
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:9000;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
# Nginx default upstream keepalive is 60s; keep it under RustFS's 75s.
|
||||
# (set `keepalive` in the matching `upstream {}` block)
|
||||
proxy_set_header Connection "";
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Accept-Encoding "identity";
|
||||
|
||||
# Do not buffer/limit large uploads.
|
||||
proxy_request_buffering off;
|
||||
client_max_body_size 0;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 300s;
|
||||
}
|
||||
```
|
||||
|
||||
## Cloudflare (orange-cloud) caveats
|
||||
|
||||
Cloudflare's proxy (orange cloud) may **buffer the entire request body** before
|
||||
forwarding, and can rewrite requests to `Transfer-Encoding: chunked`, dropping
|
||||
the client's `Content-Length`. Symptoms match this pattern exactly: tiny uploads
|
||||
succeed, larger uploads fail with `socket hang up`.
|
||||
|
||||
- For large object writes, prefer **DNS-only (grey cloud)** for the S3 endpoint,
|
||||
or a Cloudflare plan/tunnel configuration that does not buffer/re-chunk the
|
||||
request body.
|
||||
- Force `Accept-Encoding: identity` so nothing in the path negotiates
|
||||
compression (see issues #609, #1492).
|
||||
- Ensure `Content-Length` reaches RustFS; disable chunked re-encoding in tunnel
|
||||
settings (see issue #934).
|
||||
|
||||
## Diagnosis checklist
|
||||
|
||||
Run each step and note where behavior diverges:
|
||||
|
||||
1. **Bypass the proxy.** Send the failing request to `http://<host>:9000`
|
||||
directly. Success here confirms the fault is in the proxy/CDN path.
|
||||
2. **Bypass the CDN, keep the proxy.** Point the proxy straight at the origin
|
||||
(Cloudflare grey cloud / direct DNS). If it now works, the CDN was
|
||||
buffering/re-chunking the body.
|
||||
3. **Check idle reuse.** If failures are intermittent and correlate with upload
|
||||
size, it is almost always the keep-alive mismatch above. Lower the proxy
|
||||
keepalive (or disable it) and retry.
|
||||
- If instead the upload **hangs indefinitely** (rather than resetting), the
|
||||
proxy is likely forwarding a *partial* body and then going silent without
|
||||
closing the connection. RustFS bounds this wait with
|
||||
`RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` (default 300s; `0` disables) and, on
|
||||
timeout, logs a `put_object_body_read_stalled` event with the
|
||||
received/expected byte counts — grep the server log for it to confirm a
|
||||
truncated-body forwarding problem.
|
||||
4. **Compare bytes.** Confirm the proxy forwards exactly `Content-Length` body
|
||||
bytes with no compression/transformation.
|
||||
5. **Confirm signed headers survive.** `Host` and `x-amz-*` headers must reach
|
||||
RustFS unchanged; a `SignatureDoesNotMatch` (rather than a hang) points here.
|
||||
|
||||
## Related issues
|
||||
|
||||
- #3076 — Large single-request PutObject fails behind Caddy (this document)
|
||||
- #609 — Bucket inaccessible via Cloudflare proxied DNS (`Accept-Encoding`)
|
||||
- #1492 — SigV4 `SignatureDoesNotMatch` on Cloudflare tunnel (`Accept-Encoding`)
|
||||
- #934 — Console fails behind Cloudflare tunnels (chunked / `Content-Length`)
|
||||
- #1766 — Large multipart upload fails through Nginx (`ETag` stripping)
|
||||
@@ -109,7 +109,7 @@ use crate::error::ApiError;
|
||||
use crate::server::convert_ecstore_object_info;
|
||||
use crate::table_catalog;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures::{Stream, StreamExt};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use md5::Context as Md5Context;
|
||||
use metrics::{counter, histogram};
|
||||
@@ -145,6 +145,7 @@ use rustfs_utils::http::{
|
||||
};
|
||||
use rustfs_utils::path::{encode_dir_object, is_dir_object, path_join_buf};
|
||||
use rustfs_zip::{ArchiveLimits, CompressionFormat};
|
||||
use s3s::StdError;
|
||||
use s3s::dto::{
|
||||
CacheControl, Checksum, ChecksumAlgorithm, ChecksumType, ContentDisposition, ContentEncoding, ContentLanguage, ContentType,
|
||||
CopyObjectInput, CopyObjectOutput, CopyObjectResult, CopySource, DeleteObjectInput, DeleteObjectOutput, DeleteObjectsInput,
|
||||
@@ -156,7 +157,7 @@ use s3s::dto::{
|
||||
StreamingBlob, TaggingHeader, Timestamp, TimestampFormat, WebsiteRedirectLocation,
|
||||
};
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
|
||||
use s3s::stream::{ByteStream, RemainingLength};
|
||||
use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
|
||||
const DEFAULT_PUT_LARGE_CONCURRENCY_TUNING_MIN_SIZE_BYTES: i64 = 32 * 1024 * 1024;
|
||||
@@ -214,6 +215,7 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object";
|
||||
const EVENT_PUT_OBJECT_STORE_INFLIGHT_SLOW: &str = "put_object_store_inflight_slow";
|
||||
const EVENT_PUT_OBJECT_STORE_RETURNED: &str = "put_object_store_returned";
|
||||
const EVENT_GET_OBJECT_STREAM_BODY: &str = "get_object_stream_body";
|
||||
const EVENT_PUT_OBJECT_BODY_READ_STALLED: &str = "put_object_body_read_stalled";
|
||||
const GET_OBJECT_STAGE_PATH_S3_HANDLER: &str = "s3_handler";
|
||||
const GET_OBJECT_STAGE_REQUEST_INGRESS_TO_CONTEXT: &str = "request_ingress_to_context";
|
||||
const GET_OBJECT_STAGE_OUTPUT_STRATEGY: &str = "output_strategy";
|
||||
@@ -977,6 +979,154 @@ impl<R> Drop for GetObjectStreamingReader<R> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the S3 request-body inter-chunk read timeout from the environment.
|
||||
///
|
||||
/// Returns `Duration::ZERO` when disabled (`RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT=0`),
|
||||
/// in which case [`guard_put_object_body_read_timeout`] passes the body through
|
||||
/// untouched.
|
||||
fn put_object_body_read_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
))
|
||||
}
|
||||
|
||||
/// A [`ByteStream`] decorator that aborts a request body whose peer stops
|
||||
/// sending bytes without closing the connection.
|
||||
///
|
||||
/// A well-behaved short body ends with EOF and is rejected promptly by the
|
||||
/// eager/streaming readers. The failure this guards against is different: a
|
||||
/// reverse proxy or CDN forwards a *partial* body and then goes silent while
|
||||
/// holding the connection open, so the inner stream neither yields more bytes
|
||||
/// nor reports EOF. Without a bound, RustFS would wait forever for bytes that
|
||||
/// never arrive and the client eventually sees a hang/abort with no server-side
|
||||
/// explanation (issue #3076).
|
||||
///
|
||||
/// The timer resets on every chunk, so slow-but-progressing uploads are not
|
||||
/// penalized; it only fires after `timeout` of complete silence. On timeout the
|
||||
/// stall is logged with the received/expected byte counts and the read fails
|
||||
/// with an `ErrorKind::TimedOut` error instead of hanging.
|
||||
///
|
||||
/// `remaining_length` and `size_hint` are forwarded from the inner stream so
|
||||
/// wrapping is transparent to length/content handling downstream.
|
||||
struct RequestBodyReadTimeout {
|
||||
inner: DynByteStream,
|
||||
timeout: Duration,
|
||||
timer: Option<Pin<Box<tokio::time::Sleep>>>,
|
||||
received: u64,
|
||||
expected: Option<u64>,
|
||||
bucket: String,
|
||||
key: String,
|
||||
request_id: String,
|
||||
timed_out: bool,
|
||||
}
|
||||
|
||||
impl Stream for RequestBodyReadTimeout {
|
||||
type Item = Result<Bytes, StdError>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
|
||||
// Once we have surfaced a stall error, treat the stream as terminated so
|
||||
// we never poll the abandoned inner stream again.
|
||||
if this.timed_out {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
|
||||
match Pin::new(&mut this.inner).poll_next(cx) {
|
||||
Poll::Ready(Some(Ok(chunk))) => {
|
||||
this.timer = None;
|
||||
this.received = this.received.saturating_add(chunk.len() as u64);
|
||||
Poll::Ready(Some(Ok(chunk)))
|
||||
}
|
||||
Poll::Ready(other) => {
|
||||
this.timer = None;
|
||||
Poll::Ready(other)
|
||||
}
|
||||
Poll::Pending => {
|
||||
if this.timeout.is_zero() {
|
||||
return Poll::Pending;
|
||||
}
|
||||
|
||||
if this.timer.is_none() {
|
||||
this.timer = Some(Box::pin(tokio::time::sleep(this.timeout)));
|
||||
}
|
||||
|
||||
if let Some(timer) = this.timer.as_mut()
|
||||
&& std::future::Future::poll(timer.as_mut(), cx).is_ready()
|
||||
{
|
||||
this.timer = None;
|
||||
this.timed_out = true;
|
||||
let expected_display = this.expected.map(|v| v.to_string()).unwrap_or_else(|| "unknown".to_string());
|
||||
warn!(
|
||||
target: "rustfs::app::object_usecase",
|
||||
event = EVENT_PUT_OBJECT_BODY_READ_STALLED,
|
||||
component = LOG_COMPONENT_APP,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
request_id = %this.request_id,
|
||||
bucket = %this.bucket,
|
||||
key = %this.key,
|
||||
received_bytes = this.received,
|
||||
expected_bytes = %expected_display,
|
||||
timeout_secs = this.timeout.as_secs(),
|
||||
state = "stall_timeout",
|
||||
"PutObject request body read stalled; aborting. A proxy/CDN likely forwarded a partial body without closing the connection."
|
||||
);
|
||||
return Poll::Ready(Some(Err(Box::new(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
format!(
|
||||
"request body read stalled: received {} of {} bytes, no data for {}s",
|
||||
this.received,
|
||||
expected_display,
|
||||
this.timeout.as_secs()
|
||||
),
|
||||
)) as StdError)));
|
||||
}
|
||||
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteStream for RequestBodyReadTimeout {
|
||||
fn remaining_length(&self) -> RemainingLength {
|
||||
self.inner.remaining_length()
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrap an incoming request body with [`RequestBodyReadTimeout`] unless the
|
||||
/// feature is disabled (`timeout == 0`), in which case the body is returned
|
||||
/// untouched. `remaining_length` is preserved via [`StreamingBlob::new`].
|
||||
fn guard_put_object_body_read_timeout(
|
||||
body: StreamingBlob,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
expected: Option<i64>,
|
||||
timeout: Duration,
|
||||
) -> StreamingBlob {
|
||||
if timeout.is_zero() {
|
||||
return body;
|
||||
}
|
||||
|
||||
StreamingBlob::new(RequestBodyReadTimeout {
|
||||
inner: body.into(),
|
||||
timeout,
|
||||
timer: None,
|
||||
received: 0,
|
||||
expected: expected.and_then(|v| u64::try_from(v).ok()),
|
||||
bucket: bucket.to_string(),
|
||||
key: key.to_string(),
|
||||
request_id: request_id.to_string(),
|
||||
timed_out: false,
|
||||
})
|
||||
}
|
||||
|
||||
impl<R> ExtractArchiveEtagReader<R> {
|
||||
fn new(inner: R, etag: Arc<Mutex<Option<String>>>) -> Self {
|
||||
Self {
|
||||
@@ -3157,6 +3307,18 @@ impl DefaultObjectUsecase {
|
||||
|
||||
let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
|
||||
|
||||
// Guard against a proxy/CDN that forwards a partial body then goes silent
|
||||
// without closing the connection: bound the inter-chunk wait so the read
|
||||
// fails (with a diagnostic log) instead of hanging forever (issue #3076).
|
||||
let body = {
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone())
|
||||
.unwrap_or_default();
|
||||
guard_put_object_body_read_timeout(body, &bucket, &key, &request_id, content_length, put_object_body_read_timeout())
|
||||
};
|
||||
|
||||
let decoded_content_length = decoded_content_length_from_headers(&req.headers)?;
|
||||
let mut size = match (request_uses_aws_chunked(&req.headers), decoded_content_length, content_length) {
|
||||
(true, Some(decoded), _) => decoded,
|
||||
@@ -6757,6 +6919,64 @@ mod tests {
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_body_read_timeout_guard_aborts_on_stall() {
|
||||
// Inner stream never yields and never reports EOF (a proxy that forwarded
|
||||
// a partial body then went silent while holding the connection open).
|
||||
let inner = StreamingBlob::wrap(futures::stream::pending::<Result<Bytes, std::io::Error>>());
|
||||
let mut guarded = guard_put_object_body_read_timeout(
|
||||
inner,
|
||||
"test-bucket",
|
||||
"stalled-object",
|
||||
"req-1",
|
||||
Some(1024),
|
||||
Duration::from_millis(1),
|
||||
);
|
||||
|
||||
let err = guarded
|
||||
.next()
|
||||
.await
|
||||
.expect("guard should yield a stall error")
|
||||
.expect_err("stalled body should return an error");
|
||||
let io_err = err
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.expect("stall error should wrap an io::Error");
|
||||
assert_eq!(io_err.kind(), std::io::ErrorKind::TimedOut);
|
||||
|
||||
// After a stall the guard terminates the stream instead of re-polling the
|
||||
// abandoned inner stream.
|
||||
assert!(guarded.next().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_body_read_timeout_guard_preserves_length_and_passes_through() {
|
||||
let body = StreamingBlob::from(s3s::Body::from(Bytes::from_static(b"hello world")));
|
||||
assert_eq!(body.remaining_length().exact(), Some(11));
|
||||
|
||||
let mut guarded =
|
||||
guard_put_object_body_read_timeout(body, "test-bucket", "ok-object", "req-2", Some(11), Duration::from_secs(60));
|
||||
// remaining_length must be forwarded, not reset to unknown.
|
||||
assert_eq!(guarded.remaining_length().exact(), Some(11));
|
||||
|
||||
let mut collected = Vec::new();
|
||||
while let Some(chunk) = guarded.next().await {
|
||||
collected.extend_from_slice(&chunk.expect("chunk should read"));
|
||||
}
|
||||
assert_eq!(collected, b"hello world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_body_read_timeout_guard_disabled_passthrough() {
|
||||
let body = StreamingBlob::from(s3s::Body::from(Bytes::from_static(b"data")));
|
||||
let mut guarded = guard_put_object_body_read_timeout(body, "test-bucket", "ok-object", "req-3", Some(4), Duration::ZERO);
|
||||
|
||||
let mut collected = Vec::new();
|
||||
while let Some(chunk) = guarded.next().await {
|
||||
collected.extend_from_slice(&chunk.expect("chunk should read"));
|
||||
}
|
||||
assert_eq!(collected, b"data");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn get_object_streaming_reader_holds_request_guard_until_eof() {
|
||||
|
||||
Reference in New Issue
Block a user