mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-12 16:16:55 +00:00
fix(rio): map truncated put bodies to incompletebody (#3168)
* fix(rio): surface incomplete put bodies Propagate incomplete PUT request bodies as IncompleteBody instead of allowing erasure encode to treat truncated input as a normal EOF.\n\n- mark premature EOFs in HardLimitReader with an explicit IncompleteBody error\n- preserve EOF error chains through read_full and map them to S3 IncompleteBody\n- stop erasure encode from swallowing UnexpectedEof on truncated input\n- add regression tests for reader, erasure encode, and API error mapping\n\nRefs: rustfs/backlog#654 * fix(s3): honor decoded length for aws chunked put Use x-amz-decoded-content-length for aws-chunked PutObject requests so trailer-checksum uploads are sized against the decoded payload instead of the wire-encoded content-length.\n\n- prefer decoded content length for aws-chunked put bodies\n- add a regression test covering the size selection logic\n- keeps the incomplete body fix working for truly truncated uploads while restoring checksum trailer compatibility\n\nRefs: rustfs/backlog#654 * fix(io): follow up review comments on incompletebody handling Address PR review feedback by restoring read_full's existing EOF contract, adding a dedicated read_full_or_eof helper for erasure encoding, covering nested incomplete-body error chains, and documenting plus hardening aws-chunked size selection.\n\n- keep read_full returning early EOF on empty reads\n- use read_full_or_eof only in erasure encoding paths\n- detect aws-chunked via content-encoding or transfer-encoding\n- add nested error-chain and aws-chunked regression tests\n\nRefs: rustfs/backlog#654 * fix(rio): surface incomplete put bodies Propagate incomplete PUT request bodies as IncompleteBody instead of allowing erasure encode to treat truncated input as a normal EOF.\n\n- mark premature EOFs in HardLimitReader with an explicit IncompleteBody error\n- preserve EOF error chains through read_full and map them to S3 IncompleteBody\n- stop erasure encode from swallowing UnexpectedEof on truncated input\n- add regression tests for reader, erasure encode, and API error mapping\n\nRefs: rustfs/backlog#654 * fix(s3): honor decoded length for aws chunked put Use x-amz-decoded-content-length for aws-chunked PutObject requests so trailer-checksum uploads are sized against the decoded payload instead of the wire-encoded content-length.\n\n- prefer decoded content length for aws-chunked put bodies\n- add a regression test covering the size selection logic\n- keeps the incomplete body fix working for truly truncated uploads while restoring checksum trailer compatibility\n\nRefs: rustfs/backlog#654 * fix(io): follow up review comments on incompletebody handling Address PR review feedback by restoring read_full's existing EOF contract, adding a dedicated read_full_or_eof helper for erasure encoding, covering nested incomplete-body error chains, and documenting plus hardening aws-chunked size selection.\n\n- keep read_full returning early EOF on empty reads\n- use read_full_or_eof only in erasure encoding paths\n- detect aws-chunked via content-encoding or transfer-encoding\n- add nested error-chain and aws-chunked regression tests\n\nRefs: rustfs/backlog#654 * fix(rio): reject bytes beyond hard limit * fix(ecstore): reject zero-sized erasure blocks
This commit is contained in:
+34
-11
@@ -29,10 +29,15 @@ pub async fn write_all<W: AsyncWrite + Send + Sync + Unpin>(writer: &mut W, buf:
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// Read exactly buf.len() bytes into buf, or return an error if EOF is reached before.
|
||||
/// Like Go's io.ReadFull.
|
||||
#[allow(dead_code)]
|
||||
pub async fn read_full<R: AsyncRead + Send + Sync + Unpin>(mut reader: R, mut buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
/// Read up to buf.len() bytes into buf and distinguish a clean EOF from a short read.
|
||||
///
|
||||
/// Returns `Ok(None)` when EOF is reached before any bytes are read, `Ok(Some(n))` when
|
||||
/// at least one byte is read, and preserves the underlying error chain when the reader
|
||||
/// fails after a partial fill.
|
||||
pub async fn read_full_or_eof<R: AsyncRead + Send + Sync + Unpin>(
|
||||
mut reader: R,
|
||||
mut buf: &mut [u8],
|
||||
) -> std::io::Result<Option<usize>> {
|
||||
let mut total = 0;
|
||||
while !buf.is_empty() {
|
||||
let n = match reader.read(buf).await {
|
||||
@@ -46,22 +51,29 @@ pub async fn read_full<R: AsyncRead + Send + Sync + Unpin>(mut reader: R, mut bu
|
||||
if e.kind() == std::io::ErrorKind::InvalidData {
|
||||
return Err(e);
|
||||
}
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
format!("read {total} bytes, error: {e}"),
|
||||
));
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, e));
|
||||
}
|
||||
};
|
||||
if n == 0 {
|
||||
if total > 0 {
|
||||
return Ok(total);
|
||||
return Ok(Some(total));
|
||||
}
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "early EOF"));
|
||||
return Ok(None);
|
||||
}
|
||||
buf = &mut buf[n..];
|
||||
total += n;
|
||||
}
|
||||
Ok(total)
|
||||
Ok(Some(total))
|
||||
}
|
||||
|
||||
/// Read exactly buf.len() bytes into buf, or return an error if EOF is reached before any bytes are read.
|
||||
/// Like Go's io.ReadFull.
|
||||
#[allow(dead_code)]
|
||||
pub async fn read_full<R: AsyncRead + Send + Sync + Unpin>(reader: R, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
match read_full_or_eof(reader, buf).await? {
|
||||
Some(n) => Ok(n),
|
||||
None => Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "early EOF")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Encodes a u64 into buf and returns the number of bytes written.
|
||||
@@ -163,6 +175,17 @@ mod tests {
|
||||
assert_eq!(buf, data[..size / 3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_full_or_eof_returns_none_for_empty_reader() {
|
||||
let data = b"";
|
||||
let mut reader = BufReader::new(&data[..]);
|
||||
let mut buf = [0u8; 8];
|
||||
|
||||
let n = read_full_or_eof(&mut reader, &mut buf).await.unwrap();
|
||||
|
||||
assert_eq!(n, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_uvarint_and_uvarint_zero() {
|
||||
let mut buf = [0u8; 16];
|
||||
|
||||
Reference in New Issue
Block a user