fix(rio): map truncated put bodies to incompletebody (#3168)

* fix(rio): surface incomplete put bodies

Propagate incomplete PUT request bodies as IncompleteBody instead of allowing erasure encode to treat truncated input as a normal EOF.\n\n- mark premature EOFs in HardLimitReader with an explicit IncompleteBody error\n- preserve EOF error chains through read_full and map them to S3 IncompleteBody\n- stop erasure encode from swallowing UnexpectedEof on truncated input\n- add regression tests for reader, erasure encode, and API error mapping\n\nRefs: rustfs/backlog#654

* fix(s3): honor decoded length for aws chunked put

Use x-amz-decoded-content-length for aws-chunked PutObject requests so trailer-checksum uploads are sized against the decoded payload instead of the wire-encoded content-length.\n\n- prefer decoded content length for aws-chunked put bodies\n- add a regression test covering the size selection logic\n- keeps the incomplete body fix working for truly truncated uploads while restoring checksum trailer compatibility\n\nRefs: rustfs/backlog#654

* fix(io): follow up review comments on incompletebody handling

Address PR review feedback by restoring read_full's existing EOF contract, adding a dedicated read_full_or_eof helper for erasure encoding, covering nested incomplete-body error chains, and documenting plus hardening aws-chunked size selection.\n\n- keep read_full returning early EOF on empty reads\n- use read_full_or_eof only in erasure encoding paths\n- detect aws-chunked via content-encoding or transfer-encoding\n- add nested error-chain and aws-chunked regression tests\n\nRefs: rustfs/backlog#654

* fix(rio): surface incomplete put bodies

Propagate incomplete PUT request bodies as IncompleteBody instead of allowing erasure encode to treat truncated input as a normal EOF.\n\n- mark premature EOFs in HardLimitReader with an explicit IncompleteBody error\n- preserve EOF error chains through read_full and map them to S3 IncompleteBody\n- stop erasure encode from swallowing UnexpectedEof on truncated input\n- add regression tests for reader, erasure encode, and API error mapping\n\nRefs: rustfs/backlog#654

* fix(s3): honor decoded length for aws chunked put

Use x-amz-decoded-content-length for aws-chunked PutObject requests so trailer-checksum uploads are sized against the decoded payload instead of the wire-encoded content-length.\n\n- prefer decoded content length for aws-chunked put bodies\n- add a regression test covering the size selection logic\n- keeps the incomplete body fix working for truly truncated uploads while restoring checksum trailer compatibility\n\nRefs: rustfs/backlog#654

* fix(io): follow up review comments on incompletebody handling

Address PR review feedback by restoring read_full's existing EOF contract, adding a dedicated read_full_or_eof helper for erasure encoding, covering nested incomplete-body error chains, and documenting plus hardening aws-chunked size selection.\n\n- keep read_full returning early EOF on empty reads\n- use read_full_or_eof only in erasure encoding paths\n- detect aws-chunked via content-encoding or transfer-encoding\n- add nested error-chain and aws-chunked regression tests\n\nRefs: rustfs/backlog#654

* fix(rio): reject bytes beyond hard limit

* fix(ecstore): reject zero-sized erasure blocks
This commit is contained in:
houseme
2026-06-02 19:31:51 +08:00
committed by GitHub
parent 480babc0af
commit 0d00b886ac
7 changed files with 310 additions and 37 deletions
+70 -3
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::IncompleteBody;
use pin_project_lite::pin_project;
use std::io::{Error, Result};
use std::pin::Pin;
@@ -40,8 +41,25 @@ where
if self.remaining < 0 {
return Poll::Ready(Err(Error::other("input provided more bytes than specified")));
}
let original_filled = buf.filled().len();
if self.remaining == 0 {
let mut discard = [0u8; 8192];
let mut discard_buf = ReadBuf::new(&mut discard);
return match self.as_mut().project().inner.poll_read(cx, &mut discard_buf) {
Poll::Pending => Poll::Pending,
Poll::Ready(Ok(())) => {
if discard_buf.filled().is_empty() {
debug_assert_eq!(buf.filled().len(), original_filled);
Poll::Ready(Ok(()))
} else {
Poll::Ready(Err(Error::other("input provided more bytes than specified")))
}
}
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
};
}
// Save the initial length
let before = buf.filled().len();
let before = original_filled;
// Poll the inner reader
let this = self.as_mut().project();
@@ -50,6 +68,14 @@ where
if let Poll::Ready(Ok(())) = &poll {
let after = buf.filled().len();
let read = (after - before) as i64;
if read == 0 && *this.remaining > 0 {
return Poll::Ready(Err(Error::new(
std::io::ErrorKind::UnexpectedEof,
IncompleteBody {
remaining: *this.remaining,
},
)));
}
*this.remaining -= read;
if *this.remaining < 0 {
return Poll::Ready(Err(Error::other("input provided more bytes than specified")));
@@ -73,7 +99,7 @@ mod tests {
async fn test_hardlimit_reader_normal() {
let data = b"hello world";
let reader = BufReader::new(&data[..]);
let hardlimit = HardLimitReader::new(reader, 20);
let hardlimit = HardLimitReader::new(reader, data.len() as i64);
let mut r = hardlimit;
let mut buf = Vec::new();
let n = r.read_to_end(&mut buf).await.unwrap();
@@ -121,11 +147,52 @@ mod tests {
async fn test_hardlimit_reader_empty() {
let data = b"";
let reader = BufReader::new(&data[..]);
let hardlimit = HardLimitReader::new(reader, 5);
let hardlimit = HardLimitReader::new(reader, 0);
let mut r = hardlimit;
let mut buf = Vec::new();
let n = r.read_to_end(&mut buf).await.unwrap();
assert_eq!(n, 0);
assert_eq!(&buf, data);
}
#[tokio::test]
async fn test_hardlimit_reader_short_input_returns_unexpected_eof() {
let data = b"abc";
let reader = BufReader::new(&data[..]);
let mut r = HardLimitReader::new(reader, 5);
let mut buf = [0u8; 8];
let err = read_full(&mut r, &mut buf)
.await
.expect_err("short input must surface unexpected eof");
assert_eq!(err.kind(), std::io::ErrorKind::UnexpectedEof);
assert!(
err.get_ref()
.and_then(|inner| inner.downcast_ref::<std::io::Error>())
.and_then(|inner| inner.get_ref())
.and_then(|inner| inner.downcast_ref::<IncompleteBody>())
.is_some(),
"error should retain the incomplete body marker"
);
}
#[tokio::test]
async fn test_hardlimit_reader_rejects_extra_bytes_after_limit() {
let data = b"abcdef";
let reader = BufReader::new(&data[..]);
let mut r = HardLimitReader::new(reader, 3);
let mut first = [0u8; 3];
let n = read_full(&mut r, &mut first).await.expect("first read should consume limit");
assert_eq!(n, 3);
assert_eq!(&first, b"abc");
let mut second = [0u8; 1];
let err = read_full(&mut r, &mut second)
.await
.expect_err("bytes beyond the declared limit must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::Other);
assert!(err.to_string().contains("more bytes than specified"));
}
}