gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 10:18:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(filemeta): guard metacache decoding against corrupt length prefixes (#4226)

Browse Source
This commit is contained in:
Zhengchao An
2026-07-03 12:27:35 +08:00
committed by GitHub
parent c59d4e6f89
commit d2c100fd3e
6 changed files with 333 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/filemeta/codec.rs
+35 -2
View File
@@ -179,6 +179,22 @@ impl FileMeta {
self.meta_ver = meta_ver;
// `versions_len` is decoded from a potentially corrupted buffer.
// Every version contributes at least two msgpack bin headers to
// `meta`, so a count larger than the remaining bytes is corrupt;
// reject it before sizing any allocation from it (rustfs/rustfs#2715).
if versions_len > meta.len() {
error!(
"corrupt xl.meta: version count {} exceeds remaining metadata size {}",
versions_len,
meta.len()
);
return Err(Error::other(format!(
"corrupt xl.meta: version count {versions_len} exceeds metadata size {}",
meta.len()
)));
}
self.versions = Vec::with_capacity(versions_len);
let mut cur: Cursor<&[u8]> = Cursor::new(meta);
@@ -188,6 +204,11 @@ impl FileMeta {
Error::other(format!("failed to read binary length for version header: {e}"))
})? as usize;
let remaining = meta.len().saturating_sub(cur.position() as usize);
if bin_len > remaining {
error!("corrupt xl.meta: version header length {} exceeds remaining {} bytes", bin_len, remaining);
return Err(Error::other("corrupt xl.meta: version header length exceeds metadata size"));
}
let mut header_buf = vec![0u8; bin_len];
cur.read_exact(&mut header_buf)?;
@@ -203,6 +224,14 @@ impl FileMeta {
Error::other(format!("failed to read binary length for version metadata: {e}"))
})? as usize;
let remaining = meta.len().saturating_sub(cur.position() as usize);
if bin_len > remaining {
error!(
"corrupt xl.meta: version metadata length {} exceeds remaining {} bytes",
bin_len, remaining
);
return Err(Error::other("corrupt xl.meta: version metadata length exceeds metadata size"));
}
let mut ver_meta_buf = vec![0u8; bin_len];
cur.read_exact(&mut ver_meta_buf)?;
@@ -242,14 +271,18 @@ impl FileMeta {
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
let start = cur.position() as usize;
let end = start + bin_len;
let header_buf = &buf[start..end];
let header_buf = buf
.get(start..end)
.ok_or_else(|| Error::other("corrupt xl.meta: version header segment out of range"))?;
cur.set_position(end as u64);
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
let start = cur.position() as usize;
let end = start + bin_len;
let ver_meta_buf = &buf[start..end];
let ver_meta_buf = buf
.get(start..end)
.ok_or_else(|| Error::other("corrupt xl.meta: version metadata segment out of range"))?;
cur.set_position(end as u64);
crates/filemeta/src/filemeta/msgp_decode.rs
+76 -2
View File
@@ -16,6 +16,46 @@ use crate::{Error, Result};
use rmp::Marker;
use std::io::Read;
/// Maximum accepted length for a single length-prefixed msgpack element
/// (map key, string, binary blob, ext payload, or a whole serialized
/// xl.meta record inside a metacache stream).
///
/// Legitimate elements are far smaller: object keys are at most a few KiB
/// and even an xl.meta record fully populated up to the 10,000-version
/// object cap serializes to a few MiB (metacache streams carry xl.meta
/// without inline data), so 16 MiB leaves ample headroom. A larger value
/// means the length prefix itself is corrupt; decoding must fail with an
/// error instead of attempting a huge allocation that aborts the whole
/// process (see rustfs/rustfs#2715).
pub(crate) const MAX_MSGP_ELEMENT_SIZE: usize = 16 << 20;
/// Reads exactly `len` bytes into a fresh buffer, treating `len` as
/// untrusted input: lengths above [`MAX_MSGP_ELEMENT_SIZE`] and allocation
/// failures surface as decode errors instead of aborting the process.
pub(crate) fn read_exact_vec<R: Read>(rd: &mut R, len: usize) -> Result<Vec<u8>> {
if len > MAX_MSGP_ELEMENT_SIZE {
return Err(Error::other(format!(
"corrupt msgpack element: length {len} exceeds the {MAX_MSGP_ELEMENT_SIZE} byte limit"
)));
}
let mut buf = Vec::new();
buf.try_reserve_exact(len)
.map_err(|e| Error::other(format!("msgpack element allocation of {len} bytes failed: {e}")))?;
buf.resize(len, 0);
rd.read_exact(&mut buf).map_err(Error::from)?;
Ok(buf)
}
/// Bounds a decoded collection count when it is used only as a
/// pre-allocation hint. The decode loop still consumes exactly the decoded
/// number of elements (a corrupt count fails with an EOF decode error once
/// the input runs out); this merely keeps the speculative reservation from
/// aborting the process on an absurd count.
pub(crate) fn prealloc_hint(len: usize) -> usize {
const MAX_PREALLOC_ITEMS: usize = 4096;
len.min(MAX_PREALLOC_ITEMS)
}
/// Reader that prepends a single byte to the stream. Used when we've read the marker
/// and need to pass it to a decoder that expects to read the marker itself.
pub(crate) struct PrependByteReader<'a, R> {
@@ -207,8 +247,42 @@ pub(crate) fn skip_msgp_value<R: Read>(rd: &mut R) -> Result<()> {
Marker::Reserved => 0,
};
if skip_len > 0 {
let mut buf = vec![0u8; skip_len];
rd.read_exact(&mut buf).map_err(Error::from)?;
// Discard the payload without allocating a buffer sized by the
// untrusted length; a truncated stream surfaces as UnexpectedEof.
let copied = std::io::copy(&mut rd.by_ref().take(skip_len as u64), &mut std::io::sink()).map_err(Error::from)?;
if copied != skip_len as u64 {
return Err(Error::from(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"truncated msgpack value",
)));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn read_exact_vec_rejects_oversized_length() {
let mut rd: &[u8] = &[];
let err = read_exact_vec(&mut rd, MAX_MSGP_ELEMENT_SIZE + 1).expect_err("oversized length must be rejected");
assert!(err.to_string().contains("exceeds"), "unexpected error: {err}");
}
#[test]
fn read_exact_vec_reads_exact_payload() {
let mut rd: &[u8] = b"hello";
assert_eq!(read_exact_vec(&mut rd, 5).unwrap(), b"hello");
}
#[test]
fn skip_msgp_value_rejects_truncated_huge_payload_without_allocating() {
// bin32 claiming u32::MAX bytes with no payload behind it.
let mut data = vec![0xc6];
data.extend_from_slice(&u32::MAX.to_be_bytes());
let mut rd = data.as_slice();
assert!(skip_msgp_value(&mut rd).is_err());
}
}
crates/filemeta/src/filemeta/version.rs
+36 -51
View File
@@ -22,7 +22,9 @@
//! Do NOT use `FileMetaVersion::default()` + `unmarshal_msg()` directly, as that fails on
//! legacy (rmp_serde) format. `try_from` falls back to rmp_serde when hand-written decode fails.
use super::msgp_decode::{PrependByteReader, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value};
use super::msgp_decode::{
PrependByteReader, prealloc_hint, read_exact_vec, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value,
};
use super::*;
use crate::ChecksumInfo;
use rustfs_utils::HashAlgorithm;
@@ -42,16 +44,13 @@ const MSGPACK_TIME_EXT_OFFICIAL: i8 = -1;
fn read_msgp_string<R: std::io::Read>(rd: &mut R) -> Result<String> {
let len = rmp::decode::read_str_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
let buf = read_exact_vec(rd, len)?;
Ok(String::from_utf8(buf)?)
}
fn read_msgp_bin<R: std::io::Read>(rd: &mut R) -> Result<Vec<u8>> {
let len = rmp::decode::read_bin_len(rd)? as usize;
let mut buf = vec![0u8; len];
rd.read_exact(&mut buf)?;
Ok(buf)
read_exact_vec(rd, len)
}
fn deserialize_legacy_uuid_bytes<'de, D>(deserializer: D) -> std::result::Result<Vec<u8>, D::Error>
@@ -183,8 +182,7 @@ fn read_msgp_time<R: std::io::Read>(rd: &mut R) -> Result<OffsetDateTime> {
other => return Err(Error::other(format!("unsupported msgpack time marker: 0x{other:02x}"))),
};
let mut payload = vec![0u8; len];
rd.read_exact(&mut payload)?;
let payload = read_exact_vec(rd, len)?;
decode_msgp_time_payload(ext_type, &payload)
}
@@ -362,8 +360,7 @@ impl FileMetaVersion {
fields -= 1;
let key_len = rmp::decode::read_str_len(&mut cur)? as usize;
let mut key_buf = vec![0u8; key_len];
cur.read_exact(&mut key_buf)?;
let key_buf = read_exact_vec(&mut cur, key_len)?;
let key = String::from_utf8(key_buf)?;
match key.as_str() {
@@ -394,8 +391,7 @@ impl FileMetaVersion {
obj_fields -= 1;
let obj_key_len = rmp::decode::read_str_len(&mut prepend)? as usize;
let mut obj_key_buf = vec![0u8; obj_key_len];
prepend.read_exact(&mut obj_key_buf)?;
let obj_key_buf = read_exact_vec(&mut prepend, obj_key_len)?;
let obj_key = String::from_utf8(obj_key_buf)?;
if obj_key == "DDir" {
@@ -493,8 +489,7 @@ impl FileMetaVersion {
fields -= 1;
let key_len = rmp::decode::read_str_len(rd)?;
let mut key_buf = vec![0u8; key_len as usize];
rd.read_exact(&mut key_buf)?;
let key_buf = read_exact_vec(rd, key_len as usize)?;
let key = String::from_utf8(key_buf)?;
match key.as_str() {
@@ -1307,7 +1302,7 @@ impl MetaObjectV1 {
"Parts" => {
let len = rmp::decode::read_array_len(rd)? as usize;
self.parts.clear();
self.parts.reserve(len);
self.parts.reserve(prealloc_hint(len));
for _ in 0..len {
let mut part = MetaObjectV1Part::default();
part.decode_from(rd)?;
@@ -1424,7 +1419,7 @@ impl MetaObjectV1Erasure {
"Distribution" => {
let len = rmp::decode::read_array_len(rd)? as usize;
self.distribution.clear();
self.distribution.reserve(len);
self.distribution.reserve(prealloc_hint(len));
for _ in 0..len {
self.distribution.push(rmp::decode::read_int::<i64, _>(rd)? as usize);
}
@@ -1432,7 +1427,7 @@ impl MetaObjectV1Erasure {
"Checksums" => {
let len = rmp::decode::read_array_len(rd)? as usize;
self.checksums.clear();
self.checksums.reserve(len);
self.checksums.reserve(prealloc_hint(len));
for _ in 0..len {
let mut checksum = MetaObjectV1ChecksumInfo::default();
checksum.decode_from(rd)?;
@@ -1484,7 +1479,7 @@ impl MetaObjectV1Part {
"i" => self.index = Some(Bytes::from(read_msgp_bin(rd)?)),
"crc" => {
let len = rmp::decode::read_map_len(rd)? as usize;
let mut checksums = HashMap::with_capacity(len);
let mut checksums = HashMap::with_capacity(prealloc_hint(len));
for _ in 0..len {
checksums.insert(read_msgp_string(rd)?, read_msgp_string(rd)?);
}
@@ -1700,9 +1695,8 @@ impl MetaObject {
tracing::error!(error = %e, "decode_from: read_str_len key failed");
e
})?;
let mut key_buf = vec![0u8; key_len as usize];
rd.read_exact(&mut key_buf).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_exact key_buf failed");
let key_buf = read_exact_vec(rd, key_len as usize).map_err(|e| {
tracing::error!(error = %e, "decode_from: read key_buf failed");
e
})?;
let key = String::from_utf8(key_buf).map_err(|e| {
@@ -1778,7 +1772,7 @@ impl MetaObject {
e
})? as usize;
self.erasure_dist.clear();
self.erasure_dist.reserve(len);
self.erasure_dist.reserve(prealloc_hint(len));
for _ in 0..len {
let v: i64 = rmp::decode::read_int(rd).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_int EcDist item failed");
@@ -1800,7 +1794,7 @@ impl MetaObject {
e
})? as usize;
self.part_numbers.clear();
self.part_numbers.reserve(len);
self.part_numbers.reserve(prealloc_hint(len));
for _ in 0..len {
let v: i64 = rmp::decode::read_int(rd).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_int PartNums item failed");
@@ -1821,15 +1815,14 @@ impl MetaObject {
Some(n) => n,
};
self.part_etags.clear();
self.part_etags.reserve(len);
self.part_etags.reserve(prealloc_hint(len));
for _ in 0..len {
let s_len = rmp::decode::read_str_len(rd).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_str_len PartETags item failed");
e
})?;
let mut sbuf = vec![0u8; s_len as usize];
rd.read_exact(&mut sbuf).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_exact PartETags sbuf failed");
let sbuf = read_exact_vec(rd, s_len as usize).map_err(|e| {
tracing::error!(error = %e, "decode_from: read PartETags sbuf failed");
e
})?;
let s = String::from_utf8(sbuf).map_err(|e| {
@@ -1845,7 +1838,7 @@ impl MetaObject {
e
})? as usize;
self.part_sizes.clear();
self.part_sizes.reserve(len);
self.part_sizes.reserve(prealloc_hint(len));
for _ in 0..len {
let v: i64 = rmp::decode::read_int(rd).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_int PartSizes item failed");
@@ -1866,7 +1859,7 @@ impl MetaObject {
Some(n) => n,
};
self.part_actual_sizes.clear();
self.part_actual_sizes.reserve(len);
self.part_actual_sizes.reserve(prealloc_hint(len));
for _ in 0..len {
let v: i64 = rmp::decode::read_int(rd).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_int PartASizes item failed");
@@ -1881,15 +1874,14 @@ impl MetaObject {
e
})? as usize;
self.part_indices.clear();
self.part_indices.reserve(len);
self.part_indices.reserve(prealloc_hint(len));
for _ in 0..len {
let blen = rmp::decode::read_bin_len(rd).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_bin_len PartIdx item failed");
e
})? as usize;
let mut buf = vec![0u8; blen];
rd.read_exact(&mut buf).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_exact PartIdx buf failed");
let buf = read_exact_vec(rd, blen).map_err(|e| {
tracing::error!(error = %e, "decode_from: read PartIdx buf failed");
e
})?;
self.part_indices.push(Bytes::from(buf));
@@ -1933,9 +1925,8 @@ impl MetaObject {
tracing::error!(error = %e, "decode_from: read_str_len MetaSys key failed");
e
})?;
let mut kbuf = vec![0u8; k_len as usize];
rd.read_exact(&mut kbuf).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_exact MetaSys kbuf failed");
let kbuf = read_exact_vec(rd, k_len as usize).map_err(|e| {
tracing::error!(error = %e, "decode_from: read MetaSys kbuf failed");
e
})?;
let k = String::from_utf8(kbuf).map_err(|e| {
@@ -1947,9 +1938,8 @@ impl MetaObject {
tracing::error!(error = %e, "decode_from: read_bin_len MetaSys value failed");
e
})? as usize;
let mut v = vec![0u8; blen];
rd.read_exact(&mut v).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_exact MetaSys value failed");
let v = read_exact_vec(rd, blen).map_err(|e| {
tracing::error!(error = %e, "decode_from: read MetaSys value failed");
e
})?;
@@ -1973,9 +1963,8 @@ impl MetaObject {
tracing::error!(error = %e, "decode_from: read_str_len MetaUsr key failed");
e
})?;
let mut kbuf = vec![0u8; k_len as usize];
rd.read_exact(&mut kbuf).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_exact MetaUsr kbuf failed");
let kbuf = read_exact_vec(rd, k_len as usize).map_err(|e| {
tracing::error!(error = %e, "decode_from: read MetaUsr kbuf failed");
e
})?;
let k = String::from_utf8(kbuf).map_err(|e| {
@@ -1987,9 +1976,8 @@ impl MetaObject {
tracing::error!(error = %e, "decode_from: read_str_len MetaUsr value failed");
e
})?;
let mut vbuf = vec![0u8; v_len as usize];
rd.read_exact(&mut vbuf).map_err(|e| {
tracing::error!(error = %e, "decode_from: read_exact MetaUsr vbuf failed");
let vbuf = read_exact_vec(rd, v_len as usize).map_err(|e| {
tracing::error!(error = %e, "decode_from: read MetaUsr vbuf failed");
e
})?;
let v = String::from_utf8(vbuf).map_err(|e| {
@@ -2446,8 +2434,7 @@ impl MetaDeleteMarker {
fields -= 1;
let key_len = rmp::decode::read_str_len(rd)?;
let mut key_buf = vec![0u8; key_len as usize];
rd.read_exact(&mut key_buf)?;
let key_buf = read_exact_vec(rd, key_len as usize)?;
let key = String::from_utf8(key_buf)?;
match key.as_str() {
@@ -2472,13 +2459,11 @@ impl MetaDeleteMarker {
self.meta_sys.clear();
for _ in 0..len {
let k_len = rmp::decode::read_str_len(rd)?;
let mut kbuf = vec![0u8; k_len as usize];
rd.read_exact(&mut kbuf)?;
let kbuf = read_exact_vec(rd, k_len as usize)?;
let k = String::from_utf8(kbuf)?;
let blen = rmp::decode::read_bin_len(rd)? as usize;
let mut v = vec![0u8; blen];
rd.read_exact(&mut v)?;
let v = read_exact_vec(rd, blen)?;
self.meta_sys.insert(k, v);
}