fix(filemeta): support legacy xl.meta compatibility (#2304)

This commit is contained in:
weisd
2026-03-27 13:42:06 +08:00
committed by GitHub
parent af46a61fde
commit 5e21c398f5
8 changed files with 1205 additions and 19 deletions
+105
View File
@@ -1055,6 +1055,108 @@ mod test {
assert_eq!(stats.invalid_versions, 1); // Legacy is counted as invalid
}
#[test]
fn test_issue_2288_legacy_xlmeta_compatibility() {
let data = create_issue_2288_legacy_xlmeta().expect("Failed to load issue #2288 fixture");
let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap();
assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 2, 1));
let fm = FileMeta::load(&data).expect("Failed to parse legacy issue #2288 xl.meta");
assert_eq!(fm.meta_ver, 1);
assert_eq!(fm.versions.len(), 1);
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
assert_eq!(fm.versions[0].header.signature, [0x96, 0x33, 0x4c, 0x78]);
assert_eq!(fm.versions[0].header.ec_n, 0);
assert_eq!(fm.versions[0].header.ec_m, 0);
let fi = fm
.into_fileinfo("viscom", "test.txt", "", true, false, true)
.expect("Failed to extract file info from legacy issue #2288 xl.meta");
assert_eq!(fi.size, 35);
assert_eq!(fi.num_versions, 1);
assert!(fi.is_latest);
}
#[test]
fn test_issue_2265_legacy_meta_v2_object_compatibility() {
let data = create_issue_2265_legacy_meta_v2_object_xlmeta().expect("Failed to load issue #2265 object fixture");
let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap();
assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 3, 2));
let fm = FileMeta::load(&data).expect("Failed to parse legacy issue #2265 object xl.meta");
assert_eq!(fm.meta_ver, 2);
assert_eq!(fm.versions.len(), 1);
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
assert_eq!(fm.versions[0].header.ec_n, 0);
assert_eq!(fm.versions[0].header.ec_m, 1);
let fi = fm
.into_fileinfo("bucket", ".metadata.bin", "", true, false, true)
.expect("Failed to extract file info from legacy issue #2265 object xl.meta");
assert_eq!(fi.size, 707);
assert_eq!(fi.num_versions, 1);
assert_eq!(fi.metadata.get("etag").map(String::as_str), Some("4359404618e32a0bd8944e9ff6802f53"));
assert_eq!(
fi.data_dir.map(|id| id.to_string()).as_deref(),
Some("04bee19a-6eea-40c4-96fd-1f39257fcbdc")
);
assert!(fi.uses_legacy_checksum);
assert!(fi.is_latest);
}
#[test]
fn test_issue_2265_legacy_meta_v2_config_compatibility() {
let data = create_issue_2265_legacy_meta_v2_config_xlmeta().expect("Failed to load issue #2265 config fixture");
let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap();
assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 3, 2));
let fm = FileMeta::load(&data).expect("Failed to parse legacy issue #2265 config xl.meta");
assert_eq!(fm.meta_ver, 2);
assert_eq!(fm.versions.len(), 1);
assert_eq!(fm.versions[0].header.version_type, VersionType::Object);
let fi = fm
.into_fileinfo("config", "format.json", "", true, false, true)
.expect("Failed to extract file info from legacy issue #2265 config xl.meta");
assert_eq!(fi.size, 74);
assert_eq!(fi.num_versions, 1);
assert_eq!(fi.metadata.get("etag").map(String::as_str), Some("12b368ce52e496e61ac47b366c7c3b66"));
assert_eq!(
fi.data_dir.map(|id| id.to_string()).as_deref(),
Some("fba8e4c3-3f42-4242-94e0-5ab84b83ae97")
);
assert!(fi.uses_legacy_checksum);
assert!(fi.is_latest);
}
#[test]
fn test_legacy_v1_object_xlmeta_compatibility() {
let data = create_legacy_v1_object_xlmeta().expect("Failed to create legacy v1 object xl.meta");
let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap();
assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 1, 1));
let fm = FileMeta::load(&data).expect("Failed to parse legacy v1 object xl.meta");
assert_eq!(fm.meta_ver, 1);
assert_eq!(fm.versions.len(), 1);
assert_eq!(fm.versions[0].header.version_type, VersionType::Legacy);
assert_eq!(fm.versions[0].header.ec_n, 0);
assert_eq!(fm.versions[0].header.ec_m, 0);
let fi = fm
.into_fileinfo("bucket", "hello.txt", "", true, false, true)
.expect("Failed to extract file info from legacy v1 object xl.meta");
assert_eq!(fi.size, 11);
assert_eq!(fi.num_versions, 1);
assert_eq!(fi.mode, Some(0o644));
assert_eq!(fi.parts.len(), 1);
assert_eq!(fi.parts[0].etag, "etag-1");
assert_eq!(fi.parts[0].size, 11);
assert_eq!(fi.erasure.data_blocks, 4);
assert_eq!(fi.erasure.parity_blocks, 2);
assert_eq!(fi.metadata.get("content-type").map(String::as_str), Some("text/plain"));
assert!(fi.is_latest);
}
#[test]
fn test_complex_xlmeta_handling() {
// Test complex xl.meta files with many versions
@@ -1136,6 +1238,7 @@ mod test {
// Exercise creation and handling of Legacy versions
let legacy_version = FileMetaVersion {
version_type: VersionType::Legacy,
legacy_object: None,
object: None,
delete_marker: None,
write_version: 1,
@@ -1235,6 +1338,7 @@ mod test {
let mut fm = FileMeta::new();
let version = FileMetaVersion {
version_type: VersionType::Object,
legacy_object: None,
object: Some(MetaObject {
version_id: None, // Empty version ID
data_dir: None,
@@ -1468,6 +1572,7 @@ mod test {
let delete_version = FileMetaVersion {
version_type: VersionType::Delete,
legacy_object: None,
object: None,
delete_marker: Some(delete_marker),
write_version: (i + 100) as u64,
+4 -4
View File
@@ -170,7 +170,7 @@ impl FileMeta {
// Parse meta
if !meta.is_empty() {
let (versions_len, _, meta_ver, meta) = Self::decode_xl_headers(meta).map_err(|e| {
let (versions_len, header_ver, meta_ver, meta) = Self::decode_xl_headers(meta).map_err(|e| {
error!("failed to decode XL headers: {}", e);
e
})?;
@@ -193,7 +193,7 @@ impl FileMeta {
cur.read_exact(&mut header_buf)?;
let mut ver = FileMetaShallowVersion::default();
ver.header.unmarshal_msg(&header_buf).map_err(|e| {
ver.header.unmarshal_v(header_ver, &header_buf).map_err(|e| {
error!("failed to unmarshal version header: {}", e);
e
})?;
@@ -267,7 +267,7 @@ impl FileMeta {
pub fn is_latest_delete_marker(buf: &[u8]) -> bool {
let header = Self::decode_xl_headers(buf).ok();
if let Some((versions, _hdr_v, _meta_v, meta)) = header {
if let Some((versions, hdr_v, _meta_v, meta)) = header {
if versions == 0 {
return false;
}
@@ -276,7 +276,7 @@ impl FileMeta {
let _ = Self::decode_versions(meta, versions, |_: usize, hdr: &[u8], _: &[u8]| {
let mut header = FileMetaVersionHeader::default();
if header.unmarshal_msg(hdr).is_err() {
if header.unmarshal_v(hdr_v, hdr).is_err() {
return Err(Error::DoneForNow);
}
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -877,7 +877,6 @@ mod tests {
use crate::{FileMetaVersion, MetaDeleteMarker};
use std::collections::HashMap;
use std::io::Cursor;
use time::OffsetDateTime;
use uuid::Uuid;
#[tokio::test]
@@ -921,6 +920,7 @@ mod tests {
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp")),
meta_sys: HashMap::new(),
}),
legacy_object: None,
write_version: 99,
uses_legacy_checksum: false,
};
+175
View File
@@ -16,6 +16,10 @@ use crate::{ChecksumAlgo, FileMeta, FileMetaShallowVersion, FileMetaVersion, Met
use std::collections::HashMap;
use time::OffsetDateTime;
use uuid::Uuid;
use xxhash_rust::xxh64;
const MSGPACK_EXT8: u8 = 0xc7;
const MSGPACK_TIME_EXT_LEGACY: i8 = 5;
/// Create real xl.meta file data for testing
pub fn create_real_xlmeta() -> Result<Vec<u8>> {
@@ -53,6 +57,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
let file_version = FileMetaVersion {
version_type: VersionType::Object,
legacy_object: None,
object: Some(object_version),
delete_marker: None,
write_version: 1,
@@ -72,6 +77,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
let delete_file_version = FileMetaVersion {
version_type: VersionType::Delete,
legacy_object: None,
object: None,
delete_marker: Some(delete_marker),
write_version: 2,
@@ -85,6 +91,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
let legacy_version_id = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")?;
let legacy_version = FileMetaVersion {
version_type: VersionType::Legacy,
legacy_object: None,
object: None,
delete_marker: None,
write_version: 3,
@@ -102,6 +109,171 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
fm.marshal_msg()
}
fn decode_hex_fixture(input: &str) -> Result<Vec<u8>> {
let input = input.trim();
if !input.len().is_multiple_of(2) {
return Err(crate::Error::other("hex fixture must have even length"));
}
let mut out = Vec::with_capacity(input.len() / 2);
let bytes = input.as_bytes();
for idx in (0..bytes.len()).step_by(2) {
let hi = (bytes[idx] as char)
.to_digit(16)
.ok_or_else(|| crate::Error::other(format!("invalid hex at index {idx}")))?;
let lo = (bytes[idx + 1] as char)
.to_digit(16)
.ok_or_else(|| crate::Error::other(format!("invalid hex at index {}", idx + 1)))?;
out.push(((hi << 4) | lo) as u8);
}
Ok(out)
}
/// Real legacy xl.meta captured in issue #2288. Header/meta versions are 2/1.
pub fn create_issue_2288_legacy_xlmeta() -> Result<Vec<u8>> {
decode_hex_fixture(include_str!("../tests/fixtures/issue_2288_legacy_xlmeta.hex"))
}
/// Legacy xl.meta captured in issue #2265. Header/meta versions are 3/2.
pub fn create_issue_2265_legacy_meta_v2_object_xlmeta() -> Result<Vec<u8>> {
decode_hex_fixture(include_str!("../tests/fixtures/issue_2265_legacy_meta_v2_object.hex"))
}
/// Legacy config xl.meta captured in issue #2265. Header/meta versions are 3/2.
pub fn create_issue_2265_legacy_meta_v2_config_xlmeta() -> Result<Vec<u8>> {
decode_hex_fixture(include_str!("../tests/fixtures/issue_2265_legacy_meta_v2_config.hex"))
}
fn write_legacy_time(wr: &mut Vec<u8>, ts: OffsetDateTime) {
wr.push(MSGPACK_EXT8);
wr.push(12);
wr.push(MSGPACK_TIME_EXT_LEGACY as u8);
wr.extend_from_slice(&ts.unix_timestamp().to_be_bytes());
wr.extend_from_slice(&ts.nanosecond().to_be_bytes());
}
fn encode_legacy_v1_header(version_id: Uuid, mod_time: OffsetDateTime) -> Vec<u8> {
let mut wr = Vec::new();
rmp::encode::write_array_len(&mut wr, 4).unwrap();
rmp::encode::write_bin(&mut wr, version_id.as_bytes()).unwrap();
rmp::encode::write_i64(&mut wr, mod_time.unix_timestamp_nanos() as i64).unwrap();
rmp::encode::write_uint8(&mut wr, VersionType::Legacy.to_u8()).unwrap();
rmp::encode::write_uint8(&mut wr, 0).unwrap();
wr
}
fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateTime) -> Vec<u8> {
let mut wr = Vec::new();
rmp::encode::write_map_len(&mut wr, 3).unwrap();
rmp::encode::write_str(&mut wr, "Type").unwrap();
rmp::encode::write_uint8(&mut wr, VersionType::Legacy.to_u8()).unwrap();
rmp::encode::write_str(&mut wr, "V1Obj").unwrap();
rmp::encode::write_map_len(&mut wr, 8).unwrap();
rmp::encode::write_str(&mut wr, "Version").unwrap();
rmp::encode::write_str(&mut wr, "1.0.1").unwrap();
rmp::encode::write_str(&mut wr, "Format").unwrap();
rmp::encode::write_str(&mut wr, "xl").unwrap();
rmp::encode::write_str(&mut wr, "Stat").unwrap();
rmp::encode::write_map_len(&mut wr, 5).unwrap();
rmp::encode::write_str(&mut wr, "Size").unwrap();
rmp::encode::write_sint(&mut wr, 11).unwrap();
rmp::encode::write_str(&mut wr, "ModTime").unwrap();
write_legacy_time(&mut wr, mod_time);
rmp::encode::write_str(&mut wr, "Name").unwrap();
rmp::encode::write_str(&mut wr, "hello.txt").unwrap();
rmp::encode::write_str(&mut wr, "Dir").unwrap();
rmp::encode::write_bool(&mut wr, false).unwrap();
rmp::encode::write_str(&mut wr, "Mode").unwrap();
rmp::encode::write_u32(&mut wr, 0o644).unwrap();
rmp::encode::write_str(&mut wr, "Erasure").unwrap();
rmp::encode::write_map_len(&mut wr, 7).unwrap();
rmp::encode::write_str(&mut wr, "Algorithm").unwrap();
rmp::encode::write_str(&mut wr, "ReedSolomon").unwrap();
rmp::encode::write_str(&mut wr, "DataBlocks").unwrap();
rmp::encode::write_sint(&mut wr, 4).unwrap();
rmp::encode::write_str(&mut wr, "ParityBlocks").unwrap();
rmp::encode::write_sint(&mut wr, 2).unwrap();
rmp::encode::write_str(&mut wr, "BlockSize").unwrap();
rmp::encode::write_sint(&mut wr, 1_048_576).unwrap();
rmp::encode::write_str(&mut wr, "Index").unwrap();
rmp::encode::write_sint(&mut wr, 1).unwrap();
rmp::encode::write_str(&mut wr, "Distribution").unwrap();
rmp::encode::write_array_len(&mut wr, 6).unwrap();
for value in 1..=6 {
rmp::encode::write_sint(&mut wr, value).unwrap();
}
rmp::encode::write_str(&mut wr, "Checksums").unwrap();
rmp::encode::write_array_len(&mut wr, 0).unwrap();
rmp::encode::write_str(&mut wr, "Meta").unwrap();
rmp::encode::write_map_len(&mut wr, 1).unwrap();
rmp::encode::write_str(&mut wr, "content-type").unwrap();
rmp::encode::write_str(&mut wr, "text/plain").unwrap();
rmp::encode::write_str(&mut wr, "Parts").unwrap();
rmp::encode::write_array_len(&mut wr, 1).unwrap();
rmp::encode::write_map_len(&mut wr, 5).unwrap();
rmp::encode::write_str(&mut wr, "e").unwrap();
rmp::encode::write_str(&mut wr, "etag-1").unwrap();
rmp::encode::write_str(&mut wr, "n").unwrap();
rmp::encode::write_sint(&mut wr, 1).unwrap();
rmp::encode::write_str(&mut wr, "s").unwrap();
rmp::encode::write_sint(&mut wr, 11).unwrap();
rmp::encode::write_str(&mut wr, "as").unwrap();
rmp::encode::write_sint(&mut wr, 11).unwrap();
rmp::encode::write_str(&mut wr, "mt").unwrap();
write_legacy_time(&mut wr, mod_time);
rmp::encode::write_str(&mut wr, "VersionID").unwrap();
rmp::encode::write_str(&mut wr, &version_id.to_string()).unwrap();
rmp::encode::write_str(&mut wr, "DataDir").unwrap();
rmp::encode::write_str(&mut wr, &data_dir.to_string()).unwrap();
rmp::encode::write_str(&mut wr, "v").unwrap();
rmp::encode::write_uint(&mut wr, 1).unwrap();
wr
}
/// Legacy xl.meta with a V1Obj body and v1 header layout.
pub fn create_legacy_v1_object_xlmeta() -> Result<Vec<u8>> {
let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef")?;
let data_dir = Uuid::parse_str("fedcba98-7654-3210-fedc-ba9876543210")?;
let mod_time = OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789)?;
let header = encode_legacy_v1_header(version_id, mod_time);
let body = encode_legacy_v1_body(version_id, data_dir, mod_time);
let mut wr = Vec::new();
wr.extend_from_slice(b"XL2 ");
wr.extend_from_slice(&1u16.to_le_bytes());
wr.extend_from_slice(&3u16.to_le_bytes());
wr.extend_from_slice(&[0xc6, 0, 0, 0, 0]);
let offset = wr.len();
rmp::encode::write_uint(&mut wr, 1).unwrap();
rmp::encode::write_uint(&mut wr, 1).unwrap();
rmp::encode::write_sint(&mut wr, 1).unwrap();
rmp::encode::write_bin(&mut wr, &header).unwrap();
rmp::encode::write_bin(&mut wr, &body).unwrap();
let data_len = (wr.len() - offset) as u32;
wr[offset - 4..offset].copy_from_slice(&data_len.to_be_bytes());
let crc = xxh64::xxh64(&wr[offset..], 0) as u32;
wr.push(0xce);
wr.extend_from_slice(&crc.to_be_bytes());
Ok(wr)
}
/// Create a complex xl.meta file with multiple versions
pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
let mut fm = FileMeta::new();
@@ -139,6 +311,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
let file_version = FileMetaVersion {
version_type: VersionType::Object,
legacy_object: None,
object: Some(object_version),
delete_marker: None,
write_version: (i + 1) as u64,
@@ -159,6 +332,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
let delete_file_version = FileMetaVersion {
version_type: VersionType::Delete,
legacy_object: None,
object: None,
delete_marker: Some(delete_marker),
write_version: (i + 100) as u64,
@@ -247,6 +421,7 @@ pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
let file_version = FileMetaVersion {
version_type: VersionType::Object,
legacy_object: None,
object: Some(object_version),
delete_marker: None,
write_version: 1,
@@ -0,0 +1 @@
584c322001000300c600000109030201c42697c41000000000000000000000000000000000d31897f82d1d10fc3cc4040000000001040001c4dc94a64f626a656374dc0012c41000000000000000000000000000000000c410fba8e4c33f42424294e05ab84b83ae97ab52656564536f6c6f6d6f6e0100ce00100000019101ab4869676877617948617368910191d9203132623336386365353265343936653631616334376233363663376333623636914a914a91c4004a99cd07ea3a02040bce024aae3c00000081bd782d7275737466732d696e7465726e616c2d696e6c696e652d64617461947472756581a465746167d9203132623336386365353265343936653631616334376233363663376333623636c000ceb7d5e4230181d92430303030303030302d303030302d303030302d303030302d303030303030303030303030c46a6716a562ea6255d3b382987387348bd11ac5bc3e7e74b676822409cc7c4b6838301b1e0508abfe45de4ff0686b14c67ae4e7942b2f4840e2cefc26d221b96462025b3bc46a873ddff156aab76a0026a0566899426461bab906db7e25bda83ebf4989a602475c5c646517
@@ -0,0 +1 @@
584c322001000300c60000010f030201c42697c41000000000000000000000000000000000d318909e7051d94eecc4040000000001040001c4e294a64f626a656374dc0012c41000000000000000000000000000000000c41004bee19a6eea40c496fd1f39257fcbdcab52656564536f6c6f6d6f6e0100ce00100000019101ab4869676877617948617368910191d920343335393430343631386533326130626438393434653966663638303266353391cd02c391cd02c391c400cd02c399cd07ea2203143ace2fe1caec00000081bd782d7275737466732d696e7465726e616c2d696e6c696e652d64617461947472756581a465746167d9203433353934303436313865333261306264383934346539666636383032663533c000ced691a4040181d92430303030303030302d303030302d303030302d303030302d303030303030303030303030c502e430510cff66292f00568a122e173ac30926f45f75f16b17093948f03a80e0a81101000100de0019a44e616d65a3616161a74372656174656499cd07ea2203143ace2fd25e18000000ab4c6f636b456e61626c6564c2b0506f6c696379436f6e6669674a736f6e90b54e6f74696669636174696f6e436f6e666967586d6c90b24c6966656379636c65436f6e666967586d6c90b34f626a6563744c6f636b436f6e666967586d6c90b356657273696f6e696e67436f6e666967586d6c90b3456e6372797074696f6e436f6e666967586d6c90b054616767696e67436f6e666967586d6c90af51756f7461436f6e6669674a736f6e90b45265706c69636174696f6e436f6e666967586d6c90b74275636b657454617267657473436f6e6669674a736f6e90bb4275636b657454617267657473436f6e6669674d6574614a736f6e90b5506f6c696379436f6e66696755706461746564417499cd07b20100000000000000b94f626a6563744c6f636b436f6e66696755706461746564417499cd07b20100000000000000b9456e6372797074696f6e436f6e66696755706461746564417499cd07b20100000000000000b654616767696e67436f6e66696755706461746564417499cd07b20100000000000000b451756f7461436f6e66696755706461746564417499cd07b20100000000000000ba5265706c69636174696f6e436f6e66696755706461746564417499cd07b20100000000000000b956657273696f6e696e67436f6e66696755706461746564417499cd07b20100000000000000b84c6966656379636c65436f6e66696755706461746564417499cd07b20100000000000000bb4e6f74696669636174696f6e436f6e66696755706461746564417499cd07b20100000000000000bc4275636b657454617267657473436f6e66696755706461746564417499cd07b20100000000000000d9204275636b657454617267657473436f6e6669674d65746155706461746564417499cd07b2010000000000000000
@@ -0,0 +1 @@
584c322001000300c600000170020101c42495c41000000000000000000000000000000000d318774589cc776b92c40496334c780106c5014483a45479706501a556324f626ade0011a24944c41000000000000000000000000000000000a444446972c410241e46ee9fcc4df683f3eaa223d8619ca64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a65739123aa506172744153697a65739123a453697a6523a54d54696d65d318774589cc776b92a74d65746153797381bc782d6d696e696f2d696e7465726e616c2d696e6c696e652d64617461c40474727565a74d65746155737282ac636f6e74656e742d74797065b86170706c69636174696f6e2f6f637465742d73747265616da465746167d9203139616134663235356166303132376462386364346638646439326330636537a176ce6356dadbcec50305b70181a46e756c6cc44314603cf986d48aad8d01014bac476d50996dc09f7d257fdede95430b21b24d5248656c6c6f2066726f6d204d696e494f20533320766961204d6964646c657761726521