From 5e21c398f571228e7be7ad30d82dbb0f7e3e5a1d Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 27 Mar 2026 13:42:06 +0800 Subject: [PATCH] fix(filemeta): support legacy xl.meta compatibility (#2304) --- crates/filemeta/src/filemeta.rs | 105 ++ crates/filemeta/src/filemeta/codec.rs | 8 +- crates/filemeta/src/filemeta/version.rs | 931 +++++++++++++++++- crates/filemeta/src/metacache.rs | 2 +- crates/filemeta/src/test_data.rs | 175 ++++ .../issue_2265_legacy_meta_v2_config.hex | 1 + .../issue_2265_legacy_meta_v2_object.hex | 1 + .../fixtures/issue_2288_legacy_xlmeta.hex | 1 + 8 files changed, 1205 insertions(+), 19 deletions(-) create mode 100644 crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_config.hex create mode 100644 crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_object.hex create mode 100644 crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 8213aa694..2ed9d8e1f 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -1055,6 +1055,108 @@ mod test { assert_eq!(stats.invalid_versions, 1); // Legacy is counted as invalid } + #[test] + fn test_issue_2288_legacy_xlmeta_compatibility() { + let data = create_issue_2288_legacy_xlmeta().expect("Failed to load issue #2288 fixture"); + let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap(); + assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 2, 1)); + + let fm = FileMeta::load(&data).expect("Failed to parse legacy issue #2288 xl.meta"); + assert_eq!(fm.meta_ver, 1); + assert_eq!(fm.versions.len(), 1); + assert_eq!(fm.versions[0].header.version_type, VersionType::Object); + assert_eq!(fm.versions[0].header.signature, [0x96, 0x33, 0x4c, 0x78]); + assert_eq!(fm.versions[0].header.ec_n, 0); + assert_eq!(fm.versions[0].header.ec_m, 0); + + let fi = fm + .into_fileinfo("viscom", "test.txt", "", true, false, true) + .expect("Failed to extract file info from legacy issue #2288 xl.meta"); + assert_eq!(fi.size, 35); + assert_eq!(fi.num_versions, 1); + assert!(fi.is_latest); + } + + #[test] + fn test_issue_2265_legacy_meta_v2_object_compatibility() { + let data = create_issue_2265_legacy_meta_v2_object_xlmeta().expect("Failed to load issue #2265 object fixture"); + let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap(); + assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 3, 2)); + + let fm = FileMeta::load(&data).expect("Failed to parse legacy issue #2265 object xl.meta"); + assert_eq!(fm.meta_ver, 2); + assert_eq!(fm.versions.len(), 1); + assert_eq!(fm.versions[0].header.version_type, VersionType::Object); + assert_eq!(fm.versions[0].header.ec_n, 0); + assert_eq!(fm.versions[0].header.ec_m, 1); + + let fi = fm + .into_fileinfo("bucket", ".metadata.bin", "", true, false, true) + .expect("Failed to extract file info from legacy issue #2265 object xl.meta"); + assert_eq!(fi.size, 707); + assert_eq!(fi.num_versions, 1); + assert_eq!(fi.metadata.get("etag").map(String::as_str), Some("4359404618e32a0bd8944e9ff6802f53")); + assert_eq!( + fi.data_dir.map(|id| id.to_string()).as_deref(), + Some("04bee19a-6eea-40c4-96fd-1f39257fcbdc") + ); + assert!(fi.uses_legacy_checksum); + assert!(fi.is_latest); + } + + #[test] + fn test_issue_2265_legacy_meta_v2_config_compatibility() { + let data = create_issue_2265_legacy_meta_v2_config_xlmeta().expect("Failed to load issue #2265 config fixture"); + let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap(); + assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 3, 2)); + + let fm = FileMeta::load(&data).expect("Failed to parse legacy issue #2265 config xl.meta"); + assert_eq!(fm.meta_ver, 2); + assert_eq!(fm.versions.len(), 1); + assert_eq!(fm.versions[0].header.version_type, VersionType::Object); + + let fi = fm + .into_fileinfo("config", "format.json", "", true, false, true) + .expect("Failed to extract file info from legacy issue #2265 config xl.meta"); + assert_eq!(fi.size, 74); + assert_eq!(fi.num_versions, 1); + assert_eq!(fi.metadata.get("etag").map(String::as_str), Some("12b368ce52e496e61ac47b366c7c3b66")); + assert_eq!( + fi.data_dir.map(|id| id.to_string()).as_deref(), + Some("fba8e4c3-3f42-4242-94e0-5ab84b83ae97") + ); + assert!(fi.uses_legacy_checksum); + assert!(fi.is_latest); + } + + #[test] + fn test_legacy_v1_object_xlmeta_compatibility() { + let data = create_legacy_v1_object_xlmeta().expect("Failed to create legacy v1 object xl.meta"); + let (major, minor, header_ver, meta_ver) = FileMeta::read_format_versions(&data).unwrap(); + assert_eq!((major, minor, header_ver, meta_ver), (1, 3, 1, 1)); + + let fm = FileMeta::load(&data).expect("Failed to parse legacy v1 object xl.meta"); + assert_eq!(fm.meta_ver, 1); + assert_eq!(fm.versions.len(), 1); + assert_eq!(fm.versions[0].header.version_type, VersionType::Legacy); + assert_eq!(fm.versions[0].header.ec_n, 0); + assert_eq!(fm.versions[0].header.ec_m, 0); + + let fi = fm + .into_fileinfo("bucket", "hello.txt", "", true, false, true) + .expect("Failed to extract file info from legacy v1 object xl.meta"); + assert_eq!(fi.size, 11); + assert_eq!(fi.num_versions, 1); + assert_eq!(fi.mode, Some(0o644)); + assert_eq!(fi.parts.len(), 1); + assert_eq!(fi.parts[0].etag, "etag-1"); + assert_eq!(fi.parts[0].size, 11); + assert_eq!(fi.erasure.data_blocks, 4); + assert_eq!(fi.erasure.parity_blocks, 2); + assert_eq!(fi.metadata.get("content-type").map(String::as_str), Some("text/plain")); + assert!(fi.is_latest); + } + #[test] fn test_complex_xlmeta_handling() { // Test complex xl.meta files with many versions @@ -1136,6 +1238,7 @@ mod test { // Exercise creation and handling of Legacy versions let legacy_version = FileMetaVersion { version_type: VersionType::Legacy, + legacy_object: None, object: None, delete_marker: None, write_version: 1, @@ -1235,6 +1338,7 @@ mod test { let mut fm = FileMeta::new(); let version = FileMetaVersion { version_type: VersionType::Object, + legacy_object: None, object: Some(MetaObject { version_id: None, // Empty version ID data_dir: None, @@ -1468,6 +1572,7 @@ mod test { let delete_version = FileMetaVersion { version_type: VersionType::Delete, + legacy_object: None, object: None, delete_marker: Some(delete_marker), write_version: (i + 100) as u64, diff --git a/crates/filemeta/src/filemeta/codec.rs b/crates/filemeta/src/filemeta/codec.rs index 164d67fc4..c41f3ba3b 100644 --- a/crates/filemeta/src/filemeta/codec.rs +++ b/crates/filemeta/src/filemeta/codec.rs @@ -170,7 +170,7 @@ impl FileMeta { // Parse meta if !meta.is_empty() { - let (versions_len, _, meta_ver, meta) = Self::decode_xl_headers(meta).map_err(|e| { + let (versions_len, header_ver, meta_ver, meta) = Self::decode_xl_headers(meta).map_err(|e| { error!("failed to decode XL headers: {}", e); e })?; @@ -193,7 +193,7 @@ impl FileMeta { cur.read_exact(&mut header_buf)?; let mut ver = FileMetaShallowVersion::default(); - ver.header.unmarshal_msg(&header_buf).map_err(|e| { + ver.header.unmarshal_v(header_ver, &header_buf).map_err(|e| { error!("failed to unmarshal version header: {}", e); e })?; @@ -267,7 +267,7 @@ impl FileMeta { pub fn is_latest_delete_marker(buf: &[u8]) -> bool { let header = Self::decode_xl_headers(buf).ok(); - if let Some((versions, _hdr_v, _meta_v, meta)) = header { + if let Some((versions, hdr_v, _meta_v, meta)) = header { if versions == 0 { return false; } @@ -276,7 +276,7 @@ impl FileMeta { let _ = Self::decode_versions(meta, versions, |_: usize, hdr: &[u8], _: &[u8]| { let mut header = FileMetaVersionHeader::default(); - if header.unmarshal_msg(hdr).is_err() { + if header.unmarshal_v(hdr_v, hdr).is_err() { return Err(Error::DoneForNow); } diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 0cd344d45..11e716f14 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -24,12 +24,188 @@ use super::msgp_decode::{PrependByteReader, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value}; use super::*; +use crate::ChecksumInfo; +use rustfs_utils::HashAlgorithm; use rustfs_utils::http::{ SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, contains_key_bytes, get_bytes, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix, }; +const MSGPACK_EXT8: u8 = 0xc7; +const MSGPACK_EXT16: u8 = 0xc8; +const MSGPACK_EXT32: u8 = 0xc9; +const MSGPACK_FIXEXT4: u8 = 0xd6; +const MSGPACK_FIXEXT8: u8 = 0xd7; +const MSGPACK_TIME_EXT_LEGACY: i8 = 5; +const MSGPACK_TIME_EXT_OFFICIAL: i8 = -1; + +fn read_msgp_string(rd: &mut R) -> Result { + let len = rmp::decode::read_str_len(rd)? as usize; + let mut buf = vec![0u8; len]; + rd.read_exact(&mut buf)?; + Ok(String::from_utf8(buf)?) +} + +fn read_msgp_bin(rd: &mut R) -> Result> { + let len = rmp::decode::read_bin_len(rd)? as usize; + let mut buf = vec![0u8; len]; + rd.read_exact(&mut buf)?; + Ok(buf) +} + +fn decode_msgp_time_payload(ext_type: i8, payload: &[u8]) -> Result { + let (secs, nanos) = match (ext_type, payload.len()) { + (MSGPACK_TIME_EXT_LEGACY, 12) => { + let secs = i64::from_be_bytes(payload[..8].try_into().unwrap()); + let nanos = u32::from_be_bytes(payload[8..12].try_into().unwrap()); + (secs, nanos) + } + (MSGPACK_TIME_EXT_OFFICIAL, 4) => (u32::from_be_bytes(payload.try_into().unwrap()) as i64, 0), + (MSGPACK_TIME_EXT_OFFICIAL, 8) => { + let v = u64::from_be_bytes(payload.try_into().unwrap()); + let nanos = (v >> 34) as u32; + let secs = (v & ((1 << 34) - 1)) as i64; + (secs, nanos) + } + (MSGPACK_TIME_EXT_OFFICIAL, 12) => { + let nanos = u32::from_be_bytes(payload[..4].try_into().unwrap()); + let secs = i64::from_be_bytes(payload[4..12].try_into().unwrap()); + (secs, nanos) + } + _ => { + return Err(Error::other(format!( + "unsupported msgpack time ext type {ext_type} len {}", + payload.len() + ))); + } + }; + + if nanos > 999_999_999 { + return Err(Error::other(format!("invalid msgpack time nanos: {nanos}"))); + } + + OffsetDateTime::from_unix_timestamp_nanos(secs as i128 * 1_000_000_000 + nanos as i128).map_err(Error::from) +} + +fn read_msgp_time(rd: &mut R) -> Result { + let mut tag = [0u8; 1]; + rd.read_exact(&mut tag)?; + + let (len, ext_type) = match tag[0] { + MSGPACK_FIXEXT4 => { + let mut typ = [0u8; 1]; + rd.read_exact(&mut typ)?; + (4usize, typ[0] as i8) + } + MSGPACK_FIXEXT8 => { + let mut typ = [0u8; 1]; + rd.read_exact(&mut typ)?; + (8usize, typ[0] as i8) + } + MSGPACK_EXT8 => { + let mut len = [0u8; 1]; + let mut typ = [0u8; 1]; + rd.read_exact(&mut len)?; + rd.read_exact(&mut typ)?; + (len[0] as usize, typ[0] as i8) + } + MSGPACK_EXT16 => { + let mut len = [0u8; 2]; + let mut typ = [0u8; 1]; + rd.read_exact(&mut len)?; + rd.read_exact(&mut typ)?; + (u16::from_be_bytes(len) as usize, typ[0] as i8) + } + MSGPACK_EXT32 => { + let mut len = [0u8; 4]; + let mut typ = [0u8; 1]; + rd.read_exact(&mut len)?; + rd.read_exact(&mut typ)?; + (u32::from_be_bytes(len) as usize, typ[0] as i8) + } + other => return Err(Error::other(format!("unsupported msgpack time marker: 0x{other:02x}"))), + }; + + let mut payload = vec![0u8; len]; + rd.read_exact(&mut payload)?; + decode_msgp_time_payload(ext_type, &payload) +} + +fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result> { + if bytes.is_empty() { + return Ok(None); + } + + if bytes.len() != 16 { + return Err(Error::other(format!("legacy {field} must be 16 bytes, got {}", bytes.len()))); + } + + let id = Uuid::from_slice(bytes).map_err(Error::from)?; + Ok((!id.is_nil()).then_some(id)) +} + +fn parse_legacy_erasure_algo(value: &str) -> ErasureAlgo { + match value { + "ReedSolomon" => ErasureAlgo::ReedSolomon, + _ => ErasureAlgo::Invalid, + } +} + +fn parse_legacy_checksum_algo(value: &str) -> ChecksumAlgo { + match value { + "HighwayHash" => ChecksumAlgo::HighwayHash, + _ => ChecksumAlgo::Invalid, + } +} + +#[derive(Debug, Deserialize)] +enum LegacyMetaV2VersionType { + #[serde(rename = "Object")] + Object, + #[serde(rename = "Delete")] + Delete, + #[serde(rename = "DeleteMarker")] + DeleteMarker, +} + +#[derive(Debug, Deserialize)] +struct LegacyMetaV2Version { + version_type: LegacyMetaV2VersionType, + object: Option, + delete_marker: Option, + write_version: u64, +} + +#[derive(Debug, Deserialize)] +struct LegacyMetaV2Object { + version_id: Vec, + data_dir: Vec, + erasure_algorithm: String, + erasure_m: usize, + erasure_n: usize, + erasure_block_size: usize, + erasure_index: usize, + erasure_dist: Vec, + bitrot_checksum_algo: String, + part_numbers: Vec, + part_etags: Vec, + part_sizes: Vec, + part_actual_sizes: Vec, + part_indices: Vec>, + size: i64, + mod_time: Option, + meta_sys: HashMap>, + meta_user: HashMap, +} + +#[derive(Debug, Deserialize)] +struct LegacyMetaV2DeleteMarker { + version_id: Vec, + mod_time: Option, + meta_sys: HashMap>, +} + #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, Eq, PartialOrd, Ord)] pub struct FileMetaShallowVersion { pub header: FileMetaVersionHeader, @@ -63,6 +239,8 @@ impl TryFrom for FileMetaShallowVersion { pub struct FileMetaVersion { #[serde(rename = "Type")] pub version_type: VersionType, + #[serde(rename = "V1Obj")] + pub legacy_object: Option, #[serde(rename = "V2Obj")] pub object: Option, #[serde(rename = "DelObj")] @@ -86,6 +264,7 @@ impl FileMetaVersion { .as_ref() .map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some()) .unwrap_or_default(), + VersionType::Legacy => self.legacy_object.as_ref().map(MetaObjectV1::valid).unwrap_or_default(), VersionType::Delete => self .delete_marker .as_ref() @@ -113,7 +292,8 @@ impl FileMetaVersion { match self.version_type { VersionType::Object => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(), VersionType::Delete => self.delete_marker.as_ref().map(|v| v.version_id).unwrap_or_default(), - _ => None, + VersionType::Legacy => self.legacy_object.as_ref().and_then(MetaObjectV1::version_id), + VersionType::Invalid => None, } } @@ -121,16 +301,14 @@ impl FileMetaVersion { match self.version_type { VersionType::Object => self.object.as_ref().map(|v| v.mod_time).unwrap_or_default(), VersionType::Delete => self.delete_marker.as_ref().map(|v| v.mod_time).unwrap_or_default(), - _ => None, + VersionType::Legacy => self.legacy_object.as_ref().and_then(|v| v.stat.mod_time), + VersionType::Invalid => None, } } // decode_data_dir_from_meta reads data_dir from meta TODO: directly parse only data_dir from meta buf, msg.skip pub fn decode_data_dir_from_meta(buf: &[u8]) -> Result> { - let mut ver = Self::default(); - ver.decode_from(&mut std::io::Cursor::new(buf))?; - let data_dir = ver.object.map(|v| v.data_dir).unwrap_or_default(); - Ok(data_dir) + Ok(Self::try_from(buf)?.get_data_dir()) } pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { @@ -151,15 +329,18 @@ impl FileMetaVersion { self.version_type = VersionType::from_u8(v as u8); } "V1Obj" => { - // Skip V1Obj (legacy), not supported let mut buf = [0u8; 1]; rd.read_exact(&mut buf).map_err(Error::from)?; - if buf[0] != 0xc0 { + if buf[0] == 0xc0 { + self.legacy_object = None; + } else { let mut prepend = PrependByteReader { byte: Some(buf[0]), inner: rd, }; - skip_msgp_value(&mut prepend)?; + let mut obj = MetaObjectV1::default(); + obj.decode_from(&mut prepend)?; + self.legacy_object = Some(obj); } } "V2Obj" => { @@ -266,11 +447,17 @@ impl FileMetaVersion { pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> FileInfo { let mut fi = match self.version_type { - VersionType::Invalid | VersionType::Legacy => FileInfo { - name: path.to_string(), - volume: volume.to_string(), - ..Default::default() - }, + VersionType::Invalid | VersionType::Legacy => { + if let Some(ref legacy) = self.legacy_object { + legacy.to_fileinfo(volume, path) + } else { + FileInfo { + name: path.to_string(), + volume: volume.to_string(), + ..Default::default() + } + } + } VersionType::Object => self .object .as_ref() @@ -324,6 +511,7 @@ impl FileMetaVersion { [0; 4] } } + VersionType::Legacy => self.legacy_object.as_ref().map(MetaObjectV1::get_signature).unwrap_or([0; 4]), _ => [0; 4], } } @@ -332,6 +520,7 @@ impl FileMetaVersion { pub fn uses_data_dir(&self) -> bool { match self.version_type { VersionType::Object => self.object.as_ref().map(|obj| obj.uses_data_dir()).unwrap_or(false), + VersionType::Legacy => false, _ => false, } } @@ -340,6 +529,7 @@ impl FileMetaVersion { pub fn uses_inline_data(&self) -> bool { match self.version_type { VersionType::Object => self.object.as_ref().map(|obj| obj.inlinedata()).unwrap_or(false), + VersionType::Legacy => false, _ => false, } } @@ -354,6 +544,13 @@ impl TryFrom<&[u8]> for FileMetaVersion { ver.uses_legacy_checksum = false; return Ok(ver); } + + if let Ok(legacy_ver) = rmp_serde::from_slice::(value) { + let mut ver = FileMetaVersion::try_from(legacy_ver)?; + ver.uses_legacy_checksum = true; + return Ok(ver); + } + // Fallback for legacy ver_meta: rmp_serde format let mut ver: Self = rmp_serde::from_slice(value).map_err(Error::other)?; ver.uses_legacy_checksum = true; @@ -361,11 +558,34 @@ impl TryFrom<&[u8]> for FileMetaVersion { } } +impl TryFrom for FileMetaVersion { + type Error = Error; + + fn try_from(value: LegacyMetaV2Version) -> std::result::Result { + let (version_type, object, delete_marker) = match value.version_type { + LegacyMetaV2VersionType::Object => (VersionType::Object, value.object.map(TryInto::try_into).transpose()?, None), + LegacyMetaV2VersionType::Delete | LegacyMetaV2VersionType::DeleteMarker => { + (VersionType::Delete, None, value.delete_marker.map(TryInto::try_into).transpose()?) + } + }; + + Ok(Self { + version_type, + legacy_object: None, + object, + delete_marker, + write_version: value.write_version, + uses_legacy_checksum: true, + }) + } +} + impl From for FileMetaVersion { fn from(value: FileInfo) -> Self { if value.deleted { FileMetaVersion { version_type: VersionType::Delete, + legacy_object: None, delete_marker: Some(MetaDeleteMarker::from(value)), object: None, write_version: 0, @@ -374,6 +594,7 @@ impl From for FileMetaVersion { } else { FileMetaVersion { version_type: VersionType::Object, + legacy_object: None, delete_marker: None, object: Some(MetaObject::from(value)), write_version: 0, @@ -403,6 +624,16 @@ pub struct FileMetaVersionHeader { } impl FileMetaVersionHeader { + fn reset_for_unmarshal(&mut self) { + self.version_id = None; + self.mod_time = None; + self.signature = [0; 4]; + self.version_type = VersionType::Invalid; + self.flags = 0; + self.ec_n = 0; + self.ec_m = 0; + } + pub fn has_ec(&self) -> bool { self.ec_m > 0 && self.ec_n > 0 } @@ -499,7 +730,75 @@ impl FileMetaVersionHeader { Ok(wr) } + pub fn unmarshal_v(&mut self, version: u8, buf: &[u8]) -> Result { + match version { + 1 => self.unmarshal_v1(buf), + 2 => self.unmarshal_v2(buf), + 3 => self.unmarshal_msg(buf), + _ => Err(Error::other(format!("unknown xl header version: {version}"))), + } + } + + pub fn unmarshal_v1(&mut self, buf: &[u8]) -> Result { + self.reset_for_unmarshal(); + + let mut cur = Cursor::new(buf); + let alen = rmp::decode::read_array_len(&mut cur)?; + if alen != 4 { + return Err(Error::other(format!("version header array len err need 4 got {alen}"))); + } + + rmp::decode::read_bin_len(&mut cur)?; + let mut version_id = [0u8; 16]; + cur.read_exact(&mut version_id)?; + self.version_id = Some(Uuid::from_bytes(version_id)); + + let unix: i128 = rmp::decode::read_int(&mut cur)?; + let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; + if time != OffsetDateTime::UNIX_EPOCH { + self.mod_time = Some(time); + } + + let typ: u8 = rmp::decode::read_int(&mut cur)?; + self.version_type = VersionType::from_u8(typ); + self.flags = rmp::decode::read_int(&mut cur)?; + + Ok(cur.position()) + } + + pub fn unmarshal_v2(&mut self, buf: &[u8]) -> Result { + self.reset_for_unmarshal(); + + let mut cur = Cursor::new(buf); + let alen = rmp::decode::read_array_len(&mut cur)?; + if alen != 5 { + return Err(Error::other(format!("version header array len err need 5 got {alen}"))); + } + + rmp::decode::read_bin_len(&mut cur)?; + let mut version_id = [0u8; 16]; + cur.read_exact(&mut version_id)?; + self.version_id = Some(Uuid::from_bytes(version_id)); + + let unix: i128 = rmp::decode::read_int(&mut cur)?; + let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; + if time != OffsetDateTime::UNIX_EPOCH { + self.mod_time = Some(time); + } + + rmp::decode::read_bin_len(&mut cur)?; + cur.read_exact(&mut self.signature)?; + + let typ: u8 = rmp::decode::read_int(&mut cur)?; + self.version_type = VersionType::from_u8(typ); + self.flags = rmp::decode::read_int(&mut cur)?; + + Ok(cur.position()) + } + pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { + self.reset_for_unmarshal(); + let mut cur = Cursor::new(buf); let alen = rmp::decode::read_array_len(&mut cur)?; if alen != 7 { @@ -666,6 +965,396 @@ pub struct MetaObject { pub meta_user: HashMap, // Object version metadata set by user } +impl TryFrom for MetaObject { + type Error = Error; + + fn try_from(value: LegacyMetaV2Object) -> std::result::Result { + Ok(Self { + version_id: parse_legacy_uuid_bytes(&value.version_id, "version_id")?, + data_dir: parse_legacy_uuid_bytes(&value.data_dir, "data_dir")?, + erasure_algorithm: parse_legacy_erasure_algo(&value.erasure_algorithm), + erasure_m: value.erasure_m, + erasure_n: value.erasure_n, + erasure_block_size: value.erasure_block_size, + erasure_index: value.erasure_index, + erasure_dist: value.erasure_dist, + bitrot_checksum_algo: parse_legacy_checksum_algo(&value.bitrot_checksum_algo), + part_numbers: value.part_numbers, + part_etags: value.part_etags, + part_sizes: value.part_sizes, + part_actual_sizes: value.part_actual_sizes, + part_indices: value.part_indices.into_iter().map(Bytes::from).collect(), + size: value.size, + mod_time: value.mod_time, + meta_sys: value.meta_sys, + meta_user: value.meta_user, + }) + } +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] +pub struct MetaObjectV1 { + #[serde(rename = "Version")] + pub version: String, + #[serde(rename = "Format")] + pub format: String, + #[serde(rename = "Stat")] + pub stat: MetaObjectV1Stat, + #[serde(rename = "Erasure")] + pub erasure: MetaObjectV1Erasure, + #[serde(rename = "Meta")] + pub meta: HashMap, + #[serde(rename = "Parts")] + pub parts: Vec, + #[serde(rename = "VersionID")] + pub version_id: String, + #[serde(rename = "DataDir")] + pub data_dir: String, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] +pub struct MetaObjectV1Stat { + #[serde(rename = "Size")] + pub size: i64, + #[serde(rename = "ModTime")] + pub mod_time: Option, + #[serde(rename = "Name")] + pub name: String, + #[serde(rename = "Dir")] + pub dir: bool, + #[serde(rename = "Mode")] + pub mode: u32, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] +pub struct MetaObjectV1ChecksumInfo { + #[serde(rename = "PartNumber")] + pub part_number: usize, + #[serde(rename = "Algorithm")] + pub algorithm: String, + #[serde(rename = "Hash")] + pub hash: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] +pub struct MetaObjectV1Erasure { + #[serde(rename = "Algorithm")] + pub algorithm: String, + #[serde(rename = "DataBlocks")] + pub data_blocks: usize, + #[serde(rename = "ParityBlocks")] + pub parity_blocks: usize, + #[serde(rename = "BlockSize")] + pub block_size: usize, + #[serde(rename = "Index")] + pub index: usize, + #[serde(rename = "Distribution")] + pub distribution: Vec, + #[serde(rename = "Checksums")] + pub checksums: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] +pub struct MetaObjectV1Part { + #[serde(rename = "e")] + pub etag: String, + #[serde(rename = "n")] + pub number: usize, + #[serde(rename = "s")] + pub size: usize, + #[serde(rename = "as")] + pub actual_size: i64, + #[serde(rename = "mt")] + pub mod_time: Option, + #[serde(rename = "i")] + pub index: Option, + #[serde(rename = "crc")] + pub checksums: Option>, + #[serde(rename = "err")] + pub error: Option, +} + +impl MetaObjectV1 { + fn version_id(&self) -> Option { + if self.version_id.is_empty() { + None + } else { + Uuid::parse_str(&self.version_id).ok().filter(|id| !id.is_nil()) + } + } + + fn valid(&self) -> bool { + if self.format != "xl" || self.stat.mod_time.is_none() { + return false; + } + + let data_blocks = self.erasure.data_blocks; + let parity_blocks = self.erasure.parity_blocks; + data_blocks > 0 + && data_blocks >= parity_blocks + && self.erasure.index > 0 + && self.erasure.distribution.len() == data_blocks + parity_blocks + } + + fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = Self::default(); + + while fields > 0 { + fields -= 1; + let key = read_msgp_string(rd)?; + match key.as_str() { + "Version" => self.version = read_msgp_string(rd)?, + "Format" => self.format = read_msgp_string(rd)?, + "Stat" => self.stat.decode_from(rd)?, + "Erasure" => self.erasure.decode_from(rd)?, + "Meta" => { + let len = rmp::decode::read_map_len(rd)? as usize; + self.meta.clear(); + for _ in 0..len { + self.meta.insert(read_msgp_string(rd)?, read_msgp_string(rd)?); + } + } + "Parts" => { + let len = rmp::decode::read_array_len(rd)? as usize; + self.parts.clear(); + self.parts.reserve(len); + for _ in 0..len { + let mut part = MetaObjectV1Part::default(); + part.decode_from(rd)?; + self.parts.push(part); + } + } + "VersionID" => self.version_id = read_msgp_string(rd)?, + "DataDir" => self.data_dir = read_msgp_string(rd)?, + _ => skip_msgp_value(rd)?, + } + } + + Ok(()) + } + + fn get_signature(&self) -> [u8; 4] { + let mut hasher = xxhash_rust::xxh64::Xxh64::new(XXHASH_SEED); + hasher.update(self.version.as_bytes()); + hasher.update(self.format.as_bytes()); + hasher.update(&self.stat.size.to_le_bytes()); + hasher.update(&self.stat.mode.to_le_bytes()); + if let Some(mod_time) = self.stat.mod_time { + hasher.update(&mod_time.unix_timestamp_nanos().to_le_bytes()); + } + hasher.update(self.erasure.algorithm.as_bytes()); + hasher.update(&(self.erasure.data_blocks as u64).to_le_bytes()); + hasher.update(&(self.erasure.parity_blocks as u64).to_le_bytes()); + hasher.update(&(self.erasure.block_size as u64).to_le_bytes()); + for v in &self.erasure.distribution { + hasher.update(&(*v as u64).to_le_bytes()); + } + for checksum in &self.erasure.checksums { + hasher.update(&(checksum.part_number as u64).to_le_bytes()); + hasher.update(checksum.algorithm.as_bytes()); + hasher.update(&checksum.hash); + } + let mut meta_keys: Vec<_> = self.meta.iter().collect(); + meta_keys.sort_by(|a, b| a.0.cmp(b.0)); + for (k, v) in meta_keys { + hasher.update(k.as_bytes()); + hasher.update(v.as_bytes()); + } + for part in &self.parts { + hasher.update(&(part.number as u64).to_le_bytes()); + hasher.update(&(part.size as u64).to_le_bytes()); + hasher.update(&part.actual_size.to_le_bytes()); + hasher.update(part.etag.as_bytes()); + if let Some(mod_time) = part.mod_time { + hasher.update(&mod_time.unix_timestamp_nanos().to_le_bytes()); + } + if let Some(index) = &part.index { + hasher.update(index); + } + } + let hash = hasher.finish(); + let bytes = hash.to_le_bytes(); + [bytes[0], bytes[1], bytes[2], bytes[3]] + } + + fn to_fileinfo(&self, volume: &str, path: &str) -> FileInfo { + FileInfo { + volume: volume.to_string(), + name: path.to_string(), + version_id: self.version_id(), + mod_time: self.stat.mod_time, + size: self.stat.size, + mode: Some(self.stat.mode), + metadata: self.meta.clone(), + parts: self.parts.iter().cloned().map(Into::into).collect(), + erasure: self.erasure.clone().into(), + num_versions: 1, + data_dir: Uuid::parse_str(&self.data_dir).ok().filter(|id| !id.is_nil()), + ..Default::default() + } + } +} + +impl MetaObjectV1Stat { + fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = Self::default(); + + while fields > 0 { + fields -= 1; + let key = read_msgp_string(rd)?; + match key.as_str() { + "Size" => self.size = rmp::decode::read_int(rd)?, + "ModTime" => self.mod_time = Some(read_msgp_time(rd)?), + "Name" => self.name = read_msgp_string(rd)?, + "Dir" => self.dir = rmp::decode::read_bool(rd)?, + "Mode" => self.mode = rmp::decode::read_u32(rd)?, + _ => skip_msgp_value(rd)?, + } + } + + Ok(()) + } +} + +impl MetaObjectV1Erasure { + fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = Self::default(); + + while fields > 0 { + fields -= 1; + let key = read_msgp_string(rd)?; + match key.as_str() { + "Algorithm" => self.algorithm = read_msgp_string(rd)?, + "DataBlocks" => self.data_blocks = rmp::decode::read_int::(rd)? as usize, + "ParityBlocks" => self.parity_blocks = rmp::decode::read_int::(rd)? as usize, + "BlockSize" => self.block_size = rmp::decode::read_int::(rd)? as usize, + "Index" => self.index = rmp::decode::read_int::(rd)? as usize, + "Distribution" => { + let len = rmp::decode::read_array_len(rd)? as usize; + self.distribution.clear(); + self.distribution.reserve(len); + for _ in 0..len { + self.distribution.push(rmp::decode::read_int::(rd)? as usize); + } + } + "Checksums" => { + let len = rmp::decode::read_array_len(rd)? as usize; + self.checksums.clear(); + self.checksums.reserve(len); + for _ in 0..len { + let mut checksum = MetaObjectV1ChecksumInfo::default(); + checksum.decode_from(rd)?; + self.checksums.push(checksum); + } + } + _ => skip_msgp_value(rd)?, + } + } + + Ok(()) + } +} + +impl MetaObjectV1ChecksumInfo { + fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = Self::default(); + + while fields > 0 { + fields -= 1; + let key = read_msgp_string(rd)?; + match key.as_str() { + "PartNumber" => self.part_number = rmp::decode::read_int::(rd)? as usize, + "Algorithm" => self.algorithm = read_msgp_string(rd)?, + "Hash" => self.hash = read_msgp_bin(rd)?, + _ => skip_msgp_value(rd)?, + } + } + + Ok(()) + } +} + +impl MetaObjectV1Part { + fn decode_from(&mut self, rd: &mut R) -> Result<()> { + let mut fields = rmp::decode::read_map_len(rd)?; + *self = Self::default(); + + while fields > 0 { + fields -= 1; + let key = read_msgp_string(rd)?; + match key.as_str() { + "e" => self.etag = read_msgp_string(rd)?, + "n" => self.number = rmp::decode::read_int::(rd)? as usize, + "s" => self.size = rmp::decode::read_int::(rd)? as usize, + "as" => self.actual_size = rmp::decode::read_int(rd)?, + "mt" => self.mod_time = Some(read_msgp_time(rd)?), + "i" => self.index = Some(Bytes::from(read_msgp_bin(rd)?)), + "crc" => { + let len = rmp::decode::read_map_len(rd)? as usize; + let mut checksums = HashMap::with_capacity(len); + for _ in 0..len { + checksums.insert(read_msgp_string(rd)?, read_msgp_string(rd)?); + } + self.checksums = Some(checksums); + } + "err" => self.error = Some(read_msgp_string(rd)?), + _ => skip_msgp_value(rd)?, + } + } + + Ok(()) + } +} + +impl From for ErasureInfo { + fn from(value: MetaObjectV1Erasure) -> Self { + ErasureInfo { + algorithm: value.algorithm, + data_blocks: value.data_blocks, + parity_blocks: value.parity_blocks, + block_size: value.block_size, + index: value.index, + distribution: value.distribution, + checksums: value.checksums.into_iter().map(Into::into).collect(), + } + } +} + +impl From for ChecksumInfo { + fn from(value: MetaObjectV1ChecksumInfo) -> Self { + ChecksumInfo { + part_number: value.part_number, + algorithm: match value.algorithm.as_str() { + "sha256" => HashAlgorithm::SHA256, + "highwayhash256" => HashAlgorithm::HighwayHash256, + "highwayhash256S" => HashAlgorithm::HighwayHash256S, + "blake2b" | "blake2b512" => HashAlgorithm::BLAKE2b512, + _ => HashAlgorithm::HighwayHash256S, + }, + hash: Bytes::from(value.hash), + } + } +} + +impl From for ObjectPartInfo { + fn from(value: MetaObjectV1Part) -> Self { + ObjectPartInfo { + etag: value.etag, + number: value.number, + size: value.size, + actual_size: value.actual_size, + mod_time: value.mod_time, + index: value.index, + checksums: value.checksums, + error: value.error, + } + } +} + impl MetaObject { pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { let mut cur = std::io::Cursor::new(buf); @@ -1485,6 +2174,18 @@ pub struct MetaDeleteMarker { pub meta_sys: HashMap>, // Delete marker internal metadata } +impl TryFrom for MetaDeleteMarker { + type Error = Error; + + fn try_from(value: LegacyMetaV2DeleteMarker) -> std::result::Result { + Ok(Self { + version_id: parse_legacy_uuid_bytes(&value.version_id, "version_id")?, + mod_time: value.mod_time, + meta_sys: value.meta_sys, + }) + } +} + impl MetaDeleteMarker { pub fn free_version(&self) -> bool { contains_key_bytes(&self.meta_sys, SUFFIX_FREE_VERSION) @@ -2010,3 +2711,205 @@ pub async fn read_xl_meta_no_data(reader: &mut R, size: us ))), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_version_id() -> Uuid { + Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef").unwrap() + } + + fn sample_mod_time() -> OffsetDateTime { + OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789).unwrap() + } + + fn sample_header() -> FileMetaVersionHeader { + FileMetaVersionHeader { + version_id: Some(sample_version_id()), + mod_time: Some(sample_mod_time()), + signature: [0x96, 0x33, 0x4c, 0x78], + version_type: VersionType::Object, + flags: 0x06, + ec_n: 4, + ec_m: 2, + } + } + + fn encode_v1_header(header: &FileMetaVersionHeader) -> Vec { + let mut wr = Vec::new(); + rmp::encode::write_array_len(&mut wr, 4).unwrap(); + rmp::encode::write_bin(&mut wr, header.version_id.unwrap().as_bytes()).unwrap(); + rmp::encode::write_i64(&mut wr, header.mod_time.unwrap().unix_timestamp_nanos() as i64).unwrap(); + rmp::encode::write_uint8(&mut wr, header.version_type.to_u8()).unwrap(); + rmp::encode::write_uint8(&mut wr, header.flags).unwrap(); + wr + } + + fn encode_v2_header(header: &FileMetaVersionHeader) -> Vec { + let mut wr = Vec::new(); + rmp::encode::write_array_len(&mut wr, 5).unwrap(); + rmp::encode::write_bin(&mut wr, header.version_id.unwrap().as_bytes()).unwrap(); + rmp::encode::write_i64(&mut wr, header.mod_time.unwrap().unix_timestamp_nanos() as i64).unwrap(); + rmp::encode::write_bin(&mut wr, header.signature.as_slice()).unwrap(); + rmp::encode::write_uint8(&mut wr, header.version_type.to_u8()).unwrap(); + rmp::encode::write_uint8(&mut wr, header.flags).unwrap(); + wr + } + + fn write_legacy_time(wr: &mut Vec, ts: OffsetDateTime) { + wr.push(MSGPACK_EXT8); + wr.push(12); + wr.push(MSGPACK_TIME_EXT_LEGACY as u8); + wr.extend_from_slice(&ts.unix_timestamp().to_be_bytes()); + wr.extend_from_slice(&ts.nanosecond().to_be_bytes()); + } + + fn encode_legacy_v1_body() -> Vec { + let mut wr = Vec::new(); + let mod_time = sample_mod_time(); + + rmp::encode::write_map_len(&mut wr, 3).unwrap(); + + rmp::encode::write_str(&mut wr, "Type").unwrap(); + rmp::encode::write_uint8(&mut wr, VersionType::Legacy.to_u8()).unwrap(); + + rmp::encode::write_str(&mut wr, "V1Obj").unwrap(); + rmp::encode::write_map_len(&mut wr, 8).unwrap(); + + rmp::encode::write_str(&mut wr, "Version").unwrap(); + rmp::encode::write_str(&mut wr, "1.0.1").unwrap(); + rmp::encode::write_str(&mut wr, "Format").unwrap(); + rmp::encode::write_str(&mut wr, "xl").unwrap(); + + rmp::encode::write_str(&mut wr, "Stat").unwrap(); + rmp::encode::write_map_len(&mut wr, 5).unwrap(); + rmp::encode::write_str(&mut wr, "Size").unwrap(); + rmp::encode::write_sint(&mut wr, 11).unwrap(); + rmp::encode::write_str(&mut wr, "ModTime").unwrap(); + write_legacy_time(&mut wr, mod_time); + rmp::encode::write_str(&mut wr, "Name").unwrap(); + rmp::encode::write_str(&mut wr, "hello.txt").unwrap(); + rmp::encode::write_str(&mut wr, "Dir").unwrap(); + rmp::encode::write_bool(&mut wr, false).unwrap(); + rmp::encode::write_str(&mut wr, "Mode").unwrap(); + rmp::encode::write_u32(&mut wr, 0o644).unwrap(); + + rmp::encode::write_str(&mut wr, "Erasure").unwrap(); + rmp::encode::write_map_len(&mut wr, 7).unwrap(); + rmp::encode::write_str(&mut wr, "Algorithm").unwrap(); + rmp::encode::write_str(&mut wr, "ReedSolomon").unwrap(); + rmp::encode::write_str(&mut wr, "DataBlocks").unwrap(); + rmp::encode::write_sint(&mut wr, 4).unwrap(); + rmp::encode::write_str(&mut wr, "ParityBlocks").unwrap(); + rmp::encode::write_sint(&mut wr, 2).unwrap(); + rmp::encode::write_str(&mut wr, "BlockSize").unwrap(); + rmp::encode::write_sint(&mut wr, 1_048_576).unwrap(); + rmp::encode::write_str(&mut wr, "Index").unwrap(); + rmp::encode::write_sint(&mut wr, 1).unwrap(); + rmp::encode::write_str(&mut wr, "Distribution").unwrap(); + rmp::encode::write_array_len(&mut wr, 6).unwrap(); + for value in 1..=6 { + rmp::encode::write_sint(&mut wr, value).unwrap(); + } + rmp::encode::write_str(&mut wr, "Checksums").unwrap(); + rmp::encode::write_array_len(&mut wr, 0).unwrap(); + + rmp::encode::write_str(&mut wr, "Meta").unwrap(); + rmp::encode::write_map_len(&mut wr, 1).unwrap(); + rmp::encode::write_str(&mut wr, "content-type").unwrap(); + rmp::encode::write_str(&mut wr, "text/plain").unwrap(); + + rmp::encode::write_str(&mut wr, "Parts").unwrap(); + rmp::encode::write_array_len(&mut wr, 1).unwrap(); + rmp::encode::write_map_len(&mut wr, 5).unwrap(); + rmp::encode::write_str(&mut wr, "e").unwrap(); + rmp::encode::write_str(&mut wr, "etag-1").unwrap(); + rmp::encode::write_str(&mut wr, "n").unwrap(); + rmp::encode::write_sint(&mut wr, 1).unwrap(); + rmp::encode::write_str(&mut wr, "s").unwrap(); + rmp::encode::write_sint(&mut wr, 11).unwrap(); + rmp::encode::write_str(&mut wr, "as").unwrap(); + rmp::encode::write_sint(&mut wr, 11).unwrap(); + rmp::encode::write_str(&mut wr, "mt").unwrap(); + write_legacy_time(&mut wr, mod_time); + + rmp::encode::write_str(&mut wr, "VersionID").unwrap(); + rmp::encode::write_str(&mut wr, "").unwrap(); + rmp::encode::write_str(&mut wr, "DataDir").unwrap(); + rmp::encode::write_str(&mut wr, "legacy").unwrap(); + + rmp::encode::write_str(&mut wr, "v").unwrap(); + rmp::encode::write_uint(&mut wr, 1).unwrap(); + + wr + } + + #[test] + fn version_header_unmarshal_v1_uses_legacy_layout_defaults() { + let expected = sample_header(); + let encoded = encode_v1_header(&expected); + + let mut decoded = FileMetaVersionHeader::default(); + decoded.unmarshal_v(1, &encoded).unwrap(); + + assert_eq!(decoded.version_id, expected.version_id); + assert_eq!(decoded.mod_time, expected.mod_time); + assert_eq!(decoded.version_type, expected.version_type); + assert_eq!(decoded.flags, expected.flags); + assert_eq!(decoded.signature, [0; 4]); + assert_eq!(decoded.ec_n, 0); + assert_eq!(decoded.ec_m, 0); + } + + #[test] + fn version_header_unmarshal_v2_keeps_signature_and_zeroes_ec() { + let expected = sample_header(); + let encoded = encode_v2_header(&expected); + + let mut decoded = FileMetaVersionHeader::default(); + decoded.unmarshal_v(2, &encoded).unwrap(); + + assert_eq!(decoded.version_id, expected.version_id); + assert_eq!(decoded.mod_time, expected.mod_time); + assert_eq!(decoded.signature, expected.signature); + assert_eq!(decoded.version_type, expected.version_type); + assert_eq!(decoded.flags, expected.flags); + assert_eq!(decoded.ec_n, 0); + assert_eq!(decoded.ec_m, 0); + } + + #[test] + fn version_header_unmarshal_v3_round_trips_current_layout() { + let expected = sample_header(); + let encoded = expected.marshal_msg().unwrap(); + + let mut decoded = FileMetaVersionHeader::default(); + decoded.unmarshal_v(3, &encoded).unwrap(); + + assert_eq!(decoded, expected); + } + + #[test] + fn legacy_v1_object_body_decodes_into_fileinfo() { + let encoded = encode_legacy_v1_body(); + let decoded = FileMetaVersion::try_from(encoded.as_slice()).unwrap(); + + assert_eq!(decoded.version_type, VersionType::Legacy); + assert!(decoded.valid()); + assert!(decoded.legacy_object.is_some()); + + let fi = decoded.into_fileinfo("bucket", "hello.txt", true); + assert_eq!(fi.volume, "bucket"); + assert_eq!(fi.name, "hello.txt"); + assert_eq!(fi.size, 11); + assert_eq!(fi.mod_time, Some(sample_mod_time())); + assert_eq!(fi.mode, Some(0o644)); + assert_eq!(fi.parts.len(), 1); + assert_eq!(fi.parts[0].etag, "etag-1"); + assert_eq!(fi.parts[0].size, 11); + assert_eq!(fi.erasure.data_blocks, 4); + assert_eq!(fi.erasure.parity_blocks, 2); + assert_eq!(fi.metadata.get("content-type").map(String::as_str), Some("text/plain")); + } +} diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index 0cc274e65..80c857c47 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -877,7 +877,6 @@ mod tests { use crate::{FileMetaVersion, MetaDeleteMarker}; use std::collections::HashMap; use std::io::Cursor; - use time::OffsetDateTime; use uuid::Uuid; #[tokio::test] @@ -921,6 +920,7 @@ mod tests { mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_400).expect("valid timestamp")), meta_sys: HashMap::new(), }), + legacy_object: None, write_version: 99, uses_legacy_checksum: false, }; diff --git a/crates/filemeta/src/test_data.rs b/crates/filemeta/src/test_data.rs index 8e3f43861..04d405d7e 100644 --- a/crates/filemeta/src/test_data.rs +++ b/crates/filemeta/src/test_data.rs @@ -16,6 +16,10 @@ use crate::{ChecksumAlgo, FileMeta, FileMetaShallowVersion, FileMetaVersion, Met use std::collections::HashMap; use time::OffsetDateTime; use uuid::Uuid; +use xxhash_rust::xxh64; + +const MSGPACK_EXT8: u8 = 0xc7; +const MSGPACK_TIME_EXT_LEGACY: i8 = 5; /// Create real xl.meta file data for testing pub fn create_real_xlmeta() -> Result> { @@ -53,6 +57,7 @@ pub fn create_real_xlmeta() -> Result> { let file_version = FileMetaVersion { version_type: VersionType::Object, + legacy_object: None, object: Some(object_version), delete_marker: None, write_version: 1, @@ -72,6 +77,7 @@ pub fn create_real_xlmeta() -> Result> { let delete_file_version = FileMetaVersion { version_type: VersionType::Delete, + legacy_object: None, object: None, delete_marker: Some(delete_marker), write_version: 2, @@ -85,6 +91,7 @@ pub fn create_real_xlmeta() -> Result> { let legacy_version_id = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")?; let legacy_version = FileMetaVersion { version_type: VersionType::Legacy, + legacy_object: None, object: None, delete_marker: None, write_version: 3, @@ -102,6 +109,171 @@ pub fn create_real_xlmeta() -> Result> { fm.marshal_msg() } +fn decode_hex_fixture(input: &str) -> Result> { + let input = input.trim(); + if !input.len().is_multiple_of(2) { + return Err(crate::Error::other("hex fixture must have even length")); + } + + let mut out = Vec::with_capacity(input.len() / 2); + let bytes = input.as_bytes(); + for idx in (0..bytes.len()).step_by(2) { + let hi = (bytes[idx] as char) + .to_digit(16) + .ok_or_else(|| crate::Error::other(format!("invalid hex at index {idx}")))?; + let lo = (bytes[idx + 1] as char) + .to_digit(16) + .ok_or_else(|| crate::Error::other(format!("invalid hex at index {}", idx + 1)))?; + out.push(((hi << 4) | lo) as u8); + } + + Ok(out) +} + +/// Real legacy xl.meta captured in issue #2288. Header/meta versions are 2/1. +pub fn create_issue_2288_legacy_xlmeta() -> Result> { + decode_hex_fixture(include_str!("../tests/fixtures/issue_2288_legacy_xlmeta.hex")) +} + +/// Legacy xl.meta captured in issue #2265. Header/meta versions are 3/2. +pub fn create_issue_2265_legacy_meta_v2_object_xlmeta() -> Result> { + decode_hex_fixture(include_str!("../tests/fixtures/issue_2265_legacy_meta_v2_object.hex")) +} + +/// Legacy config xl.meta captured in issue #2265. Header/meta versions are 3/2. +pub fn create_issue_2265_legacy_meta_v2_config_xlmeta() -> Result> { + decode_hex_fixture(include_str!("../tests/fixtures/issue_2265_legacy_meta_v2_config.hex")) +} + +fn write_legacy_time(wr: &mut Vec, ts: OffsetDateTime) { + wr.push(MSGPACK_EXT8); + wr.push(12); + wr.push(MSGPACK_TIME_EXT_LEGACY as u8); + wr.extend_from_slice(&ts.unix_timestamp().to_be_bytes()); + wr.extend_from_slice(&ts.nanosecond().to_be_bytes()); +} + +fn encode_legacy_v1_header(version_id: Uuid, mod_time: OffsetDateTime) -> Vec { + let mut wr = Vec::new(); + rmp::encode::write_array_len(&mut wr, 4).unwrap(); + rmp::encode::write_bin(&mut wr, version_id.as_bytes()).unwrap(); + rmp::encode::write_i64(&mut wr, mod_time.unix_timestamp_nanos() as i64).unwrap(); + rmp::encode::write_uint8(&mut wr, VersionType::Legacy.to_u8()).unwrap(); + rmp::encode::write_uint8(&mut wr, 0).unwrap(); + wr +} + +fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateTime) -> Vec { + let mut wr = Vec::new(); + + rmp::encode::write_map_len(&mut wr, 3).unwrap(); + + rmp::encode::write_str(&mut wr, "Type").unwrap(); + rmp::encode::write_uint8(&mut wr, VersionType::Legacy.to_u8()).unwrap(); + + rmp::encode::write_str(&mut wr, "V1Obj").unwrap(); + rmp::encode::write_map_len(&mut wr, 8).unwrap(); + + rmp::encode::write_str(&mut wr, "Version").unwrap(); + rmp::encode::write_str(&mut wr, "1.0.1").unwrap(); + rmp::encode::write_str(&mut wr, "Format").unwrap(); + rmp::encode::write_str(&mut wr, "xl").unwrap(); + + rmp::encode::write_str(&mut wr, "Stat").unwrap(); + rmp::encode::write_map_len(&mut wr, 5).unwrap(); + rmp::encode::write_str(&mut wr, "Size").unwrap(); + rmp::encode::write_sint(&mut wr, 11).unwrap(); + rmp::encode::write_str(&mut wr, "ModTime").unwrap(); + write_legacy_time(&mut wr, mod_time); + rmp::encode::write_str(&mut wr, "Name").unwrap(); + rmp::encode::write_str(&mut wr, "hello.txt").unwrap(); + rmp::encode::write_str(&mut wr, "Dir").unwrap(); + rmp::encode::write_bool(&mut wr, false).unwrap(); + rmp::encode::write_str(&mut wr, "Mode").unwrap(); + rmp::encode::write_u32(&mut wr, 0o644).unwrap(); + + rmp::encode::write_str(&mut wr, "Erasure").unwrap(); + rmp::encode::write_map_len(&mut wr, 7).unwrap(); + rmp::encode::write_str(&mut wr, "Algorithm").unwrap(); + rmp::encode::write_str(&mut wr, "ReedSolomon").unwrap(); + rmp::encode::write_str(&mut wr, "DataBlocks").unwrap(); + rmp::encode::write_sint(&mut wr, 4).unwrap(); + rmp::encode::write_str(&mut wr, "ParityBlocks").unwrap(); + rmp::encode::write_sint(&mut wr, 2).unwrap(); + rmp::encode::write_str(&mut wr, "BlockSize").unwrap(); + rmp::encode::write_sint(&mut wr, 1_048_576).unwrap(); + rmp::encode::write_str(&mut wr, "Index").unwrap(); + rmp::encode::write_sint(&mut wr, 1).unwrap(); + rmp::encode::write_str(&mut wr, "Distribution").unwrap(); + rmp::encode::write_array_len(&mut wr, 6).unwrap(); + for value in 1..=6 { + rmp::encode::write_sint(&mut wr, value).unwrap(); + } + rmp::encode::write_str(&mut wr, "Checksums").unwrap(); + rmp::encode::write_array_len(&mut wr, 0).unwrap(); + + rmp::encode::write_str(&mut wr, "Meta").unwrap(); + rmp::encode::write_map_len(&mut wr, 1).unwrap(); + rmp::encode::write_str(&mut wr, "content-type").unwrap(); + rmp::encode::write_str(&mut wr, "text/plain").unwrap(); + + rmp::encode::write_str(&mut wr, "Parts").unwrap(); + rmp::encode::write_array_len(&mut wr, 1).unwrap(); + rmp::encode::write_map_len(&mut wr, 5).unwrap(); + rmp::encode::write_str(&mut wr, "e").unwrap(); + rmp::encode::write_str(&mut wr, "etag-1").unwrap(); + rmp::encode::write_str(&mut wr, "n").unwrap(); + rmp::encode::write_sint(&mut wr, 1).unwrap(); + rmp::encode::write_str(&mut wr, "s").unwrap(); + rmp::encode::write_sint(&mut wr, 11).unwrap(); + rmp::encode::write_str(&mut wr, "as").unwrap(); + rmp::encode::write_sint(&mut wr, 11).unwrap(); + rmp::encode::write_str(&mut wr, "mt").unwrap(); + write_legacy_time(&mut wr, mod_time); + + rmp::encode::write_str(&mut wr, "VersionID").unwrap(); + rmp::encode::write_str(&mut wr, &version_id.to_string()).unwrap(); + rmp::encode::write_str(&mut wr, "DataDir").unwrap(); + rmp::encode::write_str(&mut wr, &data_dir.to_string()).unwrap(); + + rmp::encode::write_str(&mut wr, "v").unwrap(); + rmp::encode::write_uint(&mut wr, 1).unwrap(); + + wr +} + +/// Legacy xl.meta with a V1Obj body and v1 header layout. +pub fn create_legacy_v1_object_xlmeta() -> Result> { + let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef")?; + let data_dir = Uuid::parse_str("fedcba98-7654-3210-fedc-ba9876543210")?; + let mod_time = OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789)?; + + let header = encode_legacy_v1_header(version_id, mod_time); + let body = encode_legacy_v1_body(version_id, data_dir, mod_time); + + let mut wr = Vec::new(); + wr.extend_from_slice(b"XL2 "); + wr.extend_from_slice(&1u16.to_le_bytes()); + wr.extend_from_slice(&3u16.to_le_bytes()); + wr.extend_from_slice(&[0xc6, 0, 0, 0, 0]); + + let offset = wr.len(); + rmp::encode::write_uint(&mut wr, 1).unwrap(); + rmp::encode::write_uint(&mut wr, 1).unwrap(); + rmp::encode::write_sint(&mut wr, 1).unwrap(); + rmp::encode::write_bin(&mut wr, &header).unwrap(); + rmp::encode::write_bin(&mut wr, &body).unwrap(); + + let data_len = (wr.len() - offset) as u32; + wr[offset - 4..offset].copy_from_slice(&data_len.to_be_bytes()); + + let crc = xxh64::xxh64(&wr[offset..], 0) as u32; + wr.push(0xce); + wr.extend_from_slice(&crc.to_be_bytes()); + + Ok(wr) +} + /// Create a complex xl.meta file with multiple versions pub fn create_complex_xlmeta() -> Result> { let mut fm = FileMeta::new(); @@ -139,6 +311,7 @@ pub fn create_complex_xlmeta() -> Result> { let file_version = FileMetaVersion { version_type: VersionType::Object, + legacy_object: None, object: Some(object_version), delete_marker: None, write_version: (i + 1) as u64, @@ -159,6 +332,7 @@ pub fn create_complex_xlmeta() -> Result> { let delete_file_version = FileMetaVersion { version_type: VersionType::Delete, + legacy_object: None, object: None, delete_marker: Some(delete_marker), write_version: (i + 100) as u64, @@ -247,6 +421,7 @@ pub fn create_xlmeta_with_inline_data() -> Result> { let file_version = FileMetaVersion { version_type: VersionType::Object, + legacy_object: None, object: Some(object_version), delete_marker: None, write_version: 1, diff --git a/crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_config.hex b/crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_config.hex new file mode 100644 index 000000000..a5183da96 --- /dev/null +++ b/crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_config.hex @@ -0,0 +1 @@ +584c322001000300c600000109030201c42697c41000000000000000000000000000000000d31897f82d1d10fc3cc4040000000001040001c4dc94a64f626a656374dc0012c41000000000000000000000000000000000c410fba8e4c33f42424294e05ab84b83ae97ab52656564536f6c6f6d6f6e0100ce00100000019101ab4869676877617948617368910191d9203132623336386365353265343936653631616334376233363663376333623636914a914a91c4004a99cd07ea3a02040bce024aae3c00000081bd782d7275737466732d696e7465726e616c2d696e6c696e652d64617461947472756581a465746167d9203132623336386365353265343936653631616334376233363663376333623636c000ceb7d5e4230181d92430303030303030302d303030302d303030302d303030302d303030303030303030303030c46a6716a562ea6255d3b382987387348bd11ac5bc3e7e74b676822409cc7c4b6838301b1e0508abfe45de4ff0686b14c67ae4e7942b2f4840e2cefc26d221b96462025b3bc46a873ddff156aab76a0026a0566899426461bab906db7e25bda83ebf4989a602475c5c646517 diff --git a/crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_object.hex b/crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_object.hex new file mode 100644 index 000000000..21b1f084c --- /dev/null +++ b/crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_object.hex @@ -0,0 +1 @@ +584c322001000300c60000010f030201c42697c41000000000000000000000000000000000d318909e7051d94eecc4040000000001040001c4e294a64f626a656374dc0012c41000000000000000000000000000000000c41004bee19a6eea40c496fd1f39257fcbdcab52656564536f6c6f6d6f6e0100ce00100000019101ab4869676877617948617368910191d920343335393430343631386533326130626438393434653966663638303266353391cd02c391cd02c391c400cd02c399cd07ea2203143ace2fe1caec00000081bd782d7275737466732d696e7465726e616c2d696e6c696e652d64617461947472756581a465746167d9203433353934303436313865333261306264383934346539666636383032663533c000ced691a4040181d92430303030303030302d303030302d303030302d303030302d303030303030303030303030c502e430510cff66292f00568a122e173ac30926f45f75f16b17093948f03a80e0a81101000100de0019a44e616d65a3616161a74372656174656499cd07ea2203143ace2fd25e18000000ab4c6f636b456e61626c6564c2b0506f6c696379436f6e6669674a736f6e90b54e6f74696669636174696f6e436f6e666967586d6c90b24c6966656379636c65436f6e666967586d6c90b34f626a6563744c6f636b436f6e666967586d6c90b356657273696f6e696e67436f6e666967586d6c90b3456e6372797074696f6e436f6e666967586d6c90b054616767696e67436f6e666967586d6c90af51756f7461436f6e6669674a736f6e90b45265706c69636174696f6e436f6e666967586d6c90b74275636b657454617267657473436f6e6669674a736f6e90bb4275636b657454617267657473436f6e6669674d6574614a736f6e90b5506f6c696379436f6e66696755706461746564417499cd07b20100000000000000b94f626a6563744c6f636b436f6e66696755706461746564417499cd07b20100000000000000b9456e6372797074696f6e436f6e66696755706461746564417499cd07b20100000000000000b654616767696e67436f6e66696755706461746564417499cd07b20100000000000000b451756f7461436f6e66696755706461746564417499cd07b20100000000000000ba5265706c69636174696f6e436f6e66696755706461746564417499cd07b20100000000000000b956657273696f6e696e67436f6e66696755706461746564417499cd07b20100000000000000b84c6966656379636c65436f6e66696755706461746564417499cd07b20100000000000000bb4e6f74696669636174696f6e436f6e66696755706461746564417499cd07b20100000000000000bc4275636b657454617267657473436f6e66696755706461746564417499cd07b20100000000000000d9204275636b657454617267657473436f6e6669674d65746155706461746564417499cd07b2010000000000000000 diff --git a/crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex b/crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex new file mode 100644 index 000000000..494da8bd8 --- /dev/null +++ b/crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex @@ -0,0 +1 @@ +584c322001000300c600000170020101c42495c41000000000000000000000000000000000d318774589cc776b92c40496334c780106c5014483a45479706501a556324f626ade0011a24944c41000000000000000000000000000000000a444446972c410241e46ee9fcc4df683f3eaa223d8619ca64563416c676f01a345634d01a345634e00a745634253697a65d200100000a74563496e64657801a64563446973749101a84353756d416c676f01a8506172744e756d739101a9506172744554616773c0a95061727453697a65739123aa506172744153697a65739123a453697a6523a54d54696d65d318774589cc776b92a74d65746153797381bc782d6d696e696f2d696e7465726e616c2d696e6c696e652d64617461c40474727565a74d65746155737282ac636f6e74656e742d74797065b86170706c69636174696f6e2f6f637465742d73747265616da465746167d9203139616134663235356166303132376462386364346638646439326330636537a176ce6356dadbcec50305b70181a46e756c6cc44314603cf986d48aad8d01014bac476d50996dc09f7d257fdede95430b21b24d5248656c6c6f2066726f6d204d696e494f20533320766961204d6964646c657761726521