gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve legacy metadata and admin compatibility (#2202)

Browse Source
This commit is contained in:
weisd
2026-03-18 21:05:09 +08:00
committed by GitHub
parent 84077adf17
commit b9b7d86ae4
133 changed files with 11707 additions and 1945 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+17 -22
View File
@@ -16,7 +16,10 @@ use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_C
use bytes::Bytes;
use rmp_serde::Serializer;
use rustfs_utils::HashAlgorithm;
use rustfs_utils::http::headers::{RESERVED_METADATA_PREFIX_LOWER, RUSTFS_HEALING};
use rustfs_utils::http::{
SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER,
SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, insert_str,
};
use s3s::dto::{RestoreStatus, Timestamp};
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
@@ -236,6 +239,8 @@ pub struct FileInfo {
// Combined checksum when object was uploaded
pub checksum: Option<Bytes>,
pub versioned: bool,
/// True when version meta was parsed via rmp_serde fallback (legacy format).
pub uses_legacy_checksum: bool,
}
impl FileInfo {
@@ -367,58 +372,48 @@ impl FileInfo {
}
pub fn set_healing(&mut self) {
self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
insert_str(&mut self.metadata, SUFFIX_HEALING, "true".to_string());
}
pub fn set_tier_free_version_id(&mut self, version_id: &str) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}"), version_id.to_string());
insert_str(&mut self.metadata, SUFFIX_TIER_FV_ID, version_id.to_string());
}
pub fn tier_free_version_id(&self) -> String {
self.metadata[&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}")].clone()
get_str(&self.metadata, SUFFIX_TIER_FV_ID).unwrap_or_default()
}
pub fn set_tier_free_version(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}"), "".to_string());
insert_str(&mut self.metadata, SUFFIX_TIER_FV_MARKER, "".to_string());
}
pub fn set_skip_tier_free_version(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_SKIP_FV_ID}"), "".to_string());
insert_str(&mut self.metadata, SUFFIX_TIER_SKIP_FV_ID, "".to_string());
}
pub fn skip_tier_free_version(&self) -> bool {
self.metadata
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_SKIP_FV_ID}"))
contains_key_str(&self.metadata, SUFFIX_TIER_SKIP_FV_ID)
}
pub fn tier_free_version(&self) -> bool {
self.metadata
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}"))
contains_key_str(&self.metadata, SUFFIX_TIER_FV_MARKER)
}
pub fn set_inline_data(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").to_owned(), "true".to_owned());
insert_str(&mut self.metadata, SUFFIX_INLINE_DATA, "true".to_string());
}
pub fn set_data_moved(&mut self) {
self.metadata
.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}data-moved").to_owned(), "true".to_owned());
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string());
}
pub fn inline_data(&self) -> bool {
self.metadata
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}inline-data").as_str())
&& !self.is_remote()
contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote()
}
/// Check if the object is compressed
pub fn is_compressed(&self) -> bool {
self.metadata
.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression"))
contains_key_str(&self.metadata, SUFFIX_COMPRESSION)
}
/// Check if the object is remote (transitioned to another tier)
crates/filemeta/src/filemeta.rs
+185 -95
View File
@@ -13,17 +13,20 @@
// limitations under the License.
use crate::{
ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, ObjectPartInfo, RawFileInfo, ReplicationState,
ReplicationStatusType, Result, TIER_FV_ID, TIER_FV_MARKER, VersionPurgeStatusType, is_restored_object_on_disk,
ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, NULL_VERSION_ID, ObjectPartInfo, RawFileInfo,
ReplicationState, ReplicationStatusType, Result, VersionPurgeStatusType, is_restored_object_on_disk,
replication_statuses_map, version_purge_statuses_map,
};
use byteorder::ByteOrder;
use bytes::Bytes;
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
use rustfs_utils::http::headers::{
self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS,
AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER,
VERSION_PURGE_STATUS_KEY,
AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE,
AMZ_STORAGE_CLASS,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, has_internal_suffix, insert_bytes,
is_internal_key,
};
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
@@ -48,7 +51,8 @@ pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' '];
static XL_FILE_VERSION_MAJOR: u16 = 1;
static XL_FILE_VERSION_MINOR: u16 = 3;
static XL_HEADER_VERSION: u8 = 3;
pub static XL_META_VERSION: u8 = 2;
pub static XL_META_VERSION: u8 = 3;
/// Legacy format (main branch): meta_ver=2 with file/header versions 1.3.3.
static XXHASH_SEED: u64 = 0;
const XL_FLAG_FREE_VERSION: u8 = 1 << 0;
@@ -58,6 +62,18 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2;
const META_DATA_READ_DEFAULT: usize = 4 << 10;
const MSGP_UINT32_SIZE: usize = 5;
/// Max object versions per object, default is 10000
const DEFAULT_OBJECT_MAX_VERSIONS: usize = 10000;
/// Returns the inline data map key for a version_id. "null" for null version.
pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
if version_id.is_none() || version_id == Some(Uuid::nil()) {
NULL_VERSION_ID.to_string()
} else {
version_id.unwrap_or_default().to_string()
}
}
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
@@ -68,8 +84,14 @@ pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const TRANSITION_TIER: &str = "transition-tier";
/// Returns true if the key is a transient internal flag that should not be persisted to meta_sys.
pub fn is_skip_meta_key(key: &str) -> bool {
has_internal_suffix(key, SUFFIX_HEALING) || has_internal_suffix(key, SUFFIX_DATA_MOV)
}
mod codec;
mod inline_data;
mod msgp_decode;
mod validation;
mod version;
@@ -171,11 +193,10 @@ impl FileMeta {
for (k, v) in fi.metadata.iter() {
// Split metadata into meta_user and meta_sys based on prefix
// This logic must match From<FileInfo> for MetaObject
if k.len() > RESERVED_METADATA_PREFIX.len()
&& (k.starts_with(RESERVED_METADATA_PREFIX) || k.starts_with(RESERVED_METADATA_PREFIX_LOWER))
{
let is_system = is_internal_key(k);
if is_system {
// Skip internal flags that shouldn't be persisted
if k == headers::X_RUSTFS_HEALING || k == headers::X_RUSTFS_DATA_MOV {
if is_skip_meta_key(k) {
continue;
}
// Insert into meta_sys
@@ -221,18 +242,26 @@ impl FileMeta {
}
pub fn add_version(&mut self, mut fi: FileInfo) -> Result<()> {
// empty version_id means "null" (versioning disabled/suspended)
if fi.version_id.is_none() {
fi.version_id = Some(Uuid::nil());
}
let version_key = data_key_for_version(fi.version_id);
let mut next_data = self.data.clone();
if let Some(ref data) = fi.data {
let key = fi.version_id.unwrap_or_default().to_string();
self.data.replace(&key, data.to_vec())?;
next_data.replace(&version_key, data.to_vec())?;
} else {
let _ = next_data.remove_key(&version_key)?;
}
let version = FileMetaVersion::from(fi);
self.add_version_filemata(version)
self.add_version_filemata(version)?;
self.data = next_data;
Ok(())
}
pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> {
@@ -240,13 +269,12 @@ impl FileMeta {
return Err(Error::other("file meta version invalid"));
}
// TODO: make it configurable
// 1000 is the limit of versions
// if self.versions.len() + 1 > 1000 {
// return Err(Error::other(
// "You've exceeded the limit on the number of versions you can create on this object",
// ));
// }
// check max versions limit
if self.versions.len() + 1 > DEFAULT_OBJECT_MAX_VERSIONS {
return Err(Error::other(
"You've exceeded the limit on the number of versions you can create on this object",
));
}
if self.versions.is_empty() {
self.versions.push(FileMetaShallowVersion::try_from(version)?);
@@ -255,21 +283,44 @@ impl FileMeta {
let vid = version.get_version_id();
if let Some(fidx) = self.versions.iter().position(|v| v.header.version_id == vid) {
// Match existing version for replace; null version: None and Some(nil) are equivalent
let matches = |h: &Option<Uuid>| {
let v_null = vid.is_none() || vid == Some(Uuid::nil());
let h_null = h.is_none() || *h == Some(Uuid::nil());
(v_null && h_null) || (vid == *h)
};
if let Some(fidx) = self.versions.iter().position(|v| matches(&v.header.version_id)) {
return self.set_idx(fidx, version);
}
// append placeholder to find insert position
let placeholder = FileMetaShallowVersion {
header: FileMetaVersionHeader {
mod_time: None, // None sorts before any real mod_time
..Default::default()
},
meta: Vec::new(),
};
self.versions.push(placeholder);
let mod_time = version.get_mod_time();
let new_shallow = FileMetaShallowVersion::try_from(version)?;
for (idx, exist) in self.versions.iter().enumerate() {
if let Some(ref ex_mt) = exist.header.mod_time
&& let Some(ref in_md) = mod_time
&& ex_mt <= in_md
{
self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?);
let ex_mt = exist.header.mod_time;
let insert_here = match (ex_mt, mod_time) {
(None, _) => true, // placeholder: always insert before
(Some(em), Some(nm)) => em <= nm,
(Some(_), None) => false,
};
if insert_here {
self.versions.insert(idx, new_shallow);
self.versions.pop(); // remove placeholder
return Ok(());
}
}
self.versions.pop(); // remove placeholder on fallback
Err(Error::other("add_version failed"))
// if !ver.valid() {
@@ -343,8 +394,9 @@ impl FileMeta {
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
{
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_status.clone())
@@ -353,8 +405,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -363,8 +416,9 @@ impl FileMeta {
.to_vec(),
);
} else {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
@@ -372,8 +426,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -387,15 +442,14 @@ impl FileMeta {
if !fi.version_purge_status().is_empty()
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
{
delete_marker.meta_sys.insert(
VERSION_PURGE_STATUS_KEY.to_string(),
fi.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
let value = fi
.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec();
insert_bytes(&mut delete_marker.meta_sys, SUFFIX_PURGESTATUS, value);
}
if let Some(delete_marker) = ventry.delete_marker.as_mut() {
@@ -433,8 +487,9 @@ impl FileMeta {
if let Some(delete_marker) = v.delete_marker.as_mut() {
if !fi.delete_marker_replication_status().is_empty() {
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_status.clone())
@@ -443,8 +498,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -453,8 +509,9 @@ impl FileMeta {
.to_vec(),
);
} else {
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-status"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
@@ -462,8 +519,9 @@ impl FileMeta {
.as_bytes()
.to_vec(),
);
delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replication-timestamp"),
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
@@ -502,15 +560,14 @@ impl FileMeta {
let mut v = self.get_idx(i)?;
if let Some(obj) = v.object.as_mut() {
obj.meta_sys.insert(
VERSION_PURGE_STATUS_KEY.to_string(),
fi.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
let value = fi
.replication_state_internal
.as_ref()
.map(|v| v.version_purge_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec();
insert_bytes(&mut obj.meta_sys, SUFFIX_PURGESTATUS, value);
for (k, v) in fi
.replication_state_internal
.as_ref()
@@ -617,13 +674,14 @@ impl FileMeta {
// TODO: freeVersion
if header.free_version() {
non_free_versions -= 1;
if include_free_versions && found_free_version.is_none() {
let mut found_free_fi = FileMetaVersion::default();
if found_free_fi.unmarshal_msg(&ver.meta).is_ok() && found_free_fi.version_type != VersionType::Invalid {
let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts);
free_fi.is_latest = true;
found_free_version = Some(free_fi);
}
if include_free_versions
&& found_free_version.is_none()
&& let Ok(found_free_fi) = ver.parse_version_meta()
&& found_free_fi.version_type != VersionType::Invalid
{
let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts);
free_fi.is_latest = true;
found_free_version = Some(free_fi);
}
if header.version_id != Some(vid) {
@@ -650,10 +708,10 @@ impl FileMeta {
fi.successor_mod_time = succ_mod_time;
}
if read_data {
if read_data && fi.inline_data() {
fi.data = self
.data
.find(fi.version_id.unwrap_or_default().to_string().as_str())?
.find(data_key_for_version(fi.version_id).as_str())?
.map(bytes::Bytes::from);
}
@@ -725,9 +783,7 @@ impl FileMeta {
pub fn into_file_info_versions(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfoVersions> {
let mut versions = Vec::new();
for version in self.versions.iter() {
let mut file_version = FileMetaVersion::default();
file_version.unmarshal_msg(&version.meta)?;
let fi = file_version.into_fileinfo(volume, path, all_parts);
let fi = version.into_fileinfo(volume, path, all_parts)?;
versions.push(fi);
}
@@ -770,29 +826,9 @@ impl FileMeta {
self.versions.first().unwrap().header.mod_time
}
/// Load or convert from buffer
/// Load or convert from buffer. Handles both current (meta_ver=3) and legacy (meta_ver=2) formats.
pub fn load_or_convert(buf: &[u8]) -> Result<Self> {
// Try to load as current format first
match Self::load(buf) {
Ok(meta) => Ok(meta),
Err(_) => {
// Try to convert from legacy format
Self::load_legacy(buf)
}
}
}
/// Load legacy format
pub fn load_legacy(_buf: &[u8]) -> Result<Self> {
// Implementation for loading legacy xl.meta formats
// This would handle conversion from older formats
Err(Error::other("Legacy format not yet implemented"))
}
/// Add legacy version
pub fn add_legacy(&mut self, _legacy_obj: &str) -> Result<()> {
// Implementation for adding legacy xl.meta v1 objects
Err(Error::other("Legacy version addition not yet implemented"))
Self::load(buf)
}
/// List all versions as FileInfo
@@ -1061,6 +1097,7 @@ mod test {
object: None,
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
assert!(legacy_version.is_legacy(), "Should be recognized as a Legacy version");
@@ -1178,6 +1215,7 @@ mod test {
}),
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(version).expect("Conversion failed");
@@ -1391,6 +1429,7 @@ mod test {
object: None,
delete_marker: Some(delete_marker),
write_version: (i + 100) as u64,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(delete_version).unwrap();
@@ -1458,6 +1497,57 @@ mod test {
assert!(fm.validate_integrity().is_ok());
}
#[test]
fn test_add_version_clears_stale_inline_data_for_null_version() {
let mut fm = FileMeta::new();
let mut inline_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
inline_fi.mod_time = Some(OffsetDateTime::now_utc());
inline_fi.data = Some(Bytes::new());
inline_fi.set_inline_data();
fm.add_version(inline_fi).unwrap();
let inline_version = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap();
assert!(inline_version.inline_data());
assert_eq!(inline_version.data, Some(Bytes::new()));
let mut disk_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
disk_fi.mod_time = Some(OffsetDateTime::now_utc() + time::Duration::seconds(1));
disk_fi.size = 1024;
fm.add_version(disk_fi).unwrap();
let latest = fm.into_fileinfo("bucket", "test", "", true, false, true).unwrap();
assert!(!latest.inline_data());
assert!(latest.data.is_none());
assert!(
fm.data
.find(data_key_for_version(latest.version_id).as_str())
.unwrap()
.is_none()
);
}
#[test]
fn test_add_version_keeps_inline_data_when_version_insert_fails() {
let mut fm = FileMeta::new();
let mut inline_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
inline_fi.mod_time = Some(OffsetDateTime::now_utc());
inline_fi.data = Some(Bytes::from_static(b"inline"));
inline_fi.set_inline_data();
fm.add_version(inline_fi).unwrap();
let before = fm.data.find(data_key_for_version(Some(Uuid::nil())).as_str()).unwrap();
assert_eq!(before, Some(Bytes::from_static(b"inline").to_vec()));
let invalid_disk_fi = crate::fileinfo::FileInfo::new("test", 2, 1);
let err = fm.add_version(invalid_disk_fi).unwrap_err();
assert!(err.to_string().contains("file meta version invalid"));
let after = fm.data.find(data_key_for_version(Some(Uuid::nil())).as_str()).unwrap();
assert_eq!(after, Some(Bytes::from_static(b"inline").to_vec()));
}
#[test]
fn test_version_merge_scenarios() {
// Test various version merge scenarios
crates/filemeta/src/filemeta/codec.rs
+19 -5
View File
@@ -22,10 +22,26 @@ impl FileMeta {
pub fn load(buf: &[u8]) -> Result<FileMeta> {
let mut xl = FileMeta::default();
xl.unmarshal_msg(buf)?;
Ok(xl)
}
/// Read (major, minor, header_ver, meta_ver) from xl.meta without full parse.
/// Returns Err if format is not feat (e.g. legacy uses rmp_serde in meta block).
pub fn read_format_versions(buf: &[u8]) -> Result<(u16, u16, u8, u8)> {
let (buf, major, minor) = Self::check_xl2_v1(buf)?;
if buf.len() < 5 {
return Err(Error::other("insufficient data for metadata length prefix"));
}
let (mut size_buf, buf) = buf.split_at(5);
let bin_len = rmp::decode::read_bin_len(&mut size_buf)?;
if buf.len() < bin_len as usize {
return Err(Error::other("insufficient data for metadata"));
}
let (meta, _) = buf.split_at(bin_len as usize);
let (_, header_ver, meta_ver, _) = Self::decode_xl_headers(meta)?;
Ok((major, minor, header_ver, meta_ver))
}
pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> {
if buf.len() < 8 {
return Err(Error::other("xl file header not exists"));
@@ -294,11 +310,9 @@ impl FileMeta {
let offset = wr.len();
// xl header
rmp::encode::write_uint8(&mut wr, XL_HEADER_VERSION)?;
rmp::encode::write_uint8(&mut wr, XL_META_VERSION)?;
rmp::encode::write_uint(&mut wr, XL_HEADER_VERSION as u64)?;
rmp::encode::write_uint(&mut wr, XL_META_VERSION as u64)?;
// versions
rmp::encode::write_sint(&mut wr, self.versions.len() as i64)?;
for ver in self.versions.iter() {
crates/filemeta/src/filemeta/inline_data.rs
+7 -5
View File
@@ -40,10 +40,14 @@ impl FileMeta {
/// Count shared data directories
pub fn shared_data_dir_count(&self, version_id: Option<Uuid>, data_dir: Option<Uuid>) -> usize {
let version_id = version_id.unwrap_or_default();
let vid = version_id.unwrap_or_default();
if self.data.entries().unwrap_or_default() > 0
&& self.data.find(version_id.to_string().as_str()).unwrap_or_default().is_some()
&& self
.data
.find(super::data_key_for_version(version_id).as_str())
.unwrap_or_default()
.is_some()
{
return 0;
}
@@ -51,9 +55,7 @@ impl FileMeta {
self.versions
.iter()
.filter(|v| {
v.header.version_type == VersionType::Object
&& v.header.version_id != Some(version_id)
&& v.header.uses_data_dir()
v.header.version_type == VersionType::Object && v.header.version_id != Some(vid) && v.header.uses_data_dir()
})
.filter_map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).ok())
.filter(|&dir| dir.is_some() && dir == data_dir)
crates/filemeta/src/filemeta/msgp_decode.rs
+212
View File
@@ -0,0 +1,212 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{Error, Result};
use rmp::Marker;
use std::io::Read;
/// Reader that prepends a single byte to the stream. Used when we've read the marker
/// and need to pass it to a decoder that expects to read the marker itself.
pub(crate) struct PrependByteReader<'a, R> {
pub(crate) byte: Option<u8>,
pub(crate) inner: &'a mut R,
}
impl<R: Read> Read for PrependByteReader<'_, R> {
#[allow(clippy::collapsible_if)]
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if let Some(b) = self.byte.take() {
if !buf.is_empty() {
buf[0] = b;
return Ok(1);
}
self.byte = Some(b);
}
self.inner.read(buf)
}
}
/// Read MessagePack nil or array length. Returns None for nil, Some(len) for array.
pub(crate) fn read_nil_or_array_len<R: Read>(rd: &mut R) -> Result<Option<usize>> {
let mut buf = [0u8; 1];
rd.read_exact(&mut buf).map_err(Error::from)?;
let marker = buf[0];
match marker {
0xc0 => Ok(None), // nil
0x90..=0x9f => Ok(Some((marker & 0x0f) as usize)),
0xdc => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u16::from_be_bytes(b) as usize))
}
0xdd => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u32::from_be_bytes(b) as usize))
}
_ => Err(Error::other(format!("expected nil or array, got marker 0x{marker:02x}"))),
}
}
/// Read MessagePack nil or map length. Returns None for nil, Some(len) for map.
pub(crate) fn read_nil_or_map_len<R: Read>(rd: &mut R) -> Result<Option<usize>> {
let mut buf = [0u8; 1];
rd.read_exact(&mut buf).map_err(Error::from)?;
let marker = buf[0];
match marker {
0xc0 => Ok(None), // nil
0x80..=0x8f => Ok(Some((marker & 0x0f) as usize)),
0xde => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u16::from_be_bytes(b) as usize))
}
0xdf => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
Ok(Some(u32::from_be_bytes(b) as usize))
}
_ => Err(Error::other(format!("expected nil or map, got marker 0x{marker:02x}"))),
}
}
/// Skip a single MessagePack value. Used for unknown map keys.
pub(crate) fn skip_msgp_value<R: Read>(rd: &mut R) -> Result<()> {
let marker = rmp::decode::read_marker(rd).map_err(Error::from)?;
let skip_len: usize = match marker {
Marker::Null | Marker::False | Marker::True => 0,
Marker::FixPos(_) | Marker::FixNeg(_) => 0,
Marker::U8 => 1,
Marker::U16 => 2,
Marker::U32 => 4,
Marker::U64 => 8,
Marker::I8 => 1,
Marker::I16 => 2,
Marker::I32 => 4,
Marker::I64 => 8,
Marker::F32 => 4,
Marker::F64 => 8,
Marker::FixStr(n) => n as usize,
Marker::Str8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::from)?;
b[0] as usize
}
Marker::Str16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
u16::from_be_bytes(b) as usize
}
Marker::Str32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
u32::from_be_bytes(b) as usize
}
Marker::Bin8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::from)?;
b[0] as usize
}
Marker::Bin16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
u16::from_be_bytes(b) as usize
}
Marker::Bin32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
u32::from_be_bytes(b) as usize
}
Marker::FixArray(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Array32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixMap(n) => {
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u16::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::Map32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
let n = u32::from_be_bytes(b);
for _ in 0..n {
skip_msgp_value(rd)?;
skip_msgp_value(rd)?;
}
return Ok(());
}
Marker::FixExt1 => 1,
Marker::FixExt2 => 2,
Marker::FixExt4 => 4,
Marker::FixExt8 => 8,
Marker::FixExt16 => 16,
Marker::Ext8 => {
let mut b = [0u8; 1];
rd.read_exact(&mut b).map_err(Error::from)?;
let len = b[0] as usize;
1 + len // type byte + data
}
Marker::Ext16 => {
let mut b = [0u8; 2];
rd.read_exact(&mut b).map_err(Error::from)?;
let len = u16::from_be_bytes(b) as usize;
2 + len // type bytes + data
}
Marker::Ext32 => {
let mut b = [0u8; 4];
rd.read_exact(&mut b).map_err(Error::from)?;
let len = u32::from_be_bytes(b) as usize;
4 + len // type bytes + data
}
Marker::Reserved => 0,
};
if skip_len > 0 {
let mut buf = vec![0u8; skip_len];
rd.read_exact(&mut buf).map_err(Error::from)?;
}
Ok(())
}
crates/filemeta/src/filemeta/version.rs
+768 -240
View File
File diff suppressed because it is too large Load Diff
crates/filemeta/src/filemeta_inline.rs
+52
View File
@@ -181,6 +181,58 @@ impl InlineData {
self.serialize(keys, values)
}
pub fn remove_key(&mut self, key: &str) -> Result<bool> {
let buf = self.after_version();
if buf.is_empty() {
return Ok(false);
}
let mut cur = Cursor::new(buf);
let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
let mut keys = Vec::with_capacity(fields_len);
let mut values = Vec::with_capacity(fields_len);
let mut found = false;
while fields_len > 0 {
fields_len -= 1;
let str_len = rmp::decode::read_str_len(&mut cur)?;
let mut field_buff = vec![0u8; str_len as usize];
cur.read_exact(&mut field_buff)?;
let find_key = String::from_utf8(field_buff)?;
let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
let start = cur.position() as usize;
let end = start + bin_len;
cur.set_position(end as u64);
if find_key == key {
found = true;
continue;
}
keys.push(find_key);
values.push(buf[start..end].to_vec());
}
if !found {
return Ok(false);
}
if keys.is_empty() {
self.0 = Vec::new();
return Ok(true);
}
self.serialize(keys, values)?;
Ok(true)
}
pub fn remove(&mut self, remove_keys: Vec<Uuid>) -> Result<bool> {
let buf = self.after_version();
if buf.is_empty() {
crates/filemeta/src/replication.rs
+2 -2
View File
@@ -15,7 +15,7 @@
use bytes::Bytes;
use core::fmt;
use regex::Regex;
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::http::internal_key_rustfs;
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::collections::HashMap;
@@ -859,7 +859,7 @@ pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationS
}
pub fn target_reset_header(arn: &str) -> String {
format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
internal_key_rustfs(&format!("{REPLICATION_RESET}-{arn}"))
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
crates/filemeta/src/test_data.rs
+6
View File
@@ -56,6 +56,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
object: Some(object_version),
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
@@ -74,6 +75,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
object: None,
delete_marker: Some(delete_marker),
write_version: 2,
uses_legacy_checksum: false,
};
let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
@@ -86,6 +88,7 @@ pub fn create_real_xlmeta() -> Result<Vec<u8>> {
object: None,
delete_marker: None,
write_version: 3,
uses_legacy_checksum: false,
};
let mut legacy_shallow = FileMetaShallowVersion::try_from(legacy_version)?;
@@ -139,6 +142,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
object: Some(object_version),
delete_marker: None,
write_version: (i + 1) as u64,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
@@ -158,6 +162,7 @@ pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
object: None,
delete_marker: Some(delete_marker),
write_version: (i + 100) as u64,
uses_legacy_checksum: false,
};
let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
@@ -245,6 +250,7 @@ pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
object: Some(object_version),
delete_marker: None,
write_version: 1,
uses_legacy_checksum: false,
};
let shallow_version = FileMetaShallowVersion::try_from(file_version)?;