add rio/filemeta

crates/filemeta/src/error.rs

@@ -0,0 +1,139 @@
+pub type Result<T> = core::result::Result<T, Error>;
+
+#[derive(thiserror::Error, Debug, Clone)]
+pub enum Error {
+    #[error("File not found")]
+    FileNotFound,
+    #[error("File version not found")]
+    FileVersionNotFound,
+
+    #[error("File corrupt")]
+    FileCorrupt,
+
+    #[error("Done for now")]
+    DoneForNow,
+
+    #[error("Method not allowed")]
+    MethodNotAllowed,
+
+    #[error("I/O error: {0}")]
+    Io(String),
+
+    #[error("rmp serde decode error: {0}")]
+    RmpSerdeDecode(String),
+
+    #[error("rmp serde encode error: {0}")]
+    RmpSerdeEncode(String),
+
+    #[error("Invalid UTF-8: {0}")]
+    FromUtf8(String),
+
+    #[error("rmp decode value read error: {0}")]
+    RmpDecodeValueRead(String),
+
+    #[error("rmp encode value write error: {0}")]
+    RmpEncodeValueWrite(String),
+
+    #[error("rmp decode num value read error: {0}")]
+    RmpDecodeNumValueRead(String),
+
+    #[error("rmp decode marker read error: {0}")]
+    RmpDecodeMarkerRead(String),
+
+    #[error("time component range error: {0}")]
+    TimeComponentRange(String),
+
+    #[error("uuid parse error: {0}")]
+    UuidParse(String),
+}
+
+impl Error {
+    pub fn other<E>(error: E) -> Error
+    where
+        E: Into<Box<dyn std::error::Error + Send + Sync>>,
+    {
+        std::io::Error::other(error).into()
+    }
+}
+
+impl PartialEq for Error {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Error::FileCorrupt, Error::FileCorrupt) => true,
+            (Error::DoneForNow, Error::DoneForNow) => true,
+            (Error::MethodNotAllowed, Error::MethodNotAllowed) => true,
+            (Error::FileNotFound, Error::FileNotFound) => true,
+            (Error::FileVersionNotFound, Error::FileVersionNotFound) => true,
+            (Error::Io(e1), Error::Io(e2)) => e1 == e2,
+            (Error::RmpSerdeDecode(e1), Error::RmpSerdeDecode(e2)) => e1 == e2,
+            (Error::RmpSerdeEncode(e1), Error::RmpSerdeEncode(e2)) => e1 == e2,
+            (Error::RmpDecodeValueRead(e1), Error::RmpDecodeValueRead(e2)) => e1 == e2,
+            (Error::RmpEncodeValueWrite(e1), Error::RmpEncodeValueWrite(e2)) => e1 == e2,
+            (Error::RmpDecodeNumValueRead(e1), Error::RmpDecodeNumValueRead(e2)) => e1 == e2,
+            (Error::TimeComponentRange(e1), Error::TimeComponentRange(e2)) => e1 == e2,
+            (Error::UuidParse(e1), Error::UuidParse(e2)) => e1 == e2,
+            (a, b) => a.to_string() == b.to_string(),
+        }
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(e: std::io::Error) -> Self {
+        Error::Io(e.to_string())
+    }
+}
+
+impl From<rmp_serde::decode::Error> for Error {
+    fn from(e: rmp_serde::decode::Error) -> Self {
+        Error::RmpSerdeDecode(e.to_string())
+    }
+}
+
+impl From<rmp_serde::encode::Error> for Error {
+    fn from(e: rmp_serde::encode::Error) -> Self {
+        Error::RmpSerdeEncode(e.to_string())
+    }
+}
+
+impl From<std::string::FromUtf8Error> for Error {
+    fn from(e: std::string::FromUtf8Error) -> Self {
+        Error::FromUtf8(e.to_string())
+    }
+}
+
+impl From<rmp::decode::ValueReadError> for Error {
+    fn from(e: rmp::decode::ValueReadError) -> Self {
+        Error::RmpDecodeValueRead(e.to_string())
+    }
+}
+
+impl From<rmp::encode::ValueWriteError> for Error {
+    fn from(e: rmp::encode::ValueWriteError) -> Self {
+        Error::RmpEncodeValueWrite(e.to_string())
+    }
+}
+
+impl From<rmp::decode::NumValueReadError> for Error {
+    fn from(e: rmp::decode::NumValueReadError) -> Self {
+        Error::RmpDecodeNumValueRead(e.to_string())
+    }
+}
+
+impl From<time::error::ComponentRange> for Error {
+    fn from(e: time::error::ComponentRange) -> Self {
+        Error::TimeComponentRange(e.to_string())
+    }
+}
+
+impl From<uuid::Error> for Error {
+    fn from(e: uuid::Error) -> Self {
+        Error::UuidParse(e.to_string())
+    }
+}
+
+impl From<rmp::decode::MarkerReadError> for Error {
+    fn from(e: rmp::decode::MarkerReadError) -> Self {
+        let serr = format!("{:?}", e);
+        Error::RmpDecodeMarkerRead(serr)
+    }
+}

crates/filemeta/src/fileinfo.rs

@@ -0,0 +1,438 @@
+use crate::error::{Error, Result};
+use rmp_serde::Serializer;
+use rustfs_utils::HashAlgorithm;
+use serde::Deserialize;
+use serde::Serialize;
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use crate::headers::RESERVED_METADATA_PREFIX;
+use crate::headers::RUSTFS_HEALING;
+use crate::headers::X_RUSTFS_INLINE_DATA;
+
+pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
+pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
+
+// Additional constants from Go version
+pub const NULL_VERSION_ID: &str = "null";
+// pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded";
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
+pub struct ObjectPartInfo {
+    pub etag: String,
+    pub number: usize,
+    pub size: usize,
+    pub actual_size: usize, // Original data size
+    pub mod_time: Option<OffsetDateTime>,
+    // Index holds the index of the part in the erasure coding
+    pub index: Option<Vec<u8>>,
+    // Checksums holds checksums of the part
+    pub checksums: Option<HashMap<String, String>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)]
+// ChecksumInfo - carries checksums of individual scattered parts per disk.
+pub struct ChecksumInfo {
+    pub part_number: usize,
+    pub algorithm: HashAlgorithm,
+    pub hash: Vec<u8>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)]
+pub enum ErasureAlgo {
+    #[default]
+    Invalid = 0,
+    ReedSolomon = 1,
+}
+
+impl ErasureAlgo {
+    pub fn valid(&self) -> bool {
+        *self > ErasureAlgo::Invalid
+    }
+    pub fn to_u8(&self) -> u8 {
+        match self {
+            ErasureAlgo::Invalid => 0,
+            ErasureAlgo::ReedSolomon => 1,
+        }
+    }
+
+    pub fn from_u8(u: u8) -> Self {
+        match u {
+            1 => ErasureAlgo::ReedSolomon,
+            _ => ErasureAlgo::Invalid,
+        }
+    }
+}
+
+impl std::fmt::Display for ErasureAlgo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ErasureAlgo::Invalid => write!(f, "Invalid"),
+            ErasureAlgo::ReedSolomon => write!(f, "{}", ERASURE_ALGORITHM),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)]
+// ErasureInfo holds erasure coding and bitrot related information.
+pub struct ErasureInfo {
+    // Algorithm is the String representation of erasure-coding-algorithm
+    pub algorithm: String,
+    // DataBlocks is the number of data blocks for erasure-coding
+    pub data_blocks: usize,
+    // ParityBlocks is the number of parity blocks for erasure-coding
+    pub parity_blocks: usize,
+    // BlockSize is the size of one erasure-coded block
+    pub block_size: usize,
+    // Index is the index of the current disk
+    pub index: usize,
+    // Distribution is the distribution of the data and parity blocks
+    pub distribution: Vec<usize>,
+    // Checksums holds all bitrot checksums of all erasure encoded blocks
+    pub checksums: Vec<ChecksumInfo>,
+}
+
+impl ErasureInfo {
+    pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo {
+        for sum in &self.checksums {
+            if sum.part_number == part_number {
+                return sum.clone();
+            }
+        }
+
+        ChecksumInfo {
+            algorithm: HashAlgorithm::HighwayHash256S,
+            ..Default::default()
+        }
+    }
+
+    /// Calculate the size of each shard.
+    pub fn shard_size(&self) -> usize {
+        self.block_size.div_ceil(self.data_blocks)
+    }
+    /// Calculate the total erasure file size for a given original size.
+    // Returns the final erasure size from the original size
+    pub fn shard_file_size(&self, total_length: usize) -> usize {
+        if total_length == 0 {
+            return 0;
+        }
+
+        let num_shards = total_length / self.block_size;
+        let last_block_size = total_length % self.block_size;
+        let last_shard_size = last_block_size.div_ceil(self.data_blocks);
+        num_shards * self.shard_size() + last_shard_size
+    }
+
+    /// Check if this ErasureInfo equals another ErasureInfo
+    pub fn equals(&self, other: &ErasureInfo) -> bool {
+        self.algorithm == other.algorithm
+            && self.data_blocks == other.data_blocks
+            && self.parity_blocks == other.parity_blocks
+            && self.block_size == other.block_size
+            && self.index == other.index
+            && self.distribution == other.distribution
+    }
+}
+
+// #[derive(Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
+pub struct FileInfo {
+    pub volume: String,
+    pub name: String,
+    pub version_id: Option<Uuid>,
+    pub is_latest: bool,
+    pub deleted: bool,
+    // Transition related fields
+    pub transition_status: Option<String>,
+    pub transitioned_obj_name: Option<String>,
+    pub transition_tier: Option<String>,
+    pub transition_version_id: Option<String>,
+    pub expire_restored: bool,
+    pub data_dir: Option<Uuid>,
+    pub mod_time: Option<OffsetDateTime>,
+    pub size: usize,
+    // File mode bits
+    pub mode: Option<u32>,
+    // WrittenByVersion is the unix time stamp of the version that created this version of the object
+    pub written_by_version: Option<u64>,
+    pub metadata: HashMap<String, String>,
+    pub parts: Vec<ObjectPartInfo>,
+    pub erasure: ErasureInfo,
+    // MarkDeleted marks this version as deleted
+    pub mark_deleted: bool,
+    // ReplicationState - Internal replication state to be passed back in ObjectInfo
+    // pub replication_state: Option<ReplicationState>, // TODO: implement ReplicationState
+    pub data: Option<Vec<u8>>,
+    pub num_versions: usize,
+    pub successor_mod_time: Option<OffsetDateTime>,
+    pub fresh: bool,
+    pub idx: usize,
+    // Combined checksum when object was uploaded
+    pub checksum: Option<Vec<u8>>,
+    pub versioned: bool,
+}
+
+impl FileInfo {
+    pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self {
+        let indexs = {
+            let cardinality = data_blocks + parity_blocks;
+            let mut nums = vec![0; cardinality];
+            let key_crc = crc32fast::hash(object.as_bytes());
+
+            let start = key_crc as usize % cardinality;
+            for i in 1..=cardinality {
+                nums[i - 1] = 1 + ((start + i) % cardinality);
+            }
+
+            nums
+        };
+        Self {
+            erasure: ErasureInfo {
+                algorithm: String::from(ERASURE_ALGORITHM),
+                data_blocks,
+                parity_blocks,
+                block_size: BLOCK_SIZE_V2,
+                distribution: indexs,
+                ..Default::default()
+            },
+            ..Default::default()
+        }
+    }
+
+    pub fn is_valid(&self) -> bool {
+        if self.deleted {
+            return true;
+        }
+
+        let data_blocks = self.erasure.data_blocks;
+        let parity_blocks = self.erasure.parity_blocks;
+
+        (data_blocks >= parity_blocks)
+            && (data_blocks > 0)
+            && (self.erasure.index > 0
+                && self.erasure.index <= data_blocks + parity_blocks
+                && self.erasure.distribution.len() == (data_blocks + parity_blocks))
+    }
+
+    pub fn get_etag(&self) -> Option<String> {
+        self.metadata.get("etag").cloned()
+    }
+
+    pub fn write_quorum(&self, quorum: usize) -> usize {
+        if self.deleted {
+            return quorum;
+        }
+
+        if self.erasure.data_blocks == self.erasure.parity_blocks {
+            return self.erasure.data_blocks + 1;
+        }
+
+        self.erasure.data_blocks
+    }
+
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut Serializer::new(&mut buf))?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: FileInfo = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+
+    pub fn add_object_part(
+        &mut self,
+        num: usize,
+        etag: String,
+        part_size: usize,
+        mod_time: Option<OffsetDateTime>,
+        actual_size: usize,
+    ) {
+        let part = ObjectPartInfo {
+            etag,
+            number: num,
+            size: part_size,
+            mod_time,
+            actual_size,
+            index: None,
+            checksums: None,
+        };
+
+        for p in self.parts.iter_mut() {
+            if p.number == num {
+                *p = part;
+                return;
+            }
+        }
+
+        self.parts.push(part);
+
+        self.parts.sort_by(|a, b| a.number.cmp(&b.number));
+    }
+
+    // to_part_offset gets the part index where offset is located, returns part index and offset
+    pub fn to_part_offset(&self, offset: usize) -> Result<(usize, usize)> {
+        if offset == 0 {
+            return Ok((0, 0));
+        }
+
+        let mut part_offset = offset;
+        for (i, part) in self.parts.iter().enumerate() {
+            let part_index = i;
+            if part_offset < part.size {
+                return Ok((part_index, part_offset));
+            }
+
+            part_offset -= part.size
+        }
+
+        Err(Error::other("part not found"))
+    }
+
+    pub fn set_healing(&mut self) {
+        self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
+    }
+
+    pub fn set_inline_data(&mut self) {
+        self.metadata.insert(X_RUSTFS_INLINE_DATA.to_owned(), "true".to_owned());
+    }
+    pub fn inline_data(&self) -> bool {
+        self.metadata.get(X_RUSTFS_INLINE_DATA).is_some_and(|v| v == "true")
+    }
+
+    /// Check if the object is compressed
+    pub fn is_compressed(&self) -> bool {
+        self.metadata
+            .contains_key(&format!("{}compression", RESERVED_METADATA_PREFIX))
+    }
+
+    /// Check if the object is remote (transitioned to another tier)
+    pub fn is_remote(&self) -> bool {
+        !self.transition_tier.as_ref().map_or(true, |s| s.is_empty())
+    }
+
+    /// Get the data directory for this object
+    pub fn get_data_dir(&self) -> String {
+        if self.deleted {
+            return "delete-marker".to_string();
+        }
+        self.data_dir.map_or("".to_string(), |dir| dir.to_string())
+    }
+
+    /// Read quorum returns expected read quorum for this FileInfo
+    pub fn read_quorum(&self, dquorum: usize) -> usize {
+        if self.deleted {
+            return dquorum;
+        }
+        self.erasure.data_blocks
+    }
+
+    /// Create a shallow copy with minimal information for READ MRF checks
+    pub fn shallow_copy(&self) -> Self {
+        Self {
+            volume: self.volume.clone(),
+            name: self.name.clone(),
+            version_id: self.version_id,
+            deleted: self.deleted,
+            erasure: self.erasure.clone(),
+            ..Default::default()
+        }
+    }
+
+    /// Check if this FileInfo equals another FileInfo
+    pub fn equals(&self, other: &FileInfo) -> bool {
+        // Check if both are compressed or both are not compressed
+        if self.is_compressed() != other.is_compressed() {
+            return false;
+        }
+
+        // Check transition info
+        if !self.transition_info_equals(other) {
+            return false;
+        }
+
+        // Check mod time
+        if self.mod_time != other.mod_time {
+            return false;
+        }
+
+        // Check erasure info
+        self.erasure.equals(&other.erasure)
+    }
+
+    /// Check if transition related information are equal
+    pub fn transition_info_equals(&self, other: &FileInfo) -> bool {
+        self.transition_status == other.transition_status
+            && self.transition_tier == other.transition_tier
+            && self.transitioned_obj_name == other.transitioned_obj_name
+            && self.transition_version_id == other.transition_version_id
+    }
+
+    /// Check if metadata maps are equal
+    pub fn metadata_equals(&self, other: &FileInfo) -> bool {
+        if self.metadata.len() != other.metadata.len() {
+            return false;
+        }
+        for (k, v) in &self.metadata {
+            if other.metadata.get(k) != Some(v) {
+                return false;
+            }
+        }
+        true
+    }
+
+    /// Check if replication related fields are equal
+    pub fn replication_info_equals(&self, other: &FileInfo) -> bool {
+        self.mark_deleted == other.mark_deleted
+        // TODO: Add replication_state comparison when implemented
+        // && self.replication_state == other.replication_state
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct FileInfoVersions {
+    // Name of the volume.
+    pub volume: String,
+
+    // Name of the file.
+    pub name: String,
+
+    // Represents the latest mod time of the
+    // latest version.
+    pub latest_mod_time: Option<OffsetDateTime>,
+
+    pub versions: Vec<FileInfo>,
+    pub free_versions: Vec<FileInfo>,
+}
+
+impl FileInfoVersions {
+    pub fn find_version_index(&self, v: &str) -> Option<usize> {
+        if v.is_empty() {
+            return None;
+        }
+
+        let vid = Uuid::parse_str(v).unwrap_or_default();
+
+        self.versions.iter().position(|v| v.version_id == Some(vid))
+    }
+
+    /// Calculate the total size of all versions for this object
+    pub fn size(&self) -> usize {
+        self.versions.iter().map(|v| v.size).sum()
+    }
+}
+
+#[derive(Default, Serialize, Deserialize)]
+pub struct RawFileInfo {
+    pub buf: Vec<u8>,
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct FilesInfo {
+    pub files: Vec<FileInfo>,
+    pub is_truncated: bool,
+}

crates/filemeta/src/filemeta_inline.rs

@@ -0,0 +1,242 @@
+use crate::error::{Error, Result};
+use serde::{Deserialize, Serialize};
+use std::io::{Cursor, Read};
+use uuid::Uuid;
+
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct InlineData(Vec<u8>);
+
+const INLINE_DATA_VER: u8 = 1;
+
+impl InlineData {
+    pub fn new() -> Self {
+        Self(Vec::new())
+    }
+    pub fn update(&mut self, buf: &[u8]) {
+        self.0 = buf.to_vec()
+    }
+    pub fn as_slice(&self) -> &[u8] {
+        self.0.as_slice()
+    }
+    pub fn version_ok(&self) -> bool {
+        if self.0.is_empty() {
+            return true;
+        }
+
+        self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
+    }
+
+    pub fn after_version(&self) -> &[u8] {
+        if self.0.is_empty() {
+            &self.0
+        } else {
+            &self.0[1..]
+        }
+    }
+
+    pub fn find(&self, key: &str) -> Result<Option<Vec<u8>>> {
+        if self.0.is_empty() || !self.version_ok() {
+            return Ok(None);
+        }
+
+        let buf = self.after_version();
+
+        let mut cur = Cursor::new(buf);
+
+        let mut fields_len = rmp::decode::read_map_len(&mut cur)?;
+
+        while fields_len > 0 {
+            fields_len -= 1;
+
+            let str_len = rmp::decode::read_str_len(&mut cur)?;
+
+            let mut field_buff = vec![0u8; str_len as usize];
+
+            cur.read_exact(&mut field_buff)?;
+
+            let field = String::from_utf8(field_buff)?;
+
+            let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
+            let start = cur.position() as usize;
+            let end = start + bin_len;
+            cur.set_position(end as u64);
+
+            if field.as_str() == key {
+                let buf = &buf[start..end];
+                return Ok(Some(buf.to_vec()));
+            }
+        }
+
+        Ok(None)
+    }
+
+    pub fn validate(&self) -> Result<()> {
+        if self.0.is_empty() {
+            return Ok(());
+        }
+
+        let mut cur = Cursor::new(self.after_version());
+
+        let mut fields_len = rmp::decode::read_map_len(&mut cur)?;
+
+        while fields_len > 0 {
+            fields_len -= 1;
+
+            let str_len = rmp::decode::read_str_len(&mut cur)?;
+
+            let mut field_buff = vec![0u8; str_len as usize];
+
+            cur.read_exact(&mut field_buff)?;
+
+            let field = String::from_utf8(field_buff)?;
+            if field.is_empty() {
+                return Err(Error::other("InlineData key empty"));
+            }
+
+            let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
+            let start = cur.position() as usize;
+            let end = start + bin_len;
+            cur.set_position(end as u64);
+        }
+
+        Ok(())
+    }
+
+    pub fn replace(&mut self, key: &str, value: Vec<u8>) -> Result<()> {
+        if self.after_version().is_empty() {
+            let mut keys = Vec::with_capacity(1);
+            let mut values = Vec::with_capacity(1);
+
+            keys.push(key.to_owned());
+            values.push(value);
+
+            return self.serialize(keys, values);
+        }
+
+        let buf = self.after_version();
+        let mut cur = Cursor::new(buf);
+
+        let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
+        let mut keys = Vec::with_capacity(fields_len + 1);
+        let mut values = Vec::with_capacity(fields_len + 1);
+
+        let mut replaced = false;
+
+        while fields_len > 0 {
+            fields_len -= 1;
+
+            let str_len = rmp::decode::read_str_len(&mut cur)?;
+
+            let mut field_buff = vec![0u8; str_len as usize];
+
+            cur.read_exact(&mut field_buff)?;
+
+            let find_key = String::from_utf8(field_buff)?;
+
+            let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
+            let start = cur.position() as usize;
+            let end = start + bin_len;
+            cur.set_position(end as u64);
+
+            let find_value = &buf[start..end];
+
+            if find_key.as_str() == key {
+                values.push(value.clone());
+                replaced = true
+            } else {
+                values.push(find_value.to_vec());
+            }
+
+            keys.push(find_key);
+        }
+
+        if !replaced {
+            keys.push(key.to_owned());
+            values.push(value);
+        }
+
+        self.serialize(keys, values)
+    }
+    pub fn remove(&mut self, remove_keys: Vec<Uuid>) -> Result<bool> {
+        let buf = self.after_version();
+        let mut cur = Cursor::new(buf);
+
+        let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize;
+        let mut keys = Vec::with_capacity(fields_len + 1);
+        let mut values = Vec::with_capacity(fields_len + 1);
+
+        let remove_key = |found_key: &str| {
+            for key in remove_keys.iter() {
+                if key.to_string().as_str() == found_key {
+                    return true;
+                }
+            }
+            false
+        };
+
+        let mut found = false;
+
+        while fields_len > 0 {
+            fields_len -= 1;
+
+            let str_len = rmp::decode::read_str_len(&mut cur)?;
+
+            let mut field_buff = vec![0u8; str_len as usize];
+
+            cur.read_exact(&mut field_buff)?;
+
+            let find_key = String::from_utf8(field_buff)?;
+
+            let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize;
+            let start = cur.position() as usize;
+            let end = start + bin_len;
+            cur.set_position(end as u64);
+
+            let find_value = &buf[start..end];
+
+            if !remove_key(&find_key) {
+                values.push(find_value.to_vec());
+                keys.push(find_key);
+            } else {
+                found = true;
+            }
+        }
+
+        if !found {
+            return Ok(false);
+        }
+
+        if keys.is_empty() {
+            self.0 = Vec::new();
+            return Ok(true);
+        }
+
+        self.serialize(keys, values)?;
+        Ok(true)
+    }
+    fn serialize(&mut self, keys: Vec<String>, values: Vec<Vec<u8>>) -> Result<()> {
+        assert_eq!(keys.len(), values.len(), "InlineData serialize: keys/values not match");
+
+        if keys.is_empty() {
+            self.0 = Vec::new();
+            return Ok(());
+        }
+
+        let mut wr = Vec::new();
+
+        wr.push(INLINE_DATA_VER);
+
+        let map_len = keys.len();
+
+        rmp::encode::write_map_len(&mut wr, map_len as u32)?;
+
+        for i in 0..map_len {
+            rmp::encode::write_str(&mut wr, keys[i].as_str())?;
+            rmp::encode::write_bin(&mut wr, values[i].as_slice())?;
+        }
+
+        self.0 = wr;
+
+        Ok(())
+    }
+}

crates/filemeta/src/headers.rs

@@ -0,0 +1,17 @@
+pub const AMZ_META_UNENCRYPTED_CONTENT_LENGTH: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length";
+pub const AMZ_META_UNENCRYPTED_CONTENT_MD5: &str = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5";
+
+pub const AMZ_STORAGE_CLASS: &str = "x-amz-storage-class";
+
+pub const RESERVED_METADATA_PREFIX: &str = "X-RustFS-Internal-";
+pub const RESERVED_METADATA_PREFIX_LOWER: &str = "x-rustfs-internal-";
+
+pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
+// pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
+
+pub const X_RUSTFS_INLINE_DATA: &str = "x-rustfs-inline-data";
+
+pub const VERSION_PURGE_STATUS_KEY: &str = "X-Rustfs-Internal-purgestatus";
+
+pub const X_RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
+pub const X_RUSTFS_DATA_MOV: &str = "X-Rustfs-Internal-data-mov";

crates/filemeta/src/lib.rs

@@ -0,0 +1,13 @@
+mod error;
+mod fileinfo;
+mod filemeta;
+mod filemeta_inline;
+mod headers;
+mod metacache;
+
+pub mod test_data;
+
+pub use fileinfo::*;
+pub use filemeta::*;
+pub use filemeta_inline::*;
+pub use metacache::*;

crates/filemeta/src/metacache.rs

@@ -0,0 +1,874 @@
+use crate::error::{Error, Result};
+use crate::{merge_file_meta_versions, FileInfo, FileInfoVersions, FileMeta, FileMetaShallowVersion, VersionType};
+use rmp::Marker;
+use serde::{Deserialize, Serialize};
+use std::cmp::Ordering;
+use std::str::from_utf8;
+use std::{
+    fmt::Debug,
+    future::Future,
+    pin::Pin,
+    ptr,
+    sync::{
+        atomic::{AtomicPtr, AtomicU64, Ordering as AtomicOrdering},
+        Arc,
+    },
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+use time::OffsetDateTime;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use tokio::spawn;
+use tokio::sync::Mutex;
+use tracing::warn;
+
+const SLASH_SEPARATOR: &str = "/";
+
+#[derive(Clone, Debug, Default)]
+pub struct MetadataResolutionParams {
+    pub dir_quorum: usize,
+    pub obj_quorum: usize,
+    pub requested_versions: usize,
+    pub bucket: String,
+    pub strict: bool,
+    pub candidates: Vec<Vec<FileMetaShallowVersion>>,
+}
+
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
+pub struct MetaCacheEntry {
+    /// name is the full name of the object including prefixes
+    pub name: String,
+    /// Metadata. If none is present it is not an object but only a prefix.
+    /// Entries without metadata will only be present in non-recursive scans.
+    pub metadata: Vec<u8>,
+
+    /// cached contains the metadata if decoded.
+    #[serde(skip)]
+    pub cached: Option<FileMeta>,
+
+    /// Indicates the entry can be reused and only one reference to metadata is expected.
+    pub reusable: bool,
+}
+
+impl MetaCacheEntry {
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut wr = Vec::new();
+        rmp::encode::write_bool(&mut wr, true)?;
+        rmp::encode::write_str(&mut wr, &self.name)?;
+        rmp::encode::write_bin(&mut wr, &self.metadata)?;
+        Ok(wr)
+    }
+
+    pub fn is_dir(&self) -> bool {
+        self.metadata.is_empty() && self.name.ends_with('/')
+    }
+
+    pub fn is_in_dir(&self, dir: &str, separator: &str) -> bool {
+        if dir.is_empty() {
+            let idx = self.name.find(separator);
+            return idx.is_none() || idx.unwrap() == self.name.len() - separator.len();
+        }
+
+        let ext = self.name.trim_start_matches(dir);
+
+        if ext.len() != self.name.len() {
+            let idx = ext.find(separator);
+            return idx.is_none() || idx.unwrap() == ext.len() - separator.len();
+        }
+
+        false
+    }
+
+    pub fn is_object(&self) -> bool {
+        !self.metadata.is_empty()
+    }
+
+    pub fn is_object_dir(&self) -> bool {
+        !self.metadata.is_empty() && self.name.ends_with(SLASH_SEPARATOR)
+    }
+
+    pub fn is_latest_delete_marker(&mut self) -> bool {
+        if let Some(cached) = &self.cached {
+            if cached.versions.is_empty() {
+                return true;
+            }
+            return cached.versions[0].header.version_type == VersionType::Delete;
+        }
+
+        if !FileMeta::is_xl2_v1_format(&self.metadata) {
+            return false;
+        }
+
+        match FileMeta::check_xl2_v1(&self.metadata) {
+            Ok((meta, _, _)) => {
+                if !meta.is_empty() {
+                    return FileMeta::is_latest_delete_marker(meta);
+                }
+            }
+            Err(_) => return true,
+        }
+
+        match self.xl_meta() {
+            Ok(res) => {
+                if res.versions.is_empty() {
+                    return true;
+                }
+                res.versions[0].header.version_type == VersionType::Delete
+            }
+            Err(_) => true,
+        }
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    pub fn to_fileinfo(&self, bucket: &str) -> Result<FileInfo> {
+        if self.is_dir() {
+            return Ok(FileInfo {
+                volume: bucket.to_owned(),
+                name: self.name.clone(),
+                ..Default::default()
+            });
+        }
+
+        if self.cached.is_some() {
+            let fm = self.cached.as_ref().unwrap();
+            if fm.versions.is_empty() {
+                return Ok(FileInfo {
+                    volume: bucket.to_owned(),
+                    name: self.name.clone(),
+                    deleted: true,
+                    is_latest: true,
+                    mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+                    ..Default::default()
+                });
+            }
+
+            let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
+            return Ok(fi);
+        }
+
+        let mut fm = FileMeta::new();
+        fm.unmarshal_msg(&self.metadata)?;
+        let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
+        Ok(fi)
+    }
+
+    pub fn file_info_versions(&self, bucket: &str) -> Result<FileInfoVersions> {
+        if self.is_dir() {
+            return Ok(FileInfoVersions {
+                volume: bucket.to_string(),
+                name: self.name.clone(),
+                versions: vec![FileInfo {
+                    volume: bucket.to_string(),
+                    name: self.name.clone(),
+                    ..Default::default()
+                }],
+                ..Default::default()
+            });
+        }
+
+        let mut fm = FileMeta::new();
+        fm.unmarshal_msg(&self.metadata)?;
+        fm.into_file_info_versions(bucket, self.name.as_str(), false)
+    }
+
+    pub fn matches(&self, other: Option<&MetaCacheEntry>, strict: bool) -> (Option<MetaCacheEntry>, bool) {
+        if other.is_none() {
+            return (None, false);
+        }
+
+        let other = other.unwrap();
+        if self.name != other.name {
+            if self.name < other.name {
+                return (Some(self.clone()), false);
+            }
+            return (Some(other.clone()), false);
+        }
+
+        if other.is_dir() || self.is_dir() {
+            if self.is_dir() {
+                return (Some(self.clone()), other.is_dir() == self.is_dir());
+            }
+            return (Some(other.clone()), other.is_dir() == self.is_dir());
+        }
+
+        let self_vers = match &self.cached {
+            Some(file_meta) => file_meta.clone(),
+            None => match FileMeta::load(&self.metadata) {
+                Ok(meta) => meta,
+                Err(_) => return (None, false),
+            },
+        };
+
+        let other_vers = match &other.cached {
+            Some(file_meta) => file_meta.clone(),
+            None => match FileMeta::load(&other.metadata) {
+                Ok(meta) => meta,
+                Err(_) => return (None, false),
+            },
+        };
+
+        if self_vers.versions.len() != other_vers.versions.len() {
+            match self_vers.lastest_mod_time().cmp(&other_vers.lastest_mod_time()) {
+                Ordering::Greater => return (Some(self.clone()), false),
+                Ordering::Less => return (Some(other.clone()), false),
+                _ => {}
+            }
+
+            if self_vers.versions.len() > other_vers.versions.len() {
+                return (Some(self.clone()), false);
+            }
+            return (Some(other.clone()), false);
+        }
+
+        let mut prefer = None;
+        for (s_version, o_version) in self_vers.versions.iter().zip(other_vers.versions.iter()) {
+            if s_version.header != o_version.header {
+                if s_version.header.has_ec() != o_version.header.has_ec() {
+                    // One version has EC and the other doesn't - may have been written later.
+                    // Compare without considering EC.
+                    let (mut a, mut b) = (s_version.header.clone(), o_version.header.clone());
+                    (a.ec_n, a.ec_m, b.ec_n, b.ec_m) = (0, 0, 0, 0);
+                    if a == b {
+                        continue;
+                    }
+                }
+
+                if !strict && s_version.header.matches_not_strict(&o_version.header) {
+                    if prefer.is_none() {
+                        if s_version.header.sorts_before(&o_version.header) {
+                            prefer = Some(self.clone());
+                        } else {
+                            prefer = Some(other.clone());
+                        }
+                    }
+                    continue;
+                }
+
+                if prefer.is_some() {
+                    return (prefer, false);
+                }
+
+                if s_version.header.sorts_before(&o_version.header) {
+                    return (Some(self.clone()), false);
+                }
+
+                return (Some(other.clone()), false);
+            }
+        }
+
+        if prefer.is_none() {
+            prefer = Some(self.clone());
+        }
+
+        (prefer, true)
+    }
+
+    pub fn xl_meta(&mut self) -> Result<FileMeta> {
+        if self.is_dir() {
+            return Err(Error::FileNotFound);
+        }
+
+        if let Some(meta) = &self.cached {
+            Ok(meta.clone())
+        } else {
+            if self.metadata.is_empty() {
+                return Err(Error::FileNotFound);
+            }
+
+            let meta = FileMeta::load(&self.metadata)?;
+            self.cached = Some(meta.clone());
+            Ok(meta)
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct MetaCacheEntries(pub Vec<Option<MetaCacheEntry>>);
+
+impl MetaCacheEntries {
+    #[allow(clippy::should_implement_trait)]
+    pub fn as_ref(&self) -> &[Option<MetaCacheEntry>] {
+        &self.0
+    }
+
+    pub fn resolve(&self, mut params: MetadataResolutionParams) -> Option<MetaCacheEntry> {
+        if self.0.is_empty() {
+            warn!("decommission_pool: entries resolve empty");
+            return None;
+        }
+
+        let mut dir_exists = 0;
+        let mut selected = None;
+
+        params.candidates.clear();
+        let mut objs_agree = 0;
+        let mut objs_valid = 0;
+
+        for entry in self.0.iter().flatten() {
+            let mut entry = entry.clone();
+
+            warn!("decommission_pool: entries resolve entry {:?}", entry.name);
+            if entry.name.is_empty() {
+                continue;
+            }
+            if entry.is_dir() {
+                dir_exists += 1;
+                selected = Some(entry.clone());
+                warn!("decommission_pool: entries resolve entry dir {:?}", entry.name);
+                continue;
+            }
+
+            let xl = match entry.xl_meta() {
+                Ok(xl) => xl,
+                Err(e) => {
+                    warn!("decommission_pool: entries resolve entry xl_meta {:?}", e);
+                    continue;
+                }
+            };
+
+            objs_valid += 1;
+            params.candidates.push(xl.versions.clone());
+
+            if selected.is_none() {
+                selected = Some(entry.clone());
+                objs_agree = 1;
+                warn!("decommission_pool: entries resolve entry selected {:?}", entry.name);
+                continue;
+            }
+
+            if let (prefer, true) = entry.matches(selected.as_ref(), params.strict) {
+                selected = prefer;
+                objs_agree += 1;
+                warn!("decommission_pool: entries resolve entry prefer {:?}", entry.name);
+                continue;
+            }
+        }
+
+        let Some(selected) = selected else {
+            warn!("decommission_pool: entries resolve entry no selected");
+            return None;
+        };
+
+        if selected.is_dir() && dir_exists >= params.dir_quorum {
+            warn!("decommission_pool: entries resolve entry dir selected {:?}", selected.name);
+            return Some(selected);
+        }
+
+        // If we would never be able to reach read quorum.
+        if objs_valid < params.obj_quorum {
+            warn!(
+                "decommission_pool: entries resolve entry not enough objects {} < {}",
+                objs_valid, params.obj_quorum
+            );
+            return None;
+        }
+
+        if objs_agree == objs_valid {
+            warn!("decommission_pool: entries resolve entry all agree {} == {}", objs_agree, objs_valid);
+            return Some(selected);
+        }
+
+        let Some(cached) = selected.cached else {
+            warn!("decommission_pool: entries resolve entry no cached");
+            return None;
+        };
+
+        let versions = merge_file_meta_versions(params.obj_quorum, params.strict, params.requested_versions, &params.candidates);
+        if versions.is_empty() {
+            warn!("decommission_pool: entries resolve entry no versions");
+            return None;
+        }
+
+        let metadata = match cached.marshal_msg() {
+            Ok(meta) => meta,
+            Err(e) => {
+                warn!("decommission_pool: entries resolve entry marshal_msg {:?}", e);
+                return None;
+            }
+        };
+
+        // Merge if we have disagreement.
+        // Create a new merged result.
+        let new_selected = MetaCacheEntry {
+            name: selected.name.clone(),
+            cached: Some(FileMeta {
+                meta_ver: cached.meta_ver,
+                versions,
+                ..Default::default()
+            }),
+            reusable: true,
+            metadata,
+        };
+
+        warn!("decommission_pool: entries resolve entry selected {:?}", new_selected.name);
+        Some(new_selected)
+    }
+
+    pub fn first_found(&self) -> (Option<MetaCacheEntry>, usize) {
+        (self.0.iter().find(|x| x.is_some()).cloned().unwrap_or_default(), self.0.len())
+    }
+}
+
+#[derive(Debug, Default)]
+pub struct MetaCacheEntriesSortedResult {
+    pub entries: Option<MetaCacheEntriesSorted>,
+    pub err: Option<Error>,
+}
+
+#[derive(Debug, Default)]
+pub struct MetaCacheEntriesSorted {
+    pub o: MetaCacheEntries,
+    pub list_id: Option<String>,
+    pub reuse: bool,
+    pub last_skipped_entry: Option<String>,
+}
+
+impl MetaCacheEntriesSorted {
+    pub fn entries(&self) -> Vec<&MetaCacheEntry> {
+        let entries: Vec<&MetaCacheEntry> = self.o.0.iter().flatten().collect();
+        entries
+    }
+
+    pub fn forward_past(&mut self, marker: Option<String>) {
+        if let Some(val) = marker {
+            if let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val) {
+                self.o.0 = self.o.0.split_off(idx);
+            }
+        }
+    }
+}
+
+const METACACHE_STREAM_VERSION: u8 = 2;
+
+#[derive(Debug)]
+pub struct MetacacheWriter<W> {
+    wr: W,
+    created: bool,
+    buf: Vec<u8>,
+}
+
+impl<W: AsyncWrite + Unpin> MetacacheWriter<W> {
+    pub fn new(wr: W) -> Self {
+        Self {
+            wr,
+            created: false,
+            buf: Vec::new(),
+        }
+    }
+
+    pub async fn flush(&mut self) -> Result<()> {
+        self.wr.write_all(&self.buf).await?;
+        self.buf.clear();
+        Ok(())
+    }
+
+    pub async fn init(&mut self) -> Result<()> {
+        if !self.created {
+            rmp::encode::write_u8(&mut self.buf, METACACHE_STREAM_VERSION).map_err(|e| Error::other(format!("{:?}", e)))?;
+            self.flush().await?;
+            self.created = true;
+        }
+        Ok(())
+    }
+
+    pub async fn write(&mut self, objs: &[MetaCacheEntry]) -> Result<()> {
+        if objs.is_empty() {
+            return Ok(());
+        }
+
+        self.init().await?;
+
+        for obj in objs.iter() {
+            if obj.name.is_empty() {
+                return Err(Error::other("metacacheWriter: no name"));
+            }
+
+            self.write_obj(obj).await?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn write_obj(&mut self, obj: &MetaCacheEntry) -> Result<()> {
+        self.init().await?;
+
+        rmp::encode::write_bool(&mut self.buf, true).map_err(|e| Error::other(format!("{:?}", e)))?;
+        rmp::encode::write_str(&mut self.buf, &obj.name).map_err(|e| Error::other(format!("{:?}", e)))?;
+        rmp::encode::write_bin(&mut self.buf, &obj.metadata).map_err(|e| Error::other(format!("{:?}", e)))?;
+        self.flush().await?;
+
+        Ok(())
+    }
+
+    pub async fn close(&mut self) -> Result<()> {
+        rmp::encode::write_bool(&mut self.buf, false).map_err(|e| Error::other(format!("{:?}", e)))?;
+        self.flush().await?;
+        Ok(())
+    }
+}
+
+pub struct MetacacheReader<R> {
+    rd: R,
+    init: bool,
+    err: Option<Error>,
+    buf: Vec<u8>,
+    offset: usize,
+    current: Option<MetaCacheEntry>,
+}
+
+impl<R: AsyncRead + Unpin> MetacacheReader<R> {
+    pub fn new(rd: R) -> Self {
+        Self {
+            rd,
+            init: false,
+            err: None,
+            buf: Vec::new(),
+            offset: 0,
+            current: None,
+        }
+    }
+
+    pub async fn read_more(&mut self, read_size: usize) -> Result<&[u8]> {
+        let ext_size = read_size + self.offset;
+
+        let extra = ext_size - self.offset;
+        if self.buf.capacity() >= ext_size {
+            // Extend the buffer if we have enough space.
+            self.buf.resize(ext_size, 0);
+        } else {
+            self.buf.extend(vec![0u8; extra]);
+        }
+
+        let pref = self.offset;
+
+        self.rd.read_exact(&mut self.buf[pref..ext_size]).await?;
+
+        self.offset += read_size;
+
+        let data = &self.buf[pref..ext_size];
+
+        Ok(data)
+    }
+
+    fn reset(&mut self) {
+        self.buf.clear();
+        self.offset = 0;
+    }
+
+    async fn check_init(&mut self) -> Result<()> {
+        if !self.init {
+            let ver = match rmp::decode::read_u8(&mut self.read_more(2).await?) {
+                Ok(res) => res,
+                Err(err) => {
+                    self.err = Some(Error::other(format!("{:?}", err)));
+                    0
+                }
+            };
+            match ver {
+                1 | 2 => (),
+                _ => {
+                    self.err = Some(Error::other("invalid version"));
+                }
+            }
+
+            self.init = true;
+        }
+        Ok(())
+    }
+
+    async fn read_str_len(&mut self) -> Result<u32> {
+        let mark = match rmp::decode::read_marker(&mut self.read_more(1).await?) {
+            Ok(res) => res,
+            Err(err) => {
+                let err: Error = err.into();
+                self.err = Some(err.clone());
+                return Err(err);
+            }
+        };
+
+        match mark {
+            Marker::FixStr(size) => Ok(u32::from(size)),
+            Marker::Str8 => Ok(u32::from(self.read_u8().await?)),
+            Marker::Str16 => Ok(u32::from(self.read_u16().await?)),
+            Marker::Str32 => Ok(self.read_u32().await?),
+            _marker => Err(Error::other("str marker err")),
+        }
+    }
+
+    async fn read_bin_len(&mut self) -> Result<u32> {
+        let mark = match rmp::decode::read_marker(&mut self.read_more(1).await?) {
+            Ok(res) => res,
+            Err(err) => {
+                let err: Error = err.into();
+                self.err = Some(err.clone());
+                return Err(err);
+            }
+        };
+
+        match mark {
+            Marker::Bin8 => Ok(u32::from(self.read_u8().await?)),
+            Marker::Bin16 => Ok(u32::from(self.read_u16().await?)),
+            Marker::Bin32 => Ok(self.read_u32().await?),
+            _ => Err(Error::other("bin marker err")),
+        }
+    }
+
+    async fn read_u8(&mut self) -> Result<u8> {
+        let buf = self.read_more(1).await?;
+        Ok(u8::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
+    }
+
+    async fn read_u16(&mut self) -> Result<u16> {
+        let buf = self.read_more(2).await?;
+        Ok(u16::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
+    }
+
+    async fn read_u32(&mut self) -> Result<u32> {
+        let buf = self.read_more(4).await?;
+        Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length")))
+    }
+
+    pub async fn skip(&mut self, size: usize) -> Result<()> {
+        self.check_init().await?;
+
+        if let Some(err) = &self.err {
+            return Err(err.clone());
+        }
+
+        let mut n = size;
+
+        if self.current.is_some() {
+            n -= 1;
+            self.current = None;
+        }
+
+        while n > 0 {
+            match rmp::decode::read_bool(&mut self.read_more(1).await?) {
+                Ok(res) => {
+                    if !res {
+                        return Ok(());
+                    }
+                }
+                Err(err) => {
+                    let err: Error = err.into();
+                    self.err = Some(err.clone());
+                    return Err(err);
+                }
+            };
+
+            let l = self.read_str_len().await?;
+            let _ = self.read_more(l as usize).await?;
+            let l = self.read_bin_len().await?;
+            let _ = self.read_more(l as usize).await?;
+
+            n -= 1;
+        }
+
+        Ok(())
+    }
+
+    pub async fn peek(&mut self) -> Result<Option<MetaCacheEntry>> {
+        self.check_init().await?;
+
+        if let Some(err) = &self.err {
+            return Err(err.clone());
+        }
+
+        match rmp::decode::read_bool(&mut self.read_more(1).await?) {
+            Ok(res) => {
+                if !res {
+                    return Ok(None);
+                }
+            }
+            Err(err) => {
+                let err: Error = err.into();
+                self.err = Some(err.clone());
+                return Err(err);
+            }
+        };
+
+        let l = self.read_str_len().await?;
+
+        let buf = self.read_more(l as usize).await?;
+        let name_buf = buf.to_vec();
+        let name = match from_utf8(&name_buf) {
+            Ok(decoded) => decoded.to_owned(),
+            Err(err) => {
+                self.err = Some(Error::other(err.to_string()));
+                return Err(Error::other(err.to_string()));
+            }
+        };
+
+        let l = self.read_bin_len().await?;
+
+        let buf = self.read_more(l as usize).await?;
+
+        let metadata = buf.to_vec();
+
+        self.reset();
+
+        let entry = Some(MetaCacheEntry {
+            name,
+            metadata,
+            cached: None,
+            reusable: false,
+        });
+        self.current = entry.clone();
+
+        Ok(entry)
+    }
+
+    pub async fn read_all(&mut self) -> Result<Vec<MetaCacheEntry>> {
+        let mut ret = Vec::new();
+
+        loop {
+            if let Some(entry) = self.peek().await? {
+                ret.push(entry);
+                continue;
+            }
+            break;
+        }
+
+        Ok(ret)
+    }
+}
+
+pub type UpdateFn<T> = Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<T>> + Send>> + Send + Sync + 'static>;
+
+#[derive(Clone, Debug, Default)]
+pub struct Opts {
+    pub return_last_good: bool,
+    pub no_wait: bool,
+}
+
+pub struct Cache<T: Clone + Debug + Send> {
+    update_fn: UpdateFn<T>,
+    ttl: Duration,
+    opts: Opts,
+    val: AtomicPtr<T>,
+    last_update_ms: AtomicU64,
+    updating: Arc<Mutex<bool>>,
+}
+
+impl<T: Clone + Debug + Send + 'static> Cache<T> {
+    pub fn new(update_fn: UpdateFn<T>, ttl: Duration, opts: Opts) -> Self {
+        let val = AtomicPtr::new(ptr::null_mut());
+        Self {
+            update_fn,
+            ttl,
+            opts,
+            val,
+            last_update_ms: AtomicU64::new(0),
+            updating: Arc::new(Mutex::new(false)),
+        }
+    }
+
+    #[allow(unsafe_code)]
+    pub async fn get(self: Arc<Self>) -> Result<T> {
+        let v_ptr = self.val.load(AtomicOrdering::SeqCst);
+        let v = if v_ptr.is_null() {
+            None
+        } else {
+            Some(unsafe { (*v_ptr).clone() })
+        };
+
+        let now = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+        if now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() {
+            if let Some(v) = v {
+                return Ok(v);
+            }
+        }
+
+        if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2 {
+            if self.updating.try_lock().is_ok() {
+                let this = Arc::clone(&self);
+                spawn(async move {
+                    let _ = this.update().await;
+                });
+            }
+
+            return Ok(v.unwrap());
+        }
+
+        let _ = self.updating.lock().await;
+
+        if let Ok(duration) =
+            SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst)))
+        {
+            if duration < self.ttl {
+                return Ok(v.unwrap());
+            }
+        }
+
+        match self.update().await {
+            Ok(_) => {
+                let v_ptr = self.val.load(AtomicOrdering::SeqCst);
+                let v = if v_ptr.is_null() {
+                    None
+                } else {
+                    Some(unsafe { (*v_ptr).clone() })
+                };
+                Ok(v.unwrap())
+            }
+            Err(err) => Err(err),
+        }
+    }
+
+    async fn update(&self) -> Result<()> {
+        match (self.update_fn)().await {
+            Ok(val) => {
+                self.val.store(Box::into_raw(Box::new(val)), AtomicOrdering::SeqCst);
+                let now = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .expect("Time went backwards")
+                    .as_secs();
+                self.last_update_ms.store(now, AtomicOrdering::SeqCst);
+                Ok(())
+            }
+            Err(err) => {
+                let v_ptr = self.val.load(AtomicOrdering::SeqCst);
+                if self.opts.return_last_good && !v_ptr.is_null() {
+                    return Ok(());
+                }
+
+                Err(err)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Cursor;
+
+    #[tokio::test]
+    async fn test_writer() {
+        let mut f = Cursor::new(Vec::new());
+        let mut w = MetacacheWriter::new(&mut f);
+
+        let mut objs = Vec::new();
+        for i in 0..10 {
+            let info = MetaCacheEntry {
+                name: format!("item{}", i),
+                metadata: vec![0u8, 10],
+                cached: None,
+                reusable: false,
+            };
+            objs.push(info);
+        }
+
+        w.write(&objs).await.unwrap();
+        w.close().await.unwrap();
+
+        let data = f.into_inner();
+        let nf = Cursor::new(data);
+
+        let mut r = MetacacheReader::new(nf);
+        let nobjs = r.read_all().await.unwrap();
+
+        assert_eq!(objs, nobjs);
+    }
+}

crates/filemeta/src/test_data.rs

@@ -0,0 +1,292 @@
+use crate::error::Result;
+use crate::filemeta::*;
+use std::collections::HashMap;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+/// 创建一个真实的 xl.meta 文件数据用于测试
+pub fn create_real_xlmeta() -> Result<Vec<u8>> {
+    let mut fm = FileMeta::new();
+
+    // 创建一个真实的对象版本
+    let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef")?;
+    let data_dir = Uuid::parse_str("fedcba98-7654-3210-fedc-ba9876543210")?;
+
+    let mut metadata = HashMap::new();
+    metadata.insert("Content-Type".to_string(), "text/plain".to_string());
+    metadata.insert("X-Amz-Meta-Author".to_string(), "test-user".to_string());
+    metadata.insert("X-Amz-Meta-Created".to_string(), "2024-01-15T10:30:00Z".to_string());
+
+    let object_version = MetaObject {
+        version_id: Some(version_id),
+        data_dir: Some(data_dir),
+        erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
+        erasure_m: 4,
+        erasure_n: 2,
+        erasure_block_size: 1024 * 1024, // 1MB
+        erasure_index: 1,
+        erasure_dist: vec![0, 1, 2, 3, 4, 5],
+        bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
+        part_numbers: vec![1],
+        part_etags: vec!["d41d8cd98f00b204e9800998ecf8427e".to_string()],
+        part_sizes: vec![1024],
+        part_actual_sizes: vec![1024],
+        part_indices: Vec::new(),
+        size: 1024,
+        mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200)?), // 2024-01-15 10:30:00 UTC
+        meta_sys: HashMap::new(),
+        meta_user: metadata,
+    };
+
+    let file_version = FileMetaVersion {
+        version_type: VersionType::Object,
+        object: Some(object_version),
+        delete_marker: None,
+        write_version: 1,
+    };
+
+    let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
+    fm.versions.push(shallow_version);
+
+    // 添加一个删除标记版本
+    let delete_version_id = Uuid::parse_str("11111111-2222-3333-4444-555555555555")?;
+    let delete_marker = MetaDeleteMarker {
+        version_id: Some(delete_version_id),
+        mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312260)?), // 1分钟后
+        meta_sys: None,
+    };
+
+    let delete_file_version = FileMetaVersion {
+        version_type: VersionType::Delete,
+        object: None,
+        delete_marker: Some(delete_marker),
+        write_version: 2,
+    };
+
+    let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
+    fm.versions.push(delete_shallow_version);
+
+    // 添加一个 Legacy 版本用于测试
+    let legacy_version_id = Uuid::parse_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")?;
+    let legacy_version = FileMetaVersion {
+        version_type: VersionType::Legacy,
+        object: None,
+        delete_marker: None,
+        write_version: 3,
+    };
+
+    let mut legacy_shallow = FileMetaShallowVersion::try_from(legacy_version)?;
+    legacy_shallow.header.version_id = Some(legacy_version_id);
+    legacy_shallow.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1705312140)?); // 更早的时间
+    fm.versions.push(legacy_shallow);
+
+    // 按修改时间排序（最新的在前）
+    fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
+
+    fm.marshal_msg()
+}
+
+/// 创建一个包含多个版本的复杂 xl.meta 文件
+pub fn create_complex_xlmeta() -> Result<Vec<u8>> {
+    let mut fm = FileMeta::new();
+
+    // 创建10个版本的对象
+    for i in 0..10 {
+        let version_id = Uuid::new_v4();
+        let data_dir = if i % 3 == 0 { Some(Uuid::new_v4()) } else { None };
+
+        let mut metadata = HashMap::new();
+        metadata.insert("Content-Type".to_string(), "application/octet-stream".to_string());
+        metadata.insert("X-Amz-Meta-Version".to_string(), i.to_string());
+        metadata.insert("X-Amz-Meta-Test".to_string(), format!("test-value-{}", i));
+
+        let object_version = MetaObject {
+            version_id: Some(version_id),
+            data_dir,
+            erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
+            erasure_m: 4,
+            erasure_n: 2,
+            erasure_block_size: 1024 * 1024,
+            erasure_index: (i % 6) as usize,
+            erasure_dist: vec![0, 1, 2, 3, 4, 5],
+            bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
+            part_numbers: vec![1],
+            part_etags: vec![format!("etag-{:08x}", i)],
+            part_sizes: vec![1024 * (i + 1) as usize],
+            part_actual_sizes: vec![1024 * (i + 1) as usize],
+            part_indices: Vec::new(),
+            size: 1024 * (i + 1) as usize,
+            mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200 + i * 60)?),
+            meta_sys: HashMap::new(),
+            meta_user: metadata,
+        };
+
+        let file_version = FileMetaVersion {
+            version_type: VersionType::Object,
+            object: Some(object_version),
+            delete_marker: None,
+            write_version: (i + 1) as u64,
+        };
+
+        let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
+        fm.versions.push(shallow_version);
+
+        // 每隔3个版本添加一个删除标记
+        if i % 3 == 2 {
+            let delete_version_id = Uuid::new_v4();
+            let delete_marker = MetaDeleteMarker {
+                version_id: Some(delete_version_id),
+                mod_time: Some(OffsetDateTime::from_unix_timestamp(1705312200 + i * 60 + 30)?),
+                meta_sys: None,
+            };
+
+            let delete_file_version = FileMetaVersion {
+                version_type: VersionType::Delete,
+                object: None,
+                delete_marker: Some(delete_marker),
+                write_version: (i + 100) as u64,
+            };
+
+            let delete_shallow_version = FileMetaShallowVersion::try_from(delete_file_version)?;
+            fm.versions.push(delete_shallow_version);
+        }
+    }
+
+    // 按修改时间排序（最新的在前）
+    fm.versions.sort_by(|a, b| b.header.mod_time.cmp(&a.header.mod_time));
+
+    fm.marshal_msg()
+}
+
+/// 创建一个损坏的 xl.meta 文件用于错误处理测试
+pub fn create_corrupted_xlmeta() -> Vec<u8> {
+    let mut data = vec![
+        // 正确的文件头
+        b'X', b'L', b'2', b' ', // 版本号
+        1, 0, 3, 0, // 版本号
+        0xc6, 0x00, 0x00, 0x00, 0x10, // 正确的 bin32 长度标记，但数据长度不匹配
+    ];
+
+    // 添加不足的数据（少于声明的长度）
+    data.extend_from_slice(&[0x42; 8]); // 只有8字节，但声明了16字节
+
+    data
+}
+
+/// 创建一个空的 xl.meta 文件
+pub fn create_empty_xlmeta() -> Result<Vec<u8>> {
+    let fm = FileMeta::new();
+    fm.marshal_msg()
+}
+
+/// 验证解析结果的辅助函数
+pub fn verify_parsed_metadata(fm: &FileMeta, expected_versions: usize) -> Result<()> {
+    assert_eq!(fm.versions.len(), expected_versions, "版本数量不匹配");
+    assert_eq!(fm.meta_ver, crate::filemeta::XL_META_VERSION, "元数据版本不匹配");
+
+    // 验证版本是否按修改时间排序
+    for i in 1..fm.versions.len() {
+        let prev_time = fm.versions[i - 1].header.mod_time;
+        let curr_time = fm.versions[i].header.mod_time;
+
+        if let (Some(prev), Some(curr)) = (prev_time, curr_time) {
+            assert!(prev >= curr, "版本未按修改时间正确排序");
+        }
+    }
+
+    Ok(())
+}
+
+/// 创建一个包含内联数据的 xl.meta 文件
+pub fn create_xlmeta_with_inline_data() -> Result<Vec<u8>> {
+    let mut fm = FileMeta::new();
+
+    // 添加内联数据
+    let inline_data = b"This is inline data for testing purposes";
+    let version_id = Uuid::new_v4();
+    fm.data.replace(&version_id.to_string(), inline_data.to_vec())?;
+
+    let object_version = MetaObject {
+        version_id: Some(version_id),
+        data_dir: None,
+        erasure_algorithm: crate::fileinfo::ErasureAlgo::ReedSolomon,
+        erasure_m: 1,
+        erasure_n: 1,
+        erasure_block_size: 64 * 1024,
+        erasure_index: 0,
+        erasure_dist: vec![0, 1],
+        bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
+        part_numbers: vec![1],
+        part_etags: Vec::new(),
+        part_sizes: vec![inline_data.len()],
+        part_actual_sizes: Vec::new(),
+        part_indices: Vec::new(),
+        size: inline_data.len(),
+        mod_time: Some(OffsetDateTime::now_utc()),
+        meta_sys: HashMap::new(),
+        meta_user: HashMap::new(),
+    };
+
+    let file_version = FileMetaVersion {
+        version_type: VersionType::Object,
+        object: Some(object_version),
+        delete_marker: None,
+        write_version: 1,
+    };
+
+    let shallow_version = FileMetaShallowVersion::try_from(file_version)?;
+    fm.versions.push(shallow_version);
+
+    fm.marshal_msg()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_real_xlmeta() {
+        let data = create_real_xlmeta().expect("创建测试数据失败");
+        assert!(!data.is_empty(), "生成的数据不应为空");
+
+        // 验证文件头
+        assert_eq!(&data[0..4], b"XL2 ", "文件头不正确");
+
+        // 尝试解析
+        let fm = FileMeta::load(&data).expect("解析失败");
+        verify_parsed_metadata(&fm, 3).expect("验证失败");
+    }
+
+    #[test]
+    fn test_create_complex_xlmeta() {
+        let data = create_complex_xlmeta().expect("创建复杂测试数据失败");
+        assert!(!data.is_empty(), "生成的数据不应为空");
+
+        let fm = FileMeta::load(&data).expect("解析失败");
+        assert!(fm.versions.len() >= 10, "应该有至少10个版本");
+    }
+
+    #[test]
+    fn test_create_xlmeta_with_inline_data() {
+        let data = create_xlmeta_with_inline_data().expect("创建内联数据测试失败");
+        assert!(!data.is_empty(), "生成的数据不应为空");
+
+        let fm = FileMeta::load(&data).expect("解析失败");
+        assert_eq!(fm.versions.len(), 1, "应该有1个版本");
+        assert!(!fm.data.as_slice().is_empty(), "应该包含内联数据");
+    }
+
+    #[test]
+    fn test_corrupted_xlmeta_handling() {
+        let data = create_corrupted_xlmeta();
+        let result = FileMeta::load(&data);
+        assert!(result.is_err(), "损坏的数据应该解析失败");
+    }
+
+    #[test]
+    fn test_empty_xlmeta() {
+        let data = create_empty_xlmeta().expect("创建空测试数据失败");
+        let fm = FileMeta::load(&data).expect("解析空数据失败");
+        assert_eq!(fm.versions.len(), 0, "空文件应该没有版本");
+    }
+}