// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_COMPLETE, VersionPurgeStatusType}; use bytes::Bytes; use rmp_serde::Serializer; use rustfs_utils::HashAlgorithm; use rustfs_utils::http::{ SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_FREE_VERSION, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, has_internal_suffix, insert_str, }; use s3s::dto::{RestoreStatus, Timestamp}; use s3s::header::X_AMZ_RESTORE; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use time::{format_description::FormatItem, macros::format_description}; use uuid::Uuid; pub const ERASURE_ALGORITHM: &str = "rs-vandermonde"; pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M const MAX_ERASURE_SHARDS: usize = 16; const MAX_FILEINFO_PARTS: usize = 10_000; const MAX_FILEINFO_CHECKSUMS: usize = 10_000; const FILEINFO_PART_BITMAP_WORD_BITS: usize = std::mem::size_of::() * 8; const FILEINFO_PART_BITMAP_WORDS: usize = MAX_FILEINFO_PARTS.div_ceil(FILEINFO_PART_BITMAP_WORD_BITS); // Additional constants from Go version pub const NULL_VERSION_ID: &str = "null"; // pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded"; pub const TIER_FV_ID: &str = "tier-free-versionID"; pub const TIER_FV_MARKER: &str = "tier-free-marker"; pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid"; const ERR_RESTORE_HDR_MALFORMED: &str = "x-amz-restore header malformed"; const RFC1123: &[FormatItem<'_>] = format_description!("[weekday repr:short], [day] [month repr:short] [year] [hour]:[minute]:[second] GMT"); #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] pub struct ObjectPartInfo { pub etag: String, pub number: usize, pub size: usize, pub actual_size: i64, // Original data size pub mod_time: Option, // Index holds the index of the part in the erasure coding pub index: Option, // Checksums holds checksums of the part pub checksums: Option>, pub error: Option, } impl ObjectPartInfo { pub fn marshal_msg(&self) -> Result> { let mut buf = Vec::new(); self.serialize(&mut Serializer::new(&mut buf))?; Ok(buf) } pub fn unmarshal(buf: &[u8]) -> Result { let t: ObjectPartInfo = rmp_serde::from_slice(buf)?; Ok(t) } } #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] // ChecksumInfo - carries checksums of individual scattered parts per disk. pub struct ChecksumInfo { pub part_number: usize, pub algorithm: HashAlgorithm, pub hash: Bytes, } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] pub enum ErasureAlgo { #[default] Invalid = 0, ReedSolomon = 1, } impl ErasureAlgo { pub fn valid(&self) -> bool { *self > ErasureAlgo::Invalid } pub fn to_u8(&self) -> u8 { match self { ErasureAlgo::Invalid => 0, ErasureAlgo::ReedSolomon => 1, } } pub fn from_u8(u: u8) -> Self { match u { 1 => ErasureAlgo::ReedSolomon, _ => ErasureAlgo::Invalid, } } } impl std::fmt::Display for ErasureAlgo { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { ErasureAlgo::Invalid => write!(f, "Invalid"), ErasureAlgo::ReedSolomon => write!(f, "{ERASURE_ALGORITHM}"), } } } #[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] // ErasureInfo holds erasure coding and bitrot related information. pub struct ErasureInfo { // Algorithm is the String representation of erasure-coding-algorithm pub algorithm: String, // DataBlocks is the number of data blocks for erasure-coding pub data_blocks: usize, // ParityBlocks is the number of parity blocks for erasure-coding pub parity_blocks: usize, // BlockSize is the size of one erasure-coded block pub block_size: usize, // Index is the index of the current disk pub index: usize, // Distribution is the distribution of the data and parity blocks pub distribution: Vec, // Checksums holds all bitrot checksums of all erasure encoded blocks pub checksums: Vec, } pub fn calc_shard_size(block_size: usize, data_shards: usize) -> usize { (block_size.div_ceil(data_shards) + 1) & !1 } fn checked_calc_shard_size(block_size: usize, data_shards: usize) -> Option { if data_shards == 0 { return None; } block_size.div_ceil(data_shards).checked_add(1).map(|size| size & !1) } impl ErasureInfo { pub fn get_checksum_info(&self, part_number: usize) -> ChecksumInfo { for sum in &self.checksums { if sum.part_number == part_number { return sum.clone(); } } ChecksumInfo { algorithm: HashAlgorithm::HighwayHash256S, ..Default::default() } } /// Calculate the size of each shard. pub fn shard_size(&self) -> usize { calc_shard_size(self.block_size, self.data_blocks) } /// Calculate the total erasure file size for a given original size. // Returns the final erasure size from the original size pub fn shard_file_size(&self, total_length: i64) -> i64 { if total_length == 0 { return 0; } if total_length < 0 { return total_length; } let total_length = total_length as usize; let num_shards = total_length / self.block_size; let last_block_size = total_length % self.block_size; let last_shard_size = calc_shard_size(last_block_size, self.data_blocks); (num_shards * self.shard_size() + last_shard_size) as i64 } /// Check if this ErasureInfo equals another ErasureInfo pub fn equals(&self, other: &ErasureInfo) -> bool { if self.algorithm != other.algorithm { return false; } if self.data_blocks != other.data_blocks { return false; } if self.parity_blocks != other.parity_blocks { return false; } if self.block_size != other.block_size { return false; } if self.distribution.len() != other.distribution.len() { return false; } for (i, v) in self.distribution.iter().enumerate() { if v != &other.distribution[i] { return false; } } true } } // #[derive(Debug, Clone)] #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] pub struct FileInfo { pub volume: String, pub name: String, pub version_id: Option, pub is_latest: bool, pub deleted: bool, pub transition_status: String, pub transitioned_objname: String, pub transition_tier: String, pub transition_version_id: Option, pub expire_restored: bool, pub data_dir: Option, pub mod_time: Option, pub size: i64, // File mode bits pub mode: Option, // WrittenByVersion is the unix time stamp of the version that created this version of the object pub written_by_version: Option, pub metadata: HashMap, pub parts: Vec, pub erasure: ErasureInfo, // MarkDeleted marks this version as deleted pub mark_deleted: bool, // ReplicationState - Internal replication state to be passed back in ObjectInfo pub replication_state_internal: Option, pub data: Option, pub num_versions: usize, pub successor_mod_time: Option, pub fresh: bool, pub idx: usize, // Combined checksum when object was uploaded pub checksum: Option, pub versioned: bool, /// True when version meta was parsed via rmp_serde fallback (legacy format). pub uses_legacy_checksum: bool, } /// Selects the validation policy for a trusted operation boundary. /// /// This mode is deliberately caller-selected and is never inferred from /// serialized [`FileInfo`] state such as `deleted` or `transition_status`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ValidationMode { /// Validate the complete storage erasure layout before payload access. RequireErasure, /// Validate delete metadata without requiring a payload erasure layout. DeleteOnly, } /// Erasure geometry that passed the storage-layout validation policy. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ValidatedErasureLayout { data_blocks: usize, block_size: usize, shard_size: usize, } impl ValidatedErasureLayout { /// Calculates a shard file size without overflowing the metadata's integer format. pub fn shard_file_size(&self, total_length: usize) -> Option { let full_blocks = total_length / self.block_size; let last_block_size = total_length % self.block_size; let last_shard_size = checked_calc_shard_size(last_block_size, self.data_blocks)?; let shard_file_size = full_blocks.checked_mul(self.shard_size)?.checked_add(last_shard_size)?; i64::try_from(shard_file_size).ok() } } fn mark_fileinfo_part(bitmap: &mut [u64; FILEINFO_PART_BITMAP_WORDS], part_number: usize) -> bool { let Some(index) = part_number.checked_sub(1).filter(|&index| index < MAX_FILEINFO_PARTS) else { return false; }; let word = index / FILEINFO_PART_BITMAP_WORD_BITS; let mask = 1u64 << (index % FILEINFO_PART_BITMAP_WORD_BITS); let was_absent = bitmap[word] & mask == 0; bitmap[word] |= mask; was_absent } fn has_fileinfo_part(bitmap: &[u64; FILEINFO_PART_BITMAP_WORDS], part_number: usize) -> bool { let Some(index) = part_number.checked_sub(1).filter(|&index| index < MAX_FILEINFO_PARTS) else { return false; }; bitmap[index / FILEINFO_PART_BITMAP_WORD_BITS] & (1u64 << (index % FILEINFO_PART_BITMAP_WORD_BITS)) != 0 } /// Validates that an erasure `distribution` is a permutation of `1..=n`. /// /// A well-formed distribution has exactly `n` entries and each 1-based slot /// index in `1..=n` appears exactly once. Corrupt or adversarial `xl.meta` /// can carry values of `0` or greater than `n`, which are later used as /// `distribution[k] - 1` indices into fixed-size vectors and would trigger a /// `usize` underflow / out-of-bounds panic. Rejecting such distributions here /// lets the metadata surface as a clean quorum/corruption error instead. pub(crate) fn is_valid_distribution(distribution: &[usize], n: usize) -> bool { if n == 0 || n > MAX_ERASURE_SHARDS || distribution.len() != n { return false; } let mut seen = [false; MAX_ERASURE_SHARDS]; for &block_idx in distribution { // Valid 1-based slots are `1..=n`; anything else (including `0`) is invalid. if block_idx < 1 || block_idx > n { return false; } let slot = block_idx - 1; if seen[slot] { // Duplicate slot: not a permutation. return false; } seen[slot] = true; } true } impl FileInfo { pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self { let indices = { let cardinality = data_blocks + parity_blocks; let mut nums = vec![0; cardinality]; let key_crc = { let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); hasher.update(object.as_bytes()); hasher.finalize() as u32 }; let start = key_crc as usize % cardinality; for i in 1..=cardinality { nums[i - 1] = 1 + ((start + i) % cardinality); } nums }; Self { erasure: ErasureInfo { algorithm: String::from(ERASURE_ALGORITHM), data_blocks, parity_blocks, block_size: BLOCK_SIZE_V2, distribution: indices, ..Default::default() }, ..Default::default() } } fn validate_erasure_geometry_with_index(&self, index: usize) -> Result { let erasure = &self.erasure; if erasure.data_blocks == 0 || erasure.block_size == 0 || erasure.data_blocks < erasure.parity_blocks { return Err(Error::FileCorrupt); } let total_blocks = erasure .data_blocks .checked_add(erasure.parity_blocks) .ok_or(Error::FileCorrupt)?; if total_blocks > MAX_ERASURE_SHARDS || index == 0 || index > total_blocks || !is_valid_distribution(&erasure.distribution, total_blocks) { return Err(Error::FileCorrupt); } let shard_size = checked_calc_shard_size(erasure.block_size, erasure.data_blocks) .filter(|&size| i64::try_from(size).is_ok()) .ok_or(Error::FileCorrupt)?; i64::try_from(erasure.block_size).map_err(|_| Error::FileCorrupt)?; let layout = ValidatedErasureLayout { data_blocks: erasure.data_blocks, block_size: erasure.block_size, shard_size, }; let object_size = usize::try_from(self.size).map_err(|_| Error::FileCorrupt)?; layout.shard_file_size(object_size).ok_or(Error::FileCorrupt)?; Ok(layout) } fn validate_erasure_geometry(&self) -> Result { self.validate_erasure_geometry_with_index(self.erasure.index) } fn validate_collection_bounds(&self) -> Result<()> { if self.parts.len() > MAX_FILEINFO_PARTS || self.erasure.checksums.len() > MAX_FILEINFO_CHECKSUMS || self.erasure.distribution.len() > MAX_ERASURE_SHARDS { return Err(Error::FileCorrupt); } Ok(()) } fn validate_collection_contents(&self, layout: Option<&ValidatedErasureLayout>) -> Result<()> { let mut part_numbers = [0u64; FILEINFO_PART_BITMAP_WORDS]; for part in &self.parts { if let Some(layout) = layout { i64::try_from(part.size).map_err(|_| Error::FileCorrupt)?; layout.shard_file_size(part.size).ok_or(Error::FileCorrupt)?; // A negative `actual_size` is the documented "unknown size" sentinel for // compressed streaming objects (see `ObjectPartInfo::actual_size` / // `ObjectInfo::get_actual_size`), written to xl.meta by both RustFS and // MinIO. Only shard-validate a real, non-negative size; rejecting the // sentinel would make legitimate compressed objects unreadable. if let Ok(actual_size) = usize::try_from(part.actual_size) { layout.shard_file_size(actual_size).ok_or(Error::FileCorrupt)?; } } if !mark_fileinfo_part(&mut part_numbers, part.number) { return Err(Error::FileCorrupt); } } let mut checksum_parts = [0u64; FILEINFO_PART_BITMAP_WORDS]; for checksum in &self.erasure.checksums { if !has_fileinfo_part(&part_numbers, checksum.part_number) || !mark_fileinfo_part(&mut checksum_parts, checksum.part_number) { return Err(Error::FileCorrupt); } } Ok(()) } fn validate_delete_marker_shape(&self, expected_index: usize, allow_nil_version_id: bool) -> Result<()> { if !self.deleted { return Err(Error::FileCorrupt); } let erasure = &self.erasure; let stored_free_version = contains_key_str(&self.metadata, SUFFIX_FREE_VERSION); if self.mod_time.is_none_or(|mod_time| mod_time <= OffsetDateTime::UNIX_EPOCH) || (!allow_nil_version_id && self.version_id.is_some_and(|version_id| version_id.is_nil())) || self.transition_version_id.is_some_and(|version_id| version_id.is_nil()) || self.size != 0 || self.data_dir.is_some() || self.mode.is_some() || self.written_by_version.is_some() || self.data.is_some() || self.checksum.is_some() || !self.parts.is_empty() || !erasure.algorithm.is_empty() || erasure.data_blocks != 0 || erasure.parity_blocks != 0 || erasure.block_size != 0 || erasure.index != expected_index || !erasure.distribution.is_empty() || !erasure.checksums.is_empty() || stored_free_version != self.tier_free_version() || (stored_free_version && (self.transition_tier.is_empty() || self.transitioned_objname.is_empty())) { return Err(Error::FileCorrupt); } Ok(()) } fn validate_tier_free_version_delete_shape(&self) -> Result<()> { let erasure = &self.erasure; if !self.deleted || !self.tier_free_version() || self.version_id.is_none_or(|version_id| version_id.is_nil()) || self.mod_time.is_some() || self.is_latest || !self.transition_status.is_empty() || !self.transitioned_objname.is_empty() || !self.transition_tier.is_empty() || self.transition_version_id.is_some() || self.expire_restored || self.size != 0 || self.data_dir.is_some() || self.mode.is_some() || self.written_by_version.is_some() || self.mark_deleted || self.replication_state_internal.is_some() || self.data.is_some() || self.num_versions != 0 || self.successor_mod_time.is_some() || self.fresh || self.idx != 0 || self.checksum.is_some() || self.versioned || self.uses_legacy_checksum || !self.parts.is_empty() || self.metadata.is_empty() || self.metadata.len() > 2 || self .metadata .iter() .any(|(key, value)| !has_internal_suffix(key, SUFFIX_TIER_FV_MARKER) || !value.is_empty()) || !erasure.algorithm.is_empty() || erasure.data_blocks != 0 || erasure.parity_blocks != 0 || erasure.block_size != 0 || erasure.index != 0 || !erasure.distribution.is_empty() || !erasure.checksums.is_empty() { return Err(Error::FileCorrupt); } Ok(()) } /// Validates this metadata for the caller-selected operation boundary and /// returns the payload erasure layout when one is established. /// /// All modes enforce collection bounds and checksum-to-part associations. /// Only [`ValidationMode::RequireErasure`] establishes a payload erasure /// layout; the other modes cannot be upgraded based on serialized flags and /// return `None`. pub fn validate(&self, mode: ValidationMode) -> Result> { self.validate_collection_bounds()?; let erasure_layout = match mode { ValidationMode::RequireErasure => Some(self.validate_erasure_geometry()?), ValidationMode::DeleteOnly => { self.validate_delete_marker_shape(0, false)?; None } }; self.validate_collection_contents(erasure_layout.as_ref())?; Ok(erasure_layout) } /// Validate metadata returned by a disk or peer as a strict tagged union. /// Payload entries require complete erasure geometry, including /// purge-pending object versions whose replication state sets `deleted`. /// Only entries that are not valid payloads may fall back to the canonical /// delete-marker shape, so `deleted` cannot bypass payload validation. pub fn validate_for_metadata_read(&self) -> Result<()> { if self.validate(ValidationMode::RequireErasure).is_ok() { return Ok(()); } self.validate(ValidationMode::DeleteOnly).map(|_| ()) } /// Cheap shape check for metadata that already passed /// [`Self::validate_for_metadata_read`] at its decode boundary. pub fn has_valid_metadata_shape(&self) -> bool { self.has_valid_erasure_geometry() || self.is_canonical_delete_marker() } /// Returns whether this is a canonical delete marker rather than an /// erasure-backed object version in a purge-pending state. pub fn is_canonical_delete_marker(&self) -> bool { self.validate_delete_marker_shape(0, false).is_ok() } /// Storage-only marker classification after an absent version ID has been /// normalized to the internal nil UUID. Network and disk boundaries must /// continue to use [`Self::is_canonical_delete_marker`]. pub(crate) fn is_storage_delete_marker(&self) -> bool { self.validate_delete_marker_shape(0, true).is_ok() } /// Whether this is a canonical delete marker carrying only the per-disk /// index assigned by an older rename coordinator. pub fn is_legacy_indexed_delete_marker(&self) -> bool { self.erasure.index > 0 && self.erasure.index <= MAX_ERASURE_SHARDS && self.validate_delete_marker_shape(self.erasure.index, false).is_ok() } /// Validates complete metadata before a write fanout assigns its per-disk /// shard index. An index of zero is treated only as the pending assignment; /// every other geometry and collection invariant remains mandatory. pub fn validate_for_erasure_write(&self) -> Result<()> { self.validate_collection_bounds()?; let index = if self.erasure.index == 0 { 1 } else { self.erasure.index }; let layout = self.validate_erasure_geometry_with_index(index)?; self.validate_collection_contents(Some(&layout)) } /// Validates metadata used to mutate a version list during a delete. /// Delete-marker creation requires the canonical marker shape; other /// delete operations do not access payload shards but still validate all /// collection bounds and checksum-to-part associations before mutation. pub fn validate_for_delete_operation(&self) -> Result<()> { self.validate_collection_bounds()?; if self.deleted && self.tier_free_version() { return self.validate_tier_free_version_delete_shape(); } if self.deleted { return self.validate(ValidationMode::DeleteOnly).map(|_| ()); } self.validate_collection_contents(None) } /// Performs the cheap layout check used by quorum selection after a decode /// boundary has already called [`Self::validate`]. pub fn has_valid_erasure_geometry(&self) -> bool { self.validate_erasure_geometry().is_ok() } /// Validates the complete payload metadata shape. /// /// Repeated quorum passes over metadata that was already fully validated /// should use [`Self::has_valid_erasure_geometry`] instead. pub fn is_valid(&self) -> bool { self.validate(ValidationMode::RequireErasure).is_ok() } pub fn get_etag(&self) -> Option { self.metadata.get("etag").cloned() } pub fn write_quorum(&self, quorum: usize) -> usize { if self.deleted && !self.has_valid_erasure_geometry() { return quorum; } if self.erasure.data_blocks == self.erasure.parity_blocks { return self.erasure.data_blocks + 1; } self.erasure.data_blocks } pub fn marshal_msg(&self) -> Result> { let mut buf = Vec::new(); self.serialize(&mut Serializer::new(&mut buf))?; Ok(buf) } pub fn unmarshal(buf: &[u8]) -> Result { let t: FileInfo = rmp_serde::from_slice(buf)?; Ok(t) } #[allow(clippy::too_many_arguments)] pub fn add_object_part( &mut self, num: usize, etag: String, part_size: usize, mod_time: Option, actual_size: i64, index: Option, checksums: Option>, ) { let part = ObjectPartInfo { etag, number: num, size: part_size, mod_time, actual_size, index, checksums, error: None, }; for p in self.parts.iter_mut() { if p.number == num { *p = part; return; } } self.parts.push(part); self.parts.sort_by_key(|a| a.number); } // to_part_offset gets the part index where offset is located, returns part index and offset pub fn to_part_offset(&self, offset: usize) -> Result<(usize, usize)> { if offset == 0 { return Ok((0, 0)); } let mut part_offset = offset; for (i, part) in self.parts.iter().enumerate() { let part_index = i; if part_offset < part.size { return Ok((part_index, part_offset)); } part_offset -= part.size } Err(Error::other("part not found")) } pub fn set_healing(&mut self) { insert_str(&mut self.metadata, SUFFIX_HEALING, "true".to_string()); } pub fn set_tier_free_version_id(&mut self, version_id: &str) { insert_str(&mut self.metadata, SUFFIX_TIER_FV_ID, version_id.to_string()); } pub fn tier_free_version_id(&self) -> String { get_str(&self.metadata, SUFFIX_TIER_FV_ID).unwrap_or_default() } pub fn set_tier_free_version(&mut self) { insert_str(&mut self.metadata, SUFFIX_TIER_FV_MARKER, "".to_string()); } pub fn set_skip_tier_free_version(&mut self) { insert_str(&mut self.metadata, SUFFIX_TIER_SKIP_FV_ID, "".to_string()); } pub fn skip_tier_free_version(&self) -> bool { contains_key_str(&self.metadata, SUFFIX_TIER_SKIP_FV_ID) } pub fn tier_free_version(&self) -> bool { contains_key_str(&self.metadata, SUFFIX_TIER_FV_MARKER) } pub fn set_inline_data(&mut self) { insert_str(&mut self.metadata, SUFFIX_INLINE_DATA, "true".to_string()); } pub fn set_data_moved(&mut self) { insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string()); } pub fn inline_data(&self) -> bool { contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote() } /// Check if the object is compressed pub fn is_compressed(&self) -> bool { contains_key_str(&self.metadata, SUFFIX_COMPRESSION) } /// Check if the object is remote (transitioned to another tier) pub fn is_remote(&self) -> bool { if self.transition_status != TRANSITION_COMPLETE { return false; } !is_restored_object_on_disk(&self.metadata) } /// Get the data directory for this object pub fn get_data_dir(&self) -> String { if self.deleted { return "delete-marker".to_string(); } self.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()) } /// Read quorum returns expected read quorum for this FileInfo pub fn read_quorum(&self, dquorum: usize) -> usize { if self.deleted { return dquorum; } self.erasure.data_blocks } /// Create a shallow copy with minimal information for READ MRF checks pub fn shallow_copy(&self) -> Self { Self { volume: self.volume.clone(), name: self.name.clone(), version_id: self.version_id, deleted: self.deleted, erasure: self.erasure.clone(), ..Default::default() } } /// Check if this FileInfo equals another FileInfo pub fn equals(&self, other: &FileInfo) -> bool { // Check if both are compressed or both are not compressed if self.is_compressed() != other.is_compressed() { tracing::warn!("equals: is_compressed is not equal, object_name={}", self.name); return false; } // Check transition info if !self.transition_info_equals(other) { tracing::warn!("equals: transition_info_equals is not equal, object_name={}", self.name); return false; } // Check mod time if self.mod_time != other.mod_time { tracing::warn!("equals: mod_time is not equal, object_name={}", self.name); return false; } // Check erasure info if !self.erasure.equals(&other.erasure) { tracing::warn!("equals: erasure is not equal, object_name={}", self.name); return false; } true } /// Check if transition related information are equal pub fn transition_info_equals(&self, other: &FileInfo) -> bool { self.transition_status == other.transition_status && self.transition_tier == other.transition_tier && self.transitioned_objname == other.transitioned_objname && self.transition_version_id == other.transition_version_id } /// Check if metadata maps are equal pub fn metadata_equals(&self, other: &FileInfo) -> bool { if self.metadata.len() != other.metadata.len() { return false; } for (k, v) in &self.metadata { if other.metadata.get(k) != Some(v) { return false; } } true } /// Check if replication related fields are equal pub fn replication_info_equals(&self, other: &FileInfo) -> bool { self.mark_deleted == other.mark_deleted && self.replication_state_internal == other.replication_state_internal } pub fn version_purge_status(&self) -> VersionPurgeStatusType { self.replication_state_internal .as_ref() .map(|v| v.composite_version_purge_status()) .unwrap_or(VersionPurgeStatusType::Empty) } pub fn replication_status(&self) -> ReplicationStatusType { self.replication_state_internal .as_ref() .map(|v| v.composite_replication_status()) .unwrap_or(ReplicationStatusType::Empty) } pub fn delete_marker_replication_status(&self) -> ReplicationStatusType { if self.deleted { self.replication_state_internal .as_ref() .map(|v| v.composite_replication_status()) .unwrap_or(ReplicationStatusType::Empty) } else { ReplicationStatusType::Empty } } pub fn shard_file_size(&self, total_length: i64) -> i64 { self.erasure.shard_file_size(total_length) } } #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct FileInfoVersions { // Name of the volume. pub volume: String, // Name of the file. pub name: String, // Represents the latest mod time of the // latest version. pub latest_mod_time: Option, pub versions: Vec, pub free_versions: Vec, } impl FileInfoVersions { pub fn find_version_index(&self, vid: Uuid) -> Option { self.versions.iter().position(|v| v.version_id == Some(vid)) } /// Calculate the total size of all versions for this object pub fn size(&self) -> i64 { self.versions.iter().map(|v| v.size).sum() } } #[derive(Default, Serialize, Deserialize)] pub struct RawFileInfo { pub buf: Vec, } #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct FilesInfo { pub files: Vec, pub is_truncated: bool, } pub trait RestoreStatusOps { fn expiry(&self) -> Option; fn on_going(&self) -> bool; fn on_disk(&self) -> bool; fn to_string(&self) -> String; fn to_string2(&self) -> String; } impl RestoreStatusOps for RestoreStatus { fn expiry(&self) -> Option { if self.on_going() { return None; } self.restore_expiry_date.clone().map(OffsetDateTime::from) } fn on_going(&self) -> bool { if let Some(on_going) = self.is_restore_in_progress { return on_going; } false } fn on_disk(&self) -> bool { let expiry = self.expiry(); if let Some(expiry0) = expiry && OffsetDateTime::now_utc().unix_timestamp() < expiry0.unix_timestamp() { return true; } false } fn to_string(&self) -> String { if self.on_going() { return "ongoing-request=\"true\"".to_string(); } format!( "ongoing-request=\"false\", expiry-date=\"{}\"", OffsetDateTime::from(self.restore_expiry_date.clone().unwrap()) .format(&Rfc3339) .unwrap() ) } fn to_string2(&self) -> String { if self.on_going() { return "ongoing-request=\"true\"".to_string(); } format!( "ongoing-request=\"false\", expiry-date=\"{}\"", OffsetDateTime::from(self.restore_expiry_date.clone().unwrap()) .format(&RFC1123) .unwrap() ) } } pub fn parse_restore_obj_status(restore_hdr: &str) -> Result { let tokens: Vec<&str> = restore_hdr.splitn(2, ",").collect(); let progress_tokens: Vec<&str> = tokens[0].splitn(2, "=").collect(); if progress_tokens.len() != 2 { return Err(Error::other(ERR_RESTORE_HDR_MALFORMED)); } if progress_tokens[0].trim() != "ongoing-request" { return Err(Error::other(ERR_RESTORE_HDR_MALFORMED)); } match progress_tokens[1] { "true" | "\"true\"" if tokens.len() == 1 => { return Ok(RestoreStatus { is_restore_in_progress: Some(true), ..Default::default() }); } "false" | "\"false\"" => { if tokens.len() != 2 { return Err(Error::other(ERR_RESTORE_HDR_MALFORMED)); } let expiry_tokens: Vec<&str> = tokens[1].splitn(2, "=").collect(); if expiry_tokens.len() != 2 { return Err(Error::other(ERR_RESTORE_HDR_MALFORMED)); } if expiry_tokens[0].trim() != "expiry-date" { return Err(Error::other(ERR_RESTORE_HDR_MALFORMED)); } let expiry = OffsetDateTime::parse(expiry_tokens[1].trim_matches('"'), &Rfc3339) .map_err(|_| Error::other(ERR_RESTORE_HDR_MALFORMED))?; return Ok(RestoreStatus { is_restore_in_progress: Some(false), restore_expiry_date: Some(Timestamp::from(expiry)), }); } _ => (), } Err(Error::other(ERR_RESTORE_HDR_MALFORMED)) } pub fn is_restored_object_on_disk(meta: &HashMap) -> bool { if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str()) && let Ok(restore_status) = parse_restore_obj_status(restore_hdr) { return restore_status.on_disk(); } false } #[cfg(test)] mod tests { use super::*; use proptest::collection::{hash_map, vec}; use proptest::prelude::*; // backlog#959 / ECA-18: the interleaved per-block bitrot subsystem in // rustfs-ecstore (BitrotWriter / bitrot_verify / bitrot_shard_file_size) is // only self-consistent for the streaming Highway variants. That safety rests // on production always resolving a streaming checksum algorithm, so pin the // default here: a part with no explicit ChecksumInfo must resolve to // HighwayHash256S. If this default ever changes, the ecstore bitrot layout // assumptions must be revisited in lockstep. #[test] fn get_checksum_info_defaults_to_highwayhash256s() { let ei = ErasureInfo::default(); assert!(ei.checksums.is_empty(), "default ErasureInfo carries no checksums"); let info = ei.get_checksum_info(1); assert_eq!( info.algorithm, HashAlgorithm::HighwayHash256S, "missing ChecksumInfo must default to the streaming HighwayHash256S algorithm" ); // A present entry is returned as-is; the default only applies on miss. let ei = ErasureInfo { checksums: vec![ChecksumInfo { part_number: 2, algorithm: HashAlgorithm::SHA256, hash: Bytes::new(), }], ..Default::default() }; assert_eq!(ei.get_checksum_info(2).algorithm, HashAlgorithm::SHA256); // A part_number with no entry still falls back to the streaming default. assert_eq!(ei.get_checksum_info(99).algorithm, HashAlgorithm::HighwayHash256S); } // backlog#949: distribution range/permutation validation. #[test] fn is_valid_distribution_accepts_permutation() { assert!(is_valid_distribution(&[1, 2, 3, 4], 4)); assert!(is_valid_distribution(&[4, 2, 1, 3], 4)); assert!(is_valid_distribution(&[1], 1)); } #[test] fn is_valid_distribution_rejects_zero_value() { // A `0` would underflow `block_idx - 1` in the shuffle helpers. assert!(!is_valid_distribution(&[0, 2, 3, 4], 4)); } #[test] fn is_valid_distribution_rejects_out_of_range_value() { // A value greater than N would index out of bounds in the shuffle helpers. assert!(!is_valid_distribution(&[1, 2, 3, 5], 4)); assert!(!is_valid_distribution(&[usize::MAX, 2, 3, 4], 4)); } #[test] fn is_valid_distribution_rejects_duplicates() { // In-range but not a permutation. assert!(!is_valid_distribution(&[1, 1, 3, 4], 4)); } #[test] fn is_valid_distribution_rejects_wrong_length_and_empty() { assert!(!is_valid_distribution(&[1, 2, 3], 4)); assert!(!is_valid_distribution(&[1, 2, 3, 4, 4], 4)); assert!(!is_valid_distribution(&[], 0)); assert!(!is_valid_distribution(&[], 4)); } fn distribution_test_fileinfo() -> FileInfo { // data=2, parity=2 => N=4, distribution is a valid permutation of 1..=4. let mut fi = FileInfo::new("bucket/object", 2, 2); fi.erasure.index = 1; fi } #[test] fn is_valid_accepts_well_formed_distribution() { assert!(distribution_test_fileinfo().is_valid()); } #[test] fn is_valid_rejects_corrupt_distribution_values() { // Zero value. let mut fi = distribution_test_fileinfo(); fi.erasure.distribution = vec![0, 2, 3, 4]; assert!(!fi.is_valid()); // Out-of-range value. let mut fi = distribution_test_fileinfo(); fi.erasure.distribution = vec![1, 2, 3, 9]; assert!(!fi.is_valid()); // Duplicate value (not a permutation). let mut fi = distribution_test_fileinfo(); fi.erasure.distribution = vec![1, 1, 3, 4]; assert!(!fi.is_valid()); // Wrong length. let mut fi = distribution_test_fileinfo(); fi.erasure.distribution = vec![1, 2, 3]; assert!(!fi.is_valid()); } fn validation_test_fileinfo() -> FileInfo { let mut fi = FileInfo::new("bucket/object", 4, 2); fi.erasure.index = 1; fi.parts = vec![ ObjectPartInfo { number: 1, ..Default::default() }, ObjectPartInfo { number: 2, ..Default::default() }, ]; fi.erasure.checksums = vec![ ChecksumInfo { part_number: 1, algorithm: HashAlgorithm::HighwayHash256S, hash: Bytes::from_static(b"checksum-one"), }, ChecksumInfo { part_number: 2, algorithm: HashAlgorithm::SHA256, hash: Bytes::from_static(b"checksum-two"), }, ]; fi } fn assert_file_corrupt(fi: &FileInfo, mode: ValidationMode) { assert_eq!(fi.validate(mode).expect_err("invalid FileInfo must be rejected"), Error::FileCorrupt); } #[test] fn validate_require_erasure_returns_checked_layout() { let fi = validation_test_fileinfo(); let layout = fi .validate(ValidationMode::RequireErasure) .expect("well-formed erasure metadata must validate") .expect("strict validation must return an erasure layout"); assert_eq!(layout.data_blocks, 4); assert_eq!(layout.block_size, BLOCK_SIZE_V2); assert_eq!(layout.shard_size, calc_shard_size(BLOCK_SIZE_V2, 4)); let shard_size = i64::try_from(layout.shard_size).expect("validated shard size must fit i64"); assert_eq!(layout.shard_file_size(0), Some(0)); assert_eq!(layout.shard_file_size(BLOCK_SIZE_V2 - 1), Some(shard_size)); assert_eq!(layout.shard_file_size(BLOCK_SIZE_V2), Some(shard_size)); assert_eq!(layout.shard_file_size(BLOCK_SIZE_V2 + 1), Some(shard_size + 2)); } #[test] fn validate_require_erasure_accepts_zero_parity() { let mut fi = FileInfo::new("bucket/object", 16, 0); fi.erasure.index = 1; let layout = fi .validate(ValidationMode::RequireErasure) .expect("zero parity is a valid storage layout"); assert_eq!(layout.expect("strict layout must be present").data_blocks, 16); } #[test] fn validate_for_erasure_write_only_relaxes_pending_shard_index() { let mut fi = validation_test_fileinfo(); fi.erasure.index = 0; assert_file_corrupt(&fi, ValidationMode::RequireErasure); fi.validate_for_erasure_write() .expect("write preparation must accept an index that fanout has not assigned yet"); fi.erasure.index = fi.erasure.data_blocks + fi.erasure.parity_blocks + 1; assert_eq!(fi.validate_for_erasure_write(), Err(Error::FileCorrupt)); fi.erasure.index = 0; fi.parts.push(fi.parts[0].clone()); assert_eq!(fi.validate_for_erasure_write(), Err(Error::FileCorrupt)); } #[test] fn validate_require_erasure_rejects_invalid_geometry() { let mut fi = validation_test_fileinfo(); fi.erasure.data_blocks = 0; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = validation_test_fileinfo(); fi.erasure.block_size = 0; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = validation_test_fileinfo(); fi.erasure.data_blocks = usize::MAX; fi.erasure.parity_blocks = 1; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = validation_test_fileinfo(); fi.erasure.data_blocks = 9; fi.erasure.parity_blocks = 8; fi.erasure.index = 1; fi.erasure.distribution = (1..=17).collect(); assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = validation_test_fileinfo(); fi.erasure.data_blocks = 1; fi.erasure.parity_blocks = 2; fi.erasure.index = 1; fi.erasure.distribution = vec![1, 2, 3]; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = validation_test_fileinfo(); fi.erasure.index = 0; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = validation_test_fileinfo(); fi.erasure.index = 7; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = validation_test_fileinfo(); fi.erasure.distribution = vec![1, 2, 3, 4, 5, 5]; assert_file_corrupt(&fi, ValidationMode::RequireErasure); } fn one_shard_validation_fileinfo(block_size: usize) -> FileInfo { let mut fi = FileInfo::new("bucket/object", 1, 0); fi.erasure.index = 1; fi.erasure.block_size = block_size; fi } #[test] fn validate_require_erasure_rejects_unrepresentable_shard_sizes() { let fi = one_shard_validation_fileinfo(usize::MAX); assert_file_corrupt(&fi, ValidationMode::RequireErasure); if let Some((max_i64, above_i64_max)) = usize::try_from(i64::MAX) .ok() .and_then(|max_i64| max_i64.checked_add(1).map(|above_i64_max| (max_i64, above_i64_max))) { let mut fi = FileInfo::new("bucket/object", 2, 0); fi.erasure.index = 1; fi.erasure.block_size = above_i64_max; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = FileInfo::new("bucket/object", 2, 0); fi.erasure.index = 1; fi.erasure.block_size = max_i64; fi.parts = vec![ObjectPartInfo { number: 1, size: above_i64_max, ..Default::default() }]; assert_file_corrupt(&fi, ValidationMode::RequireErasure); } let mut fi = one_shard_validation_fileinfo(2); fi.size = -1; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = one_shard_validation_fileinfo(2); fi.size = i64::MAX; assert_file_corrupt(&fi, ValidationMode::RequireErasure); let mut fi = one_shard_validation_fileinfo(2); fi.parts = vec![ObjectPartInfo { number: 1, size: usize::MAX, ..Default::default() }]; assert_file_corrupt(&fi, ValidationMode::RequireErasure); // A negative `actual_size` is the compressed "unknown size" sentinel, not an // unrepresentable size: it must stay readable so existing compressed objects // remain accessible. Only real (non-negative) sizes are shard-validated. let mut fi = one_shard_validation_fileinfo(2); fi.parts = vec![ObjectPartInfo { number: 1, actual_size: -1, ..Default::default() }]; fi.validate(ValidationMode::RequireErasure) .expect("negative actual_size sentinel (compressed objects) must remain readable"); let mut fi = one_shard_validation_fileinfo(2); fi.parts = vec![ObjectPartInfo { number: 1, actual_size: i64::MAX, ..Default::default() }]; assert_file_corrupt(&fi, ValidationMode::RequireErasure); } #[test] fn metadata_read_validation_rejects_flags_that_try_to_relax_payload_validation() { let mut fi = FileInfo::new("bucket/legacy", 0, 2); fi.erasure.index = 1; fi.deleted = true; fi.transition_status = TRANSITION_COMPLETE.to_owned(); fi.transitioned_objname = "remote/object".to_owned(); fi.transition_tier = "WARM".to_owned(); assert_file_corrupt(&fi, ValidationMode::RequireErasure); assert!(!fi.is_valid(), "wire-controlled state flags must not relax is_valid"); assert!(matches!(fi.validate_for_metadata_read(), Err(Error::FileCorrupt))); assert_file_corrupt(&fi, ValidationMode::DeleteOnly); } #[test] fn metadata_read_validation_requires_canonical_delete_marker_shape() { let marker = FileInfo { volume: "bucket".to_string(), name: "object".to_string(), version_id: Some(Uuid::new_v4()), deleted: true, mod_time: Some(OffsetDateTime::now_utc()), ..Default::default() }; marker .validate_for_metadata_read() .expect("canonical delete marker should validate"); let mut missing_time = marker.clone(); missing_time.mod_time = None; assert!(matches!(missing_time.validate_for_metadata_read(), Err(Error::FileCorrupt))); let mut epoch_time = marker.clone(); epoch_time.mod_time = Some(OffsetDateTime::UNIX_EPOCH); assert!(matches!(epoch_time.validate_for_metadata_read(), Err(Error::FileCorrupt))); let mut disguised_payload = marker.clone(); disguised_payload.erasure.data_blocks = 1; assert!(matches!(disguised_payload.validate_for_metadata_read(), Err(Error::FileCorrupt))); let mut marker_with_part = marker; marker_with_part.parts.push(ObjectPartInfo { number: 1, ..Default::default() }); assert!(matches!(marker_with_part.validate_for_metadata_read(), Err(Error::FileCorrupt))); let mut purge_pending = validation_test_fileinfo(); purge_pending.deleted = true; purge_pending .validate_for_metadata_read() .expect("erasure-backed purge-pending objects remain valid payload metadata"); assert!(!purge_pending.is_canonical_delete_marker()); } #[test] fn rename_validation_accepts_only_legacy_assigned_delete_marker_index() { let marker = FileInfo { volume: "bucket".to_string(), name: "object".to_string(), version_id: Some(Uuid::new_v4()), deleted: true, mod_time: Some(OffsetDateTime::now_utc()), ..Default::default() }; let mut legacy_marker = marker.clone(); legacy_marker.erasure.index = 2; assert_eq!(legacy_marker.validate_for_metadata_read(), Err(Error::FileCorrupt)); assert!(legacy_marker.is_legacy_indexed_delete_marker()); legacy_marker.erasure.index = 0; legacy_marker .validate_for_metadata_read() .expect("normalized legacy marker should pass strict metadata validation"); let mut malformed_marker = legacy_marker; malformed_marker.erasure.index = 2; malformed_marker.erasure.data_blocks = 1; assert!(!malformed_marker.is_legacy_indexed_delete_marker()); assert_eq!(malformed_marker.validate_for_metadata_read(), Err(Error::FileCorrupt)); let mut forged_free_marker = marker.clone(); insert_str(&mut forged_free_marker.metadata, SUFFIX_FREE_VERSION, String::new()); assert_eq!(forged_free_marker.validate_for_metadata_read(), Err(Error::FileCorrupt)); let mut free_marker = marker.clone(); insert_str(&mut free_marker.metadata, SUFFIX_FREE_VERSION, String::new()); free_marker.set_tier_free_version(); free_marker.transition_tier = "WARM".to_string(); free_marker.transitioned_objname = "remote/object".to_string(); free_marker .validate_for_metadata_read() .expect("complete tier free-version marker should remain valid"); let mut out_of_range_marker = marker; out_of_range_marker.erasure.index = MAX_ERASURE_SHARDS + 1; assert!(!out_of_range_marker.is_legacy_indexed_delete_marker()); assert_eq!(out_of_range_marker.validate_for_metadata_read(), Err(Error::FileCorrupt)); } #[test] fn delete_operation_accepts_only_the_worker_tier_free_version_request_shape() { let mut worker_request = FileInfo { volume: "bucket".to_string(), name: "object".to_string(), version_id: Some(Uuid::new_v4()), deleted: true, ..Default::default() }; worker_request.set_tier_free_version(); worker_request .validate_for_delete_operation() .expect("the lifecycle worker request shape must remain valid"); let mut timestamped = worker_request.clone(); timestamped.mod_time = Some(OffsetDateTime::now_utc()); assert_eq!(timestamped.validate_for_delete_operation(), Err(Error::FileCorrupt)); let mut with_payload = worker_request.clone(); with_payload.size = 1; assert_eq!(with_payload.validate_for_delete_operation(), Err(Error::FileCorrupt)); let mut with_unrelated_metadata = worker_request; with_unrelated_metadata .metadata .insert("unexpected".to_string(), "value".to_string()); assert_eq!(with_unrelated_metadata.validate_for_delete_operation(), Err(Error::FileCorrupt)); } #[test] fn erasure_and_delete_operation_validation_enforce_basic_collection_bounds() { let mut fi = validation_test_fileinfo(); fi.parts = (1..=10_001) .map(|number| ObjectPartInfo { number, ..Default::default() }) .collect(); assert_file_corrupt(&fi, ValidationMode::RequireErasure); assert_eq!(fi.validate_for_delete_operation(), Err(Error::FileCorrupt)); let mut fi = validation_test_fileinfo(); fi.erasure.checksums = (1..=10_001) .map(|part_number| ChecksumInfo { part_number, ..Default::default() }) .collect(); assert_file_corrupt(&fi, ValidationMode::RequireErasure); assert_eq!(fi.validate_for_delete_operation(), Err(Error::FileCorrupt)); let mut fi = validation_test_fileinfo(); fi.erasure.distribution = vec![1; 17]; assert_file_corrupt(&fi, ValidationMode::RequireErasure); assert_eq!(fi.validate_for_delete_operation(), Err(Error::FileCorrupt)); } #[test] fn validate_accepts_exact_collection_limits() { let mut fi = validation_test_fileinfo(); fi.parts = (1..=10_000) .map(|number| ObjectPartInfo { number, ..Default::default() }) .collect(); fi.erasure.checksums = (1..=10_000) .map(|part_number| ChecksumInfo { part_number, ..Default::default() }) .collect(); fi.validate(ValidationMode::RequireErasure) .expect("exact collection limits must remain valid"); fi.validate_for_delete_operation() .expect("non-marker delete requests must accept exact collection limits"); } #[test] fn erasure_and_delete_operation_validation_reject_checksum_association_errors() { let mut fi = validation_test_fileinfo(); fi.erasure.checksums[0].part_number = 99; assert_file_corrupt(&fi, ValidationMode::RequireErasure); assert_eq!(fi.validate_for_delete_operation(), Err(Error::FileCorrupt)); let mut fi = validation_test_fileinfo(); let duplicate = fi.erasure.checksums[0].clone(); fi.erasure.checksums.push(duplicate); assert_file_corrupt(&fi, ValidationMode::RequireErasure); assert_eq!(fi.validate_for_delete_operation(), Err(Error::FileCorrupt)); } #[test] fn erasure_and_delete_operation_validation_reject_duplicate_parts() { let mut fi = validation_test_fileinfo(); fi.parts[1].number = fi.parts[0].number; fi.erasure.checksums.clear(); assert_file_corrupt(&fi, ValidationMode::RequireErasure); assert_eq!(fi.validate_for_delete_operation(), Err(Error::FileCorrupt)); } #[test] fn write_quorum_distinguishes_purge_pending_payload_from_delete_marker() { let mut purge_pending = FileInfo::new("bucket/object", 5, 1); purge_pending.erasure.index = 1; purge_pending.deleted = true; assert_eq!(purge_pending.write_quorum(4), 5); let marker = FileInfo { deleted: true, mod_time: Some(OffsetDateTime::now_utc()), ..Default::default() }; assert_eq!(marker.write_quorum(4), 4); let malformed_marker = FileInfo { deleted: true, ..Default::default() }; assert_eq!(malformed_marker.write_quorum(4), 4, "invalid marker metadata must never reduce quorum"); } fn small_string_strategy() -> impl Strategy { proptest::string::string_regex("[A-Za-z0-9._/-]{0,16}").expect("small string regex should compile") } fn small_required_string_strategy() -> impl Strategy { proptest::string::string_regex("[A-Za-z0-9._/-]{1,16}").expect("required string regex should compile") } fn bytes_strategy(max_len: usize) -> impl Strategy { vec(any::(), 0..=max_len).prop_map(Bytes::from) } fn uuid_strategy() -> impl Strategy { any::<[u8; 16]>().prop_map(Uuid::from_bytes) } fn optional_uuid_strategy() -> impl Strategy> { proptest::option::of(uuid_strategy()) } fn timestamp_strategy() -> impl Strategy { (0i64..=4_102_444_800i64) .prop_map(|seconds| OffsetDateTime::from_unix_timestamp(seconds).expect("bounded timestamp should be valid")) } fn optional_timestamp_strategy() -> impl Strategy> { proptest::option::of(timestamp_strategy()) } fn hash_algorithm_strategy() -> impl Strategy { prop_oneof![ Just(HashAlgorithm::SHA256), Just(HashAlgorithm::HighwayHash256), Just(HashAlgorithm::HighwayHash256S), Just(HashAlgorithm::HighwayHash256SLegacy), Just(HashAlgorithm::BLAKE2b512), Just(HashAlgorithm::Md5), Just(HashAlgorithm::None), ] } fn checksum_info_strategy() -> impl Strategy { (0usize..=4, hash_algorithm_strategy(), bytes_strategy(32)).prop_map(|(part_number, algorithm, hash)| ChecksumInfo { part_number, algorithm, hash, }) } fn erasure_info_strategy() -> impl Strategy { (1usize..=4, 0usize..=3, 2usize..=4096) .prop_flat_map(|(data_blocks, parity_blocks, block_size)| { let total_blocks = data_blocks + parity_blocks; ( Just(data_blocks), Just(parity_blocks), Just(block_size), 0usize..=total_blocks, vec(1usize..=(total_blocks.max(1)), total_blocks), vec(checksum_info_strategy(), 0..=3), small_required_string_strategy(), ) }) .prop_map( |(data_blocks, parity_blocks, block_size, index, distribution, checksums, algorithm)| ErasureInfo { algorithm, data_blocks, parity_blocks, block_size, index, distribution, checksums, }, ) } fn object_part_info_strategy() -> impl Strategy { ( small_string_strategy(), 0usize..=8, 0usize..=4096, -1_000_000i64..=1_000_000i64, optional_timestamp_strategy(), proptest::option::of(bytes_strategy(16)), proptest::option::of(hash_map(small_string_strategy(), small_string_strategy(), 0..=3)), proptest::option::of(small_string_strategy()), ) .prop_map(|(etag, number, size, actual_size, mod_time, index, checksums, error)| ObjectPartInfo { etag, number, size, actual_size, mod_time, index, checksums, error, }) } fn file_info_strategy() -> impl Strategy { ( ( small_string_strategy(), small_string_strategy(), optional_uuid_strategy(), any::(), any::(), small_string_strategy(), small_string_strategy(), small_string_strategy(), optional_uuid_strategy(), any::(), ), ( optional_uuid_strategy(), optional_timestamp_strategy(), -1_000_000i64..=1_000_000i64, proptest::option::of(any::()), proptest::option::of(any::()), hash_map(small_string_strategy(), small_string_strategy(), 0..=4), vec(object_part_info_strategy(), 0..=3), erasure_info_strategy(), any::(), proptest::option::of(bytes_strategy(32)), ), ( 0usize..=4, optional_timestamp_strategy(), any::(), 0usize..=4, proptest::option::of(bytes_strategy(32)), any::(), any::(), ), ) .prop_map(|(head, middle, tail)| { let ( volume, name, version_id, is_latest, deleted, transition_status, transitioned_objname, transition_tier, transition_version_id, expire_restored, ) = head; let (data_dir, mod_time, size, mode, written_by_version, metadata, parts, erasure, mark_deleted, data) = middle; let (num_versions, successor_mod_time, fresh, idx, checksum, versioned, uses_legacy_checksum) = tail; FileInfo { volume, name, version_id, is_latest, deleted, transition_status, transitioned_objname, transition_tier, transition_version_id, expire_restored, data_dir, mod_time, size, mode, written_by_version, metadata, parts, erasure, mark_deleted, replication_state_internal: None, data, num_versions, successor_mod_time, fresh, idx, checksum, versioned, uses_legacy_checksum, } }) } proptest! { #[test] fn fileinfo_msgpack_round_trips(value in file_info_strategy()) { let encoded = value .marshal_msg() .expect("FileInfo should serialize in the property roundtrip test"); let decoded = FileInfo::unmarshal(&encoded) .expect("FileInfo should deserialize in the property roundtrip test"); prop_assert_eq!(decoded, value); } #[test] fn fileinfo_unmarshal_never_panics(input in vec(any::(), 0..=1024)) { let result = std::panic::catch_unwind(|| FileInfo::unmarshal(&input)); prop_assert!(result.is_ok(), "FileInfo::unmarshal panicked for arbitrary input"); } } #[test] fn replication_info_equals_compares_mark_deleted_and_replication_state() { let base = FileInfo::default(); // Identical (both default) infos are equal. assert!(base.replication_info_equals(&FileInfo::default())); // Differing mark_deleted breaks equality. let marked = FileInfo { mark_deleted: true, ..Default::default() }; assert!(!base.replication_info_equals(&marked)); // Differing replication_state_internal breaks equality (regression guard: // this field used to be ignored by replication_info_equals). let with_state = FileInfo { replication_state_internal: Some(ReplicationState { replicate_decision_str: "arn:aws:s3:::dest".to_string(), ..Default::default() }), ..Default::default() }; assert!(!base.replication_info_equals(&with_state)); // Equal replication states remain equal. let with_state_clone = FileInfo { replication_state_internal: with_state.replication_state_internal.clone(), ..Default::default() }; assert!(with_state.replication_info_equals(&with_state_clone)); } }