gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 16:46:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(rebalance): converge multipart data movement retries (#6057)

Browse Source 
* fix(rebalance): converge multipart data movement retries

* fix(rebalance): harden multipart retry replacement

* fix(rebalance): isolate internal multipart uploads

* test(ecstore): adapt metadata mutation fixtures

* fix(rebalance): preserve transition metadata semantics

* refactor(ecstore): reuse internal metadata matcher

* Revert "refactor(ecstore): reuse internal metadata matcher"

This reverts commit c87ca0328f.

* refactor(rebalance): reuse data movement log constants

* fix(rebalance): isolate migration-owned state

* fix(rebalance): preserve pre-gate retry compatibility
This commit is contained in:
cxymds
2026-08-13 14:12:26 +08:00
committed by GitHub
parent 11eecdc888
commit e11fcfbd08
36 changed files with 6725 additions and 845 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/Cargo.toml
+1 -1
View File
@@ -37,6 +37,7 @@ crc-fast = { workspace = true }
rmp.workspace = true
rmp-serde.workspace = true
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
tokio = { workspace = true, features = ["io-util", "macros", "sync", "fs", "rt-multi-thread"] }
@@ -54,7 +55,6 @@ arc-swap.workspace = true
criterion = { workspace = true, features = ["html_reports"] }
tempfile = { workspace = true }
proptest = "1"
serde_json.workspace = true
[[bench]]
name = "xl_meta_bench"
crates/filemeta/examples/dump_fileinfo.rs
+1
View File
@@ -52,6 +52,7 @@ fn main() {
FileInfoOpts {
data: false,
include_free_versions: true,
include_part_checksums: true,
},
)
.expect("decode file info");
crates/filemeta/src/fileinfo.rs
+21 -3
View File
@@ -17,9 +17,9 @@ use bytes::Bytes;
use rmp_serde::Serializer;
use rustfs_utils::HashAlgorithm;
use rustfs_utils::http::{
SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_FREE_VERSION, SUFFIX_HEALING, SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID,
SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str, has_internal_suffix, insert_str,
is_encryption_metadata_key, starts_with_ignore_ascii_case,
AMZ_OBJECT_TAGGING, SUFFIX_COMPRESSION, SUFFIX_DATA_MOVED, SUFFIX_DATA_MOVED_TAGS, SUFFIX_FREE_VERSION, SUFFIX_HEALING,
SUFFIX_INLINE_DATA, SUFFIX_TIER_FV_ID, SUFFIX_TIER_FV_MARKER, SUFFIX_TIER_SKIP_FV_ID, contains_key_str, get_str,
has_internal_suffix, insert_str, is_encryption_metadata_key, starts_with_ignore_ascii_case,
};
use s3s::dto::{RestoreStatus, Timestamp};
use s3s::header::X_AMZ_RESTORE;
@@ -232,6 +232,17 @@ pub enum TransitionVersionState {
Exact,
}
impl TransitionVersionState {
pub const fn as_str(self) -> &'static str {
match self {
Self::Unknown => "unknown",
Self::KnownDisabled => "known-disabled",
Self::SuspendedNull => "suspended-null",
Self::Exact => "exact",
}
}
}
#[derive(PartialEq, Clone, Default)]
pub struct FileInfo {
pub volume: String,
@@ -1151,9 +1162,16 @@ impl FileInfo {
}
pub fn set_data_moved(&mut self) {
let tags_proof = format!("v1:{}", self.metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str).unwrap_or_default());
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED_TAGS, tags_proof);
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, "true".to_string());
}
pub fn acknowledge_data_movement(&mut self) {
// Keep both empty aliases so mixed-version disks retain one metadata identity.
insert_str(&mut self.metadata, SUFFIX_DATA_MOVED, String::new());
}
pub fn inline_data(&self) -> bool {
contains_key_str(&self.metadata, SUFFIX_INLINE_DATA) && !self.is_remote()
}
crates/filemeta/src/filemeta.rs
+156 -97
View File
@@ -206,6 +206,42 @@ fn persist_reset_statuses(meta_sys: &mut HashMap<String, Vec<u8>>, reset_statuse
}
}
pub fn parse_replication_timestamp(value: &str) -> Option<OffsetDateTime> {
const DISPLAY_FORMAT: &[time::format_description::BorrowedFormatItem<'_>] = time::macros::format_description!(
"[year sign:automatic]-[month]-[day] [hour padding:none]:[minute]:[second].[subsecond] [offset_hour sign:mandatory]:[offset_minute]:[offset_second]"
);
OffsetDateTime::parse(value, &Rfc3339)
.or_else(|_| OffsetDateTime::parse(value, DISPLAY_FORMAT))
.ok()
}
fn format_replication_timestamp(value: Option<OffsetDateTime>) -> String {
let value = value.unwrap_or(OffsetDateTime::UNIX_EPOCH);
value
.to_offset(time::UtcOffset::UTC)
.format(&Rfc3339)
.unwrap_or_else(|_| value.to_string())
}
fn persist_delete_marker_replication_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: &ReplicationState) {
if !state.replica_status.is_empty() {
insert_bytes(meta_sys, SUFFIX_REPLICA_STATUS, state.replica_status.as_str().as_bytes().to_vec());
insert_bytes(
meta_sys,
SUFFIX_REPLICA_TIMESTAMP,
format_replication_timestamp(state.replica_timestamp).into_bytes(),
);
}
if let Some(status) = state.replication_status_internal.as_ref().filter(|status| !status.is_empty()) {
insert_bytes(meta_sys, SUFFIX_REPLICATION_STATUS, status.as_bytes().to_vec());
insert_bytes(
meta_sys,
SUFFIX_REPLICATION_TIMESTAMP,
format_replication_timestamp(state.replication_timestamp).into_bytes(),
);
}
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct FileMeta {
pub versions: Vec<FileMetaShallowVersion>,
@@ -213,6 +249,13 @@ pub struct FileMeta {
pub meta_ver: u8,
}
struct FileInfoDecodeOptions {
read_data: bool,
include_free_versions: bool,
all_parts: bool,
include_part_checksums: bool,
}
impl FileMeta {
pub fn new() -> Self {
Self {
@@ -511,53 +554,8 @@ impl FileMeta {
}
if fi.deleted {
if !fi.delete_marker_replication_status().is_empty()
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
{
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_status.clone())
.unwrap_or_default()
.as_str()
.as_bytes()
.to_vec(),
);
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
} else {
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
}
if let (Some(delete_marker), Some(state)) = (ventry.delete_marker.as_mut(), fi.replication_state_internal.as_ref()) {
persist_delete_marker_replication_state(&mut delete_marker.meta_sys, state);
}
if !fi.version_purge_status().is_empty()
@@ -609,51 +607,8 @@ impl FileMeta {
}
if let Some(delete_marker) = v.delete_marker.as_mut() {
if !fi.delete_marker_replication_status().is_empty() {
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_status.clone())
.unwrap_or_default()
.as_str()
.as_bytes()
.to_vec(),
);
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICA_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replica_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
} else {
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_STATUS,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_status_internal.clone().unwrap_or_default())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
insert_bytes(
&mut delete_marker.meta_sys,
SUFFIX_REPLICATION_TIMESTAMP,
fi.replication_state_internal
.as_ref()
.map(|v| v.replication_timestamp.unwrap_or(OffsetDateTime::UNIX_EPOCH).to_string())
.unwrap_or_default()
.as_bytes()
.to_vec(),
);
}
if let Some(state) = fi.replication_state_internal.as_ref() {
persist_delete_marker_replication_state(&mut delete_marker.meta_sys, state);
}
if let Some(state) = fi.replication_state_internal.as_ref() {
@@ -773,6 +728,47 @@ impl FileMeta {
read_data: bool,
include_free_versions: bool,
all_parts: bool,
) -> Result<FileInfo> {
self.to_fileinfo_with_part_checksums(
volume,
path,
version_id,
FileInfoDecodeOptions {
read_data,
include_free_versions,
all_parts,
include_part_checksums: true,
},
)
}
pub fn into_fileinfo_without_part_checksums(
&self,
volume: &str,
path: &str,
version_id: &str,
read_data: bool,
include_free_versions: bool,
) -> Result<FileInfo> {
self.to_fileinfo_with_part_checksums(
volume,
path,
version_id,
FileInfoDecodeOptions {
read_data,
include_free_versions,
all_parts: true,
include_part_checksums: false,
},
)
}
fn to_fileinfo_with_part_checksums(
&self,
volume: &str,
path: &str,
version_id: &str,
opts: FileInfoDecodeOptions,
) -> Result<FileInfo> {
let vid = {
if !version_id.is_empty() {
@@ -795,7 +791,7 @@ impl FileMeta {
if header.free_version() {
non_free_versions -= 1;
if include_free_versions
if opts.include_free_versions
&& found_free_version.is_none()
&& let Ok(found_free_fi) = ver.parse_version_meta()
&& found_free_fi.version_type != VersionType::Invalid
@@ -806,7 +802,8 @@ impl FileMeta {
// Known side effect: if a disk holds only free versions and they are
// corrupt, `into_fileinfo` falls through to `FileNotFound` (not
// `FileCorrupt`), so that disk is not enqueued for heal.
match found_free_fi.into_fileinfo(volume, path, all_parts) {
match found_free_fi.to_fileinfo_with_part_checksums(volume, path, opts.all_parts, opts.include_part_checksums)
{
Ok(mut free_fi) => {
free_fi.is_latest = true;
found_free_version = Some(free_fi);
@@ -834,14 +831,14 @@ impl FileMeta {
found = true;
let mut fi = ver.into_fileinfo(volume, path, all_parts)?;
let mut fi = ver.to_fileinfo_with_part_checksums(volume, path, opts.all_parts, opts.include_part_checksums)?;
fi.is_latest = is_latest;
if let Some(_d) = succ_mod_time {
fi.successor_mod_time = succ_mod_time;
}
if read_data && fi.inline_data() {
if opts.read_data && fi.inline_data() {
fi.data = self.find_inline_data_for_version(fi.version_id)?.map(bytes::Bytes::from);
}
@@ -850,7 +847,7 @@ impl FileMeta {
if !found {
if version_id.is_empty() {
if include_free_versions
if opts.include_free_versions
&& non_free_versions == 0
&& let Some(free_version) = found_free_version
{
@@ -1537,6 +1534,68 @@ mod test {
assert_eq!(meta_sys2.len(), 2, "must not create a double-prefixed key");
}
#[test]
fn persist_delete_marker_replication_state_keeps_replica_and_target_statuses() {
let replica_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
let replication_timestamp = replica_timestamp + time::Duration::SECOND;
let state = ReplicationState {
replica_status: ReplicationStatusType::Replica,
replica_timestamp: Some(replica_timestamp),
replication_status_internal: Some("arn:target=COMPLETED;".to_string()),
replication_timestamp: Some(replication_timestamp),
..Default::default()
};
let mut meta_sys = HashMap::new();
let replica_timestamp_string = replica_timestamp
.format(&Rfc3339)
.expect("timestamp should format as RFC3339");
let replication_timestamp_string = replication_timestamp
.format(&Rfc3339)
.expect("timestamp should format as RFC3339");
persist_delete_marker_replication_state(&mut meta_sys, &state);
assert_eq!(
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICA_STATUS).as_deref(),
Some(b"REPLICA".as_slice())
);
assert_eq!(
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICA_TIMESTAMP).as_deref(),
Some(replica_timestamp_string.as_bytes())
);
assert_eq!(
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICATION_STATUS).as_deref(),
Some(b"arn:target=COMPLETED;".as_slice())
);
assert_eq!(
rustfs_utils::http::get_bytes(&meta_sys, SUFFIX_REPLICATION_TIMESTAMP).as_deref(),
Some(replication_timestamp_string.as_bytes())
);
}
#[test]
fn persist_delete_marker_replication_timestamp_normalizes_second_offset_to_utc() {
let timestamp = OffsetDateTime::UNIX_EPOCH.to_offset(time::UtcOffset::from_hms(5, 30, 15).expect("valid offset"));
assert_eq!(parse_replication_timestamp(&timestamp.to_string()), Some(timestamp));
let state = ReplicationState {
replica_status: ReplicationStatusType::Replica,
replica_timestamp: Some(timestamp),
replication_status_internal: Some("arn:target=COMPLETED;".to_string()),
replication_timestamp: Some(timestamp),
..Default::default()
};
let mut meta_sys = HashMap::new();
persist_delete_marker_replication_state(&mut meta_sys, &state);
for suffix in [SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_TIMESTAMP] {
let persisted = rustfs_utils::http::get_bytes(&meta_sys, suffix).expect("timestamp must be persisted");
let persisted = std::str::from_utf8(&persisted).expect("timestamp must be UTF-8");
assert_eq!(parse_replication_timestamp(persisted), Some(timestamp));
assert_ne!(persisted, OffsetDateTime::UNIX_EPOCH.to_string());
}
}
/// Regression test for rustfs/rustfs#2715: a corrupted version count in
/// xl.meta must yield a decode error instead of sizing a huge allocation
/// from the bogus count (which aborts the whole process).
crates/filemeta/src/filemeta/version.rs
+603 -120
View File
@@ -29,11 +29,12 @@ use super::*;
use crate::{ChecksumInfo, TransitionVersionState};
use rustfs_utils::HashAlgorithm;
use rustfs_utils::http::{
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID,
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PART_CHECKSUMS, SUFFIX_PURGESTATUS,
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX, SUFFIX_TIER_FV_ID,
SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
strip_internal_prefix, target_delete_marker_versions,
strip_internal_prefix, strip_internal_prefix_preserving_case, target_delete_marker_versions,
};
const MSGPACK_EXT8: u8 = 0xc7;
@@ -258,63 +259,181 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result<Option<Uuid>> {
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
/// malformed bytes are not usable remote versions.
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Result<Option<String>> {
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) {
return Ok(None);
}
let Some(value) = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) else {
return Ok(None);
fn transition_version_state_from_bytes(value: Option<&[u8]>) -> Result<TransitionVersionState> {
let Some(value) = value else {
return Ok(TransitionVersionState::Unknown);
};
let value = value.to_vec();
match value {
b"known-disabled" => Ok(TransitionVersionState::KnownDisabled),
b"suspended-null" => Ok(TransitionVersionState::SuspendedNull),
b"exact" => Ok(TransitionVersionState::Exact),
b"unknown" => Ok(TransitionVersionState::Unknown),
_ => Err(Error::FileCorrupt),
}
}
fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersionState) -> Option<String> {
let value = value?;
if value.is_empty() {
return Ok(None);
return None;
}
if let Ok(id) = Uuid::from_slice(&value) {
return Ok((!id.is_nil()).then(|| id.to_string()));
if state == TransitionVersionState::Unknown
&& let Ok(id) = Uuid::from_slice(value)
{
return (!id.is_nil()).then(|| id.to_string());
}
let Ok(value) = String::from_utf8(value) else {
return Ok(None);
let Ok(value) = std::str::from_utf8(value) else {
return None;
};
if value.is_empty()
|| value.len() > MAX_TRANSITION_VERSION_LEN
|| value.chars().any(char::is_control)
|| Uuid::parse_str(&value).is_ok_and(|id| id.is_nil())
|| Uuid::parse_str(value).is_ok_and(|id| id.is_nil())
{
Ok(None)
None
} else {
Ok(Some(value))
Some(value.to_string())
}
}
fn transition_version_state_from_meta_sys(
meta_sys: &HashMap<String, Vec<u8>>,
version: Option<&str>,
) -> Result<TransitionVersionState> {
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
return Ok(TransitionVersionState::Unknown);
}
let value = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or(Error::FileCorrupt)?;
let state = match value {
b"known-disabled" => TransitionVersionState::KnownDisabled,
b"suspended-null" => TransitionVersionState::SuspendedNull,
b"exact" => TransitionVersionState::Exact,
b"unknown" => TransitionVersionState::Unknown,
_ => return Err(Error::FileCorrupt),
};
fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> {
let valid = match state {
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
TransitionVersionState::SuspendedNull => version == Some("null"),
TransitionVersionState::Exact => version.is_some_and(|value| value != "null"),
};
valid.then_some(state).ok_or(Error::FileCorrupt)
valid.then_some(()).ok_or(Error::FileCorrupt)
}
fn transition_version_state_bytes(state: TransitionVersionState) -> &'static [u8] {
match state {
TransitionVersionState::Unknown => b"unknown",
TransitionVersionState::KnownDisabled => b"known-disabled",
TransitionVersionState::SuspendedNull => b"suspended-null",
TransitionVersionState::Exact => b"exact",
#[derive(Default)]
struct DerivedInternalMetadata<'a> {
checksum: Option<&'a [u8]>,
part_checksums: Option<&'a [u8]>,
transition_status: Option<&'a [u8]>,
transitioned_object: Option<&'a [u8]>,
transitioned_version: Option<&'a [u8]>,
transitioned_version_state: Option<&'a [u8]>,
transition_tier: Option<&'a [u8]>,
}
impl<'a> DerivedInternalMetadata<'a> {
fn from_meta_sys(meta_sys: &'a HashMap<String, Vec<u8>>) -> Result<Self> {
let mut canonical = Self::default();
let mut legacy = Self::default();
for (key, value) in meta_sys {
let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
continue;
};
let (canonical_slot, legacy_slot, expected_suffix) = if suffix.eq_ignore_ascii_case(SUFFIX_CRC) {
(&mut canonical.checksum, &mut legacy.checksum, SUFFIX_CRC)
} else if suffix.eq_ignore_ascii_case(SUFFIX_PART_CHECKSUMS) {
(&mut canonical.part_checksums, &mut legacy.part_checksums, SUFFIX_PART_CHECKSUMS)
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITION_STATUS) {
(&mut canonical.transition_status, &mut legacy.transition_status, SUFFIX_TRANSITION_STATUS)
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_OBJECTNAME) {
(
&mut canonical.transitioned_object,
&mut legacy.transitioned_object,
SUFFIX_TRANSITIONED_OBJECTNAME,
)
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID) {
(
&mut canonical.transitioned_version,
&mut legacy.transitioned_version,
SUFFIX_TRANSITIONED_VERSION_ID,
)
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_STATE) {
(
&mut canonical.transitioned_version_state,
&mut legacy.transitioned_version_state,
SUFFIX_TRANSITIONED_VERSION_STATE,
)
} else if suffix.eq_ignore_ascii_case(SUFFIX_TRANSITION_TIER) {
(&mut canonical.transition_tier, &mut legacy.transition_tier, SUFFIX_TRANSITION_TIER)
} else {
continue;
};
let slot = if suffix == expected_suffix
&& (key.starts_with(RUSTFS_INTERNAL_PREFIX) || key.starts_with(rustfs_utils::http::MINIO_INTERNAL_PREFIX))
{
canonical_slot
} else {
legacy_slot
};
if slot.is_some_and(|current| current != value.as_slice()) {
return Err(Error::FileCorrupt);
}
*slot = Some(value.as_slice());
}
Ok(Self {
checksum: canonical.checksum.or(legacy.checksum),
part_checksums: canonical.part_checksums.or(legacy.part_checksums),
transition_status: canonical.transition_status.or(legacy.transition_status),
transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object),
transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version),
transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state),
transition_tier: canonical.transition_tier.or(legacy.transition_tier),
})
}
}
struct UniquePartChecksums(HashMap<String, String>);
impl<'de> serde::Deserialize<'de> for UniquePartChecksums {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct UniquePartChecksumsVisitor;
impl<'de> serde::de::Visitor<'de> for UniquePartChecksumsVisitor {
type Value = UniquePartChecksums;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("an array of unique checksum name and value pairs")
}
fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'de>,
{
let mut checksums = HashMap::with_capacity(seq.size_hint().unwrap_or_default());
while let Some((key, value)) = seq.next_element::<(String, String)>()? {
if checksums.insert(key, value).is_some() {
return Err(serde::de::Error::custom("duplicate part checksum name"));
}
}
Ok(UniquePartChecksums(checksums))
}
}
deserializer.deserialize_seq(UniquePartChecksumsVisitor)
}
}
impl FileInfo {
pub fn hydrate_data_movement_part_checksums(&mut self) -> Result<()> {
let present = self
.metadata
.keys()
.any(|key| has_internal_suffix(key, SUFFIX_PART_CHECKSUMS));
if !present {
return Ok(());
}
let encoded = rustfs_utils::http::get_consistent_str(&self.metadata, SUFFIX_PART_CHECKSUMS).ok_or(Error::FileCorrupt)?;
let persisted = serde_json::from_str::<Vec<(usize, UniquePartChecksums)>>(encoded).map_err(|_| Error::FileCorrupt)?;
let mut part_indices = HashMap::with_capacity(self.parts.len());
for (index, part) in self.parts.iter().enumerate() {
if part_indices.insert(part.number, index).is_some() {
return Err(Error::FileCorrupt);
}
}
for (part_number, UniquePartChecksums(checksums)) in persisted {
let index = part_indices.remove(&part_number).ok_or(Error::FileCorrupt)?;
let part = self.parts.get_mut(index).ok_or(Error::FileCorrupt)?;
part.checksums = Some(checksums);
}
Ok(())
}
}
@@ -322,21 +441,10 @@ fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state:
if state == TransitionVersionState::Unknown {
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
} else {
insert_bytes(
meta_sys,
SUFFIX_TRANSITIONED_VERSION_STATE,
transition_version_state_bytes(state).to_vec(),
);
insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec());
}
}
fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
transitioned_version_from_meta_sys(meta_sys)
.ok()
.flatten()
.and_then(|value| Uuid::parse_str(&value).ok())
}
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
fi.transition_version
.as_ref()
@@ -448,6 +556,17 @@ impl FileMetaShallowVersion {
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
self.parse_version_meta()?.into_fileinfo(volume, path, all_parts)
}
pub(super) fn to_fileinfo_with_part_checksums(
&self,
volume: &str,
path: &str,
all_parts: bool,
include_part_checksums: bool,
) -> Result<FileInfo> {
self.parse_version_meta()?
.to_fileinfo_with_part_checksums(volume, path, all_parts, include_part_checksums)
}
}
impl TryFrom<FileMetaVersion> for FileMetaShallowVersion {
@@ -769,8 +888,16 @@ impl FileMetaVersion {
}
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
// Only the Object arm carries part arrays and can fail the length guard; the
// Legacy and Delete arms have no part arrays and stay infallible.
self.to_fileinfo_with_part_checksums(volume, path, all_parts, true)
}
pub(super) fn to_fileinfo_with_part_checksums(
&self,
volume: &str,
path: &str,
all_parts: bool,
include_part_checksums: bool,
) -> Result<FileInfo> {
let mut fi = match self.version_type {
VersionType::Invalid | VersionType::Legacy => {
if let Some(ref legacy) = self.legacy_object {
@@ -788,14 +915,14 @@ impl FileMetaVersion {
self.object
.as_ref()
.unwrap_or(&default_object)
.into_fileinfo(volume, path, all_parts)?
.to_fileinfo_with_part_checksums(volume, path, all_parts, include_part_checksums)?
}
VersionType::Delete => {
let default_marker = MetaDeleteMarker::default();
self.delete_marker
.as_ref()
.unwrap_or(&default_marker)
.into_fileinfo(volume, path, all_parts)
.into_fileinfo(volume, path, all_parts)?
}
};
fi.uses_legacy_checksum = self.uses_legacy_checksum;
@@ -2390,7 +2517,18 @@ impl MetaObject {
}
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
self.to_fileinfo_with_part_checksums(volume, path, all_parts, true)
}
fn to_fileinfo_with_part_checksums(
&self,
volume: &str,
path: &str,
all_parts: bool,
include_part_checksums: bool,
) -> Result<FileInfo> {
let version_id = self.version_id.filter(|&vid| !vid.is_nil());
let derived_metadata = DerivedInternalMetadata::from_meta_sys(&self.meta_sys)?;
let parts = if all_parts {
let n = self.part_numbers.len();
@@ -2474,7 +2612,10 @@ impl MetaObject {
}
}
let checksum = get_bytes(&self.meta_sys, SUFFIX_CRC).map(Bytes::from);
let checksum = derived_metadata
.checksum
.filter(|checksum| !checksum.is_empty())
.map(Bytes::copy_from_slice);
let erasure = ErasureInfo {
algorithm: self.erasure_algorithm.to_string(),
@@ -2486,20 +2627,29 @@ impl MetaObject {
..Default::default()
};
let transition_status = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_STATUS)
.map(|v| String::from_utf8_lossy(&v).to_string())
let transition_status = derived_metadata
.transition_status
.filter(|value| !value.is_empty())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
.map(|v| String::from_utf8_lossy(&v).to_string())
let transitioned_objname = derived_metadata
.transitioned_object
.filter(|value| !value.is_empty())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys)?;
let transition_version_state = transition_version_state_from_meta_sys(&self.meta_sys, transition_version.as_deref())?;
let transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
let transition_version = transitioned_version_from_bytes(derived_metadata.transitioned_version, transition_version_state);
if derived_metadata.transitioned_version_state.is_some() {
validate_transition_version_state(transition_version_state, transition_version.as_deref())?;
}
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
.map(|v| String::from_utf8_lossy(&v).to_string())
let transition_tier = derived_metadata
.transition_tier
.filter(|value| !value.is_empty())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
Ok(FileInfo {
let mut file_info = FileInfo {
version_id,
erasure,
data_dir: self.data_dir,
@@ -2519,7 +2669,11 @@ impl MetaObject {
transition_version_state,
transition_tier,
..Default::default()
})
};
if all_parts && include_part_checksums {
file_info.hydrate_data_movement_part_checksums()?;
}
Ok(file_info)
}
pub fn set_transition(&mut self, fi: &FileInfo) {
@@ -2710,39 +2864,31 @@ fn get_internal_replication_state(metadata: &HashMap<String, String>) -> Option<
continue;
}
let sub_key_opt = strip_internal_prefix(k);
if let Some(ref sub_key) = sub_key_opt {
match sub_key.as_str() {
"replica-timestamp" => {
has = true;
rs.replica_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
}
"replica-status" => {
has = true;
rs.replica_status = ReplicationStatusType::from(v.as_str());
}
"replication-timestamp" => {
has = true;
rs.replication_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH))
}
"replication-status" => {
has = true;
rs.replication_status_internal = Some(v.clone());
rs.targets = replication_statuses_map(v.as_str());
}
_ => {
if let Some(arn) = sub_key.strip_prefix("replication-reset-") {
has = true;
// Store the canonical full-header key so the map matches
// the key `target_reset_header()` produces on the
// write/lookup side. Storing the bare ARN keyed the map
// inconsistently (bare on read, full on write), which
// could drop reset state across merge/reflatten cycles
// (backlog#799 B16).
rs.reset_statuses_map
.insert(crate::replication::target_reset_header(arn), v.clone());
}
}
if let Some(sub_key) = strip_internal_prefix_preserving_case(k) {
if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICA_TIMESTAMP) {
has = true;
rs.replica_timestamp = Some(parse_replication_timestamp(v).unwrap_or(OffsetDateTime::UNIX_EPOCH));
} else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICA_STATUS) {
has = true;
rs.replica_status = ReplicationStatusType::from(v.as_str());
} else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICATION_TIMESTAMP) {
has = true;
rs.replication_timestamp = Some(parse_replication_timestamp(v).unwrap_or(OffsetDateTime::UNIX_EPOCH))
} else if sub_key.eq_ignore_ascii_case(SUFFIX_REPLICATION_STATUS) {
has = true;
rs.replication_status_internal = Some(v.clone());
rs.targets = replication_statuses_map(v.as_str());
} else if let Some(arn) = rustfs_utils::http::internal_key_strip_suffix_prefix(k, SUFFIX_REPLICATION_RESET_ARN_PREFIX)
{
has = true;
// Store the canonical full-header key so the map matches
// the key `target_reset_header()` produces on the
// write/lookup side. Storing the bare ARN keyed the map
// inconsistently (bare on read, full on write), which
// could drop reset state across merge/reflatten cycles
// (backlog#799 B16).
rs.reset_statuses_map
.insert(crate::replication::target_reset_header(&arn), v.clone());
}
}
}
@@ -2786,7 +2932,7 @@ impl MetaDeleteMarker {
contains_key_bytes(&self.meta_sys, SUFFIX_FREE_VERSION)
}
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo {
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> Result<FileInfo> {
let metadata = self
.meta_sys
.clone()
@@ -2808,22 +2954,27 @@ impl MetaDeleteMarker {
if self.free_version() {
fi.set_tier_free_version();
fi.transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
.map(|v| String::from_utf8_lossy(&v).to_string())
let derived_metadata = DerivedInternalMetadata::from_meta_sys(&self.meta_sys)?;
fi.transition_tier = derived_metadata
.transition_tier
.filter(|value| !value.is_empty())
.map(|value| String::from_utf8_lossy(value).to_string())
.unwrap_or_default();
fi.transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
.map(|v| String::from_utf8_lossy(&v).to_string())
fi.transitioned_objname = derived_metadata
.transitioned_object
.filter(|value| !value.is_empty())
.map(|value| String::from_utf8_lossy(value).to_string())
.unwrap_or_default();
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys).ok().flatten();
fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys);
fi.transition_version_state =
transition_version_state_from_meta_sys(&self.meta_sys, fi.transition_version.as_deref())
.unwrap_or(TransitionVersionState::Unknown);
fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
fi.transition_version =
transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state);
fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
if derived_metadata.transitioned_version_state.is_some() {
validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?;
}
}
fi
Ok(fi)
}
pub fn encode_to<W: std::io::Write>(&self, wr: &mut W) -> Result<()> {
@@ -2941,6 +3092,13 @@ impl From<FileInfo> for MetaDeleteMarker {
if !is_internal_key(key) || is_skip_meta_key(key) {
continue;
}
if rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_REPLICATION_RESET_ARN_PREFIX).is_some()
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX)
.is_some()
{
meta_sys.insert(key.clone(), metadata_value.as_bytes().to_vec());
continue;
}
let Some(suffix) = strip_internal_prefix(key) else {
continue;
};
@@ -2982,6 +3140,11 @@ impl From<FileInfo> for MetaDeleteMarker {
if !value.transition_tier.is_empty() {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
}
if let Some(state) = value.replication_state_internal.as_ref() {
persist_delete_marker_replication_state(&mut meta_sys, state);
persist_reset_statuses(&mut meta_sys, &state.reset_statuses_map);
persist_target_delete_marker_versions(&mut meta_sys, &state.target_delete_marker_version_ids, &value.metadata);
}
Self {
version_id: value.version_id,
mod_time: value.mod_time,
@@ -3278,6 +3441,7 @@ pub fn file_info_from_raw(
FileInfoOpts {
data: read_data,
include_free_versions,
include_part_checksums: true,
},
)
}
@@ -3285,6 +3449,7 @@ pub fn file_info_from_raw(
pub struct FileInfoOpts {
pub data: bool,
pub include_free_versions: bool,
pub include_part_checksums: bool,
}
pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result<FileInfo> {
@@ -3309,7 +3474,11 @@ pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opt
});
}
let fi = meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)?;
let fi = if opts.include_part_checksums {
meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)?
} else {
meta.into_fileinfo_without_part_checksums(volume, path, version_id, opts.data, opts.include_free_versions)?
};
Ok(fi)
}
@@ -3555,6 +3724,56 @@ mod tests {
assert!(!converted.meta_sys.contains_key("content-type"));
}
#[test]
fn delete_marker_conversion_does_not_lowercase_dynamic_replication_targets() {
let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket";
let reset_suffix = format!("{SUFFIX_REPLICATION_RESET_ARN_PREFIX}{arn}");
let version_suffix = format!("{SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX}{arn}");
let mut marker = FileInfo::default();
marker.metadata.insert(
format!("{}{reset_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
"2026-08-12T00:00:00Z;COMPLETED".to_string(),
);
marker.metadata.insert(
format!("{}{version_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
"remote-version".to_string(),
);
marker.replication_state_internal = get_internal_replication_state(&marker.metadata);
let converted = MetaDeleteMarker::from(marker);
assert!(converted.meta_sys.keys().any(|key| key.ends_with(&reset_suffix)));
assert!(converted.meta_sys.keys().any(|key| key.ends_with(&version_suffix)));
assert!(!converted.meta_sys.keys().any(|key| key.contains("tenanta")));
let mut conflicting = FileInfo::default();
conflicting
.metadata
.insert(format!("{RUSTFS_INTERNAL_PREFIX}{version_suffix}"), "remote-version-a".to_string());
conflicting.metadata.insert(
format!("{}{version_suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
"remote-version-b".to_string(),
);
conflicting.replication_state_internal = get_internal_replication_state(&conflicting.metadata);
assert!(
conflicting
.replication_state_internal
.as_ref()
.is_some_and(|state| state.target_delete_marker_version_ids_corrupt)
);
let roundtrip = MetaDeleteMarker::from(conflicting)
.into_fileinfo("bucket", "object", false)
.expect("dynamic replication aliases should remain decodable");
assert!(
roundtrip
.replication_state_internal
.as_ref()
.is_some_and(|state| state.target_delete_marker_version_ids_corrupt),
"conflicting dynamic aliases must remain corrupt across persistence"
);
}
#[derive(Serialize)]
enum LegacyDeleteVersionTypeFixture {
#[serde(rename = "DeleteMarker")]
@@ -3663,6 +3882,138 @@ mod tests {
assert!(matches!(res, Err(Error::FileCorrupt)), "short part_sizes must map to FileCorrupt");
}
#[test]
fn into_fileinfo_rejects_conflicting_derived_internal_aliases() {
for suffix in [
SUFFIX_CRC,
SUFFIX_TRANSITION_STATUS,
SUFFIX_TRANSITIONED_OBJECTNAME,
SUFFIX_TRANSITIONED_VERSION_ID,
SUFFIX_TRANSITIONED_VERSION_STATE,
SUFFIX_TRANSITION_TIER,
] {
let mut meta_sys = HashMap::from([(format!("{RUSTFS_INTERNAL_PREFIX}{suffix}"), vec![0xff, 1])]);
meta_sys.insert(format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX), vec![0xfe, 2]);
let object = MetaObject {
meta_sys,
..Default::default()
};
assert_eq!(
object
.into_fileinfo("bucket", "key", false)
.expect_err("conflicting aliases must fail closed"),
Error::FileCorrupt,
"suffix {suffix}"
);
}
}
#[test]
fn into_fileinfo_recovers_noncanonical_binary_checksum_alias() {
let checksum = vec![0xff, 0x00, 0x80, 0x01];
let object = MetaObject {
meta_sys: HashMap::from([("X-Minio-Internal-crc".to_string(), checksum.clone())]),
..Default::default()
};
let file_info = object
.into_fileinfo("bucket", "key", false)
.expect("a single legacy checksum alias should remain readable");
assert_eq!(file_info.checksum.as_deref(), Some(checksum.as_slice()));
}
#[test]
fn into_fileinfo_prefers_canonical_rewrite_over_stale_mixed_case_alias() {
let checksum = vec![0xff, 0x00, 0x80, 0x01];
let mut object = MetaObject {
meta_sys: HashMap::from([("X-Minio-Internal-crc".to_string(), b"stale".to_vec())]),
..Default::default()
};
insert_bytes(&mut object.meta_sys, SUFFIX_CRC, checksum.clone());
let file_info = object
.into_fileinfo("bucket", "key", false)
.expect("canonical rewrites should supersede legacy mixed-case aliases");
assert_eq!(file_info.checksum.as_deref(), Some(checksum.as_slice()));
}
#[test]
fn into_fileinfo_recovers_data_movement_part_checksums() {
let mut object = object_with_parts(vec![1], vec![16], vec![16]);
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, br#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_vec());
let file_info = object
.into_fileinfo("bucket", "key", true)
.expect("data movement part checksums should decode");
assert_eq!(
file_info.parts[0]
.checksums
.as_ref()
.and_then(|checksums| checksums.get("CRC32C"))
.map(String::as_str),
Some("AAAAAA==")
);
let mut deferred = object
.to_fileinfo_with_part_checksums("bucket", "key", true, false)
.expect("quorum candidates should retain raw checksum metadata");
assert!(deferred.parts[0].checksums.is_none());
deferred
.hydrate_data_movement_part_checksums()
.expect("the selected candidate should hydrate checksums once");
assert_eq!(deferred.parts[0].checksums, file_info.parts[0].checksums);
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, b"not-json".to_vec());
assert_eq!(
object
.into_fileinfo("bucket", "key", true)
.expect_err("malformed data movement part checksums must fail closed"),
Error::FileCorrupt
);
for encoded in [
br#"[[1,[["CRC32C","AAAAAA=="]]],[1,[["CRC32C","BBBBBB=="]]]]"#.as_slice(),
br#"[[1,[["CRC32C","AAAAAA=="],["CRC32C","BBBBBB=="]]]]"#.as_slice(),
] {
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, encoded.to_vec());
assert_eq!(
object
.into_fileinfo("bucket", "key", true)
.expect_err("duplicate part checksum keys must fail closed"),
Error::FileCorrupt
);
}
insert_bytes(&mut object.meta_sys, SUFFIX_PART_CHECKSUMS, br#"[[2,[["CRC32C","AAAAAA=="]]]]"#.to_vec());
assert_eq!(
object
.into_fileinfo("bucket", "key", true)
.expect_err("a sidecar for an unknown part must fail closed"),
Error::FileCorrupt
);
object.meta_sys = HashMap::from([
(
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_PART_CHECKSUMS}"),
br#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_vec(),
),
(
format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, SUFFIX_PART_CHECKSUMS),
br#"[[1,[["CRC32C","BBBBBB=="]]]]"#.to_vec(),
),
]);
assert_eq!(
object
.into_fileinfo("bucket", "key", true)
.expect_err("conflicting sidecar aliases must fail closed"),
Error::FileCorrupt
);
}
#[test]
fn into_fileinfo_rejects_short_part_actual_sizes_including_empty() {
let obj = object_with_parts(vec![1, 2], vec![10, 20], vec![]);
@@ -4230,6 +4581,31 @@ mod tests {
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
}
#[test]
fn meta_object_transition_exact_rejects_legacy_raw_uuid_encoding() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, sample_version_id().as_bytes().to_vec());
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_STATE, b"exact".to_vec());
let err = make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect_err("exact remote versions must use their UTF-8 provider representation");
assert_eq!(err, Error::FileCorrupt);
}
#[test]
fn meta_object_transition_version_id_mixed_case_alias_is_recovered() {
let id = sample_version_id();
let sys = HashMap::from([("X-Minio-Internal-transitioned-versionID".to_string(), id.as_bytes().to_vec())]);
let fi = make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect("a legacy mixed-case transition version alias should decode");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
}
#[test]
fn meta_object_transition_version_id_opaque_text_is_preserved() {
let mut sys = HashMap::new();
@@ -4271,8 +4647,10 @@ mod tests {
.map(Vec::as_slice),
Some(b"exact".as_slice())
);
let persisted_version = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
assert_eq!(
legacy_transitioned_version_id_from_meta_sys(&object.meta_sys),
transitioned_version_from_bytes(persisted_version, TransitionVersionState::Unknown)
.and_then(|value| Uuid::parse_str(&value).ok()),
Some(id),
"UUID exact writes must remain readable by the legacy UUID consumer"
);
@@ -4281,6 +4659,26 @@ mod tests {
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str()));
}
#[test]
fn meta_object_transition_version_state_exact_preserves_sixteen_byte_opaque_text() {
let expected_version = "opaque.wasabi_01";
assert_eq!(expected_version.len(), 16);
let fi = FileInfo {
transition_status: "complete".to_string(),
transition_version: Some(expected_version.to_string()),
transition_version_state: TransitionVersionState::Exact,
..Default::default()
};
let decoded = MetaObject::from(fi)
.into_fileinfo("b", "k", false)
.expect("exact opaque transition version should round trip");
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version));
assert_eq!(decoded.transition_version_id, None);
assert_eq!(decoded.transition_version_state, TransitionVersionState::Exact);
}
#[test]
fn set_transition_known_disabled_removes_stale_version_dual_keys() {
let mut meta_sys = HashMap::new();
@@ -4375,7 +4773,8 @@ mod tests {
mod_time: None,
meta_sys: sys,
}
.into_fileinfo("b", "k", false);
.into_fileinfo("b", "k", false)
.expect("nil tier version should remain an absent remote version");
assert_eq!(fi.transition_version_id, None);
}
@@ -4390,7 +4789,8 @@ mod tests {
mod_time: None,
meta_sys: sys,
}
.into_fileinfo("b", "k", false);
.into_fileinfo("b", "k", false)
.expect("legacy binary UUID tier version should decode");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
}
@@ -4407,7 +4807,8 @@ mod tests {
mod_time: Some(sample_mod_time()),
meta_sys: sys,
}
.into_fileinfo("b", "k", false);
.into_fileinfo("b", "k", false)
.expect("opaque tier version should remain readable");
assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
@@ -4429,12 +4830,67 @@ mod tests {
mod_time: Some(sample_mod_time()),
meta_sys: sys,
}
.into_fileinfo("b", "k", false);
.into_fileinfo("b", "k", false)
.expect("mixed-case tier aliases should decode");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
}
#[test]
fn delete_marker_free_version_recovers_mixed_case_transition_aliases() {
let id = sample_version_id();
let id_text = id.to_string();
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
for (suffix, value) in [
(SUFFIX_TRANSITIONED_VERSION_ID, id_text.as_bytes()),
(SUFFIX_TRANSITIONED_VERSION_STATE, b"exact".as_slice()),
(SUFFIX_TRANSITION_TIER, b"WARM".as_slice()),
(SUFFIX_TRANSITIONED_OBJECTNAME, b"remote-object".as_slice()),
] {
sys.insert(format!("X-Minio-Internal-{suffix}"), value.to_vec());
}
let fi = MetaDeleteMarker {
version_id: Some(sample_version_id()),
mod_time: Some(sample_mod_time()),
meta_sys: sys,
}
.into_fileinfo("b", "k", false)
.expect("mixed-case tier aliases should decode");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
assert_eq!(fi.transition_version_state, TransitionVersionState::Exact);
assert_eq!(fi.transition_tier, "WARM");
assert_eq!(fi.transitioned_objname, "remote-object");
}
#[test]
fn delete_marker_free_version_rejects_conflicting_transition_aliases() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
sys.insert(
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}"),
b"source-version".to_vec(),
);
sys.insert(
format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, SUFFIX_TRANSITIONED_VERSION_ID),
b"target-version".to_vec(),
);
let err = MetaDeleteMarker {
version_id: Some(sample_version_id()),
mod_time: Some(sample_mod_time()),
meta_sys: sys,
}
.into_fileinfo("b", "k", false)
.expect_err("conflicting transition aliases must fail closed");
assert_eq!(err, Error::FileCorrupt);
}
#[test]
fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
let object = FileMetaVersionHeader {
@@ -4758,7 +5214,7 @@ mod tests {
#[test]
fn target_delete_marker_version_metadata_is_forward_and_backward_compatible() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket";
let suffix = format!("{}{arn}", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
let mut metadata = HashMap::from([(format!("{RUSTFS_INTERNAL_PREFIX}replication-status"), format!("{arn}=COMPLETED;"))]);
@@ -4803,7 +5259,7 @@ mod tests {
// must keep it keyed by `target_reset_header(arn)` (not the bare ARN) so
// `ReplicationState::target_state` finds it after a round trip
// (backlog#799 B16).
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let arn = "arn:rustfs:replication:us-east-1:TenantA:bucket";
let ts = "2026-06-30T00:00:00Z;reset-1".to_string();
let key = crate::replication::target_reset_header(arn);
let mut metadata = HashMap::new();
@@ -4822,6 +5278,33 @@ mod tests {
);
}
#[test]
fn get_internal_replication_state_accepts_rfc3339_and_rustfs_display_timestamps() {
let replica_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
let replication_timestamp = replica_timestamp + time::Duration::SECOND;
let metadata = HashMap::from([
(format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICA_STATUS}"), "REPLICA".to_string()),
(
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICA_TIMESTAMP}"),
replica_timestamp.to_string(),
),
(
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICATION_STATUS}"),
"arn:rustfs:replication:us-east-1:TenantA:bucket=COMPLETED;".to_string(),
),
(
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_REPLICATION_TIMESTAMP}"),
replication_timestamp
.format(&Rfc3339)
.expect("RFC3339 timestamp should format"),
),
]);
let state = get_internal_replication_state(&metadata).expect("replication metadata should parse");
assert_eq!(state.replica_timestamp, Some(replica_timestamp));
assert_eq!(state.replication_timestamp, Some(replication_timestamp));
}
// ---- Header signature (backlog#861 / B12) ----
fn signed_object() -> MetaObject {
crates/filemeta/src/metacache.rs
+36 -1
View File
@@ -157,7 +157,7 @@ impl MetaCacheEntry {
});
}
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false, true)?;
let fi = fm.into_fileinfo_without_part_checksums(bucket, self.name.as_str(), "", false, false)?;
return Ok(fi);
}
@@ -169,6 +169,7 @@ impl MetaCacheEntry {
FileInfoOpts {
data: false,
include_free_versions: false,
include_part_checksums: false,
},
)
}
@@ -2157,4 +2158,38 @@ mod tests {
let inner = caught.expect("file_info_versions must not panic");
assert!(matches!(inner, Err(Error::FileCorrupt)), "expected FileCorrupt");
}
#[test]
fn metacache_to_fileinfo_ignores_part_checksum_sidecar_regardless_of_cache_state() {
let mut meta = FileMeta::load(&create_real_xlmeta().expect("create real xl.meta")).expect("load real xl.meta");
let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef").expect("valid fixture version id");
let (index, mut version) = meta.find_version(Some(version_id)).expect("find fixture object version");
rustfs_utils::http::insert_bytes(
&mut version.object.as_mut().expect("fixture object").meta_sys,
rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
b"not-json".to_vec(),
);
meta.versions[index] = FileMetaShallowVersion::try_from(version).expect("replace fixture object version");
let encoded = meta.marshal_msg().expect("marshal object metadata");
let uncached = MetaCacheEntry {
name: "object".to_string(),
metadata: encoded.clone(),
cached: None,
reusable: false,
}
.to_fileinfo("bucket")
.expect("uncached metacache conversion must stay lazy");
let cached = MetaCacheEntry {
name: "object".to_string(),
metadata: encoded,
cached: Some(meta),
reusable: false,
}
.to_fileinfo("bucket")
.expect("cached metacache conversion must stay lazy");
assert_eq!(cached, uncached);
assert!(cached.parts.iter().all(|part| part.checksums.is_none()));
}
}