diff --git a/crates/filemeta/src/fileinfo.rs b/crates/filemeta/src/fileinfo.rs index 27721300d..4fd556aee 100644 --- a/crates/filemeta/src/fileinfo.rs +++ b/crates/filemeta/src/fileinfo.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::metadata_keys; use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_COMPLETE, VersionPurgeStatusType}; use bytes::Bytes; use rmp_serde::Serializer; @@ -23,7 +24,6 @@ use rustfs_utils::http::{ starts_with_ignore_ascii_case, }; use s3s::dto::{RestoreStatus, Timestamp}; -use s3s::header::X_AMZ_RESTORE; use serde::de::{self, MapAccess, SeqAccess, Visitor, value::MapAccessDeserializer}; use serde::ser::SerializeMap; use serde::{Deserialize, Serialize}; @@ -1482,7 +1482,7 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result { } pub fn is_restored_object_on_disk(meta: &HashMap) -> bool { - if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str()) + if let Some(restore_hdr) = meta.get(metadata_keys::RESTORE) && let Ok(restore_status) = parse_restore_obj_status(restore_hdr) { return restore_status.on_disk(); @@ -2756,7 +2756,7 @@ mod tests { fn minio_restored_object_is_recognised_as_on_disk() { let mut meta = HashMap::new(); meta.insert( - X_AMZ_RESTORE.as_str().to_string(), + metadata_keys::RESTORE.to_string(), "ongoing-request=\"false\", expiry-date=\"Fri, 01 Jan 9999 00:00:00 GMT\"".to_string(), ); assert!(is_restored_object_on_disk(&meta)); diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 9549afcd5..27b5afcd1 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -17,22 +17,18 @@ use crate::replication::{ }; use crate::{ ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, NULL_VERSION_ID, ObjectPartInfo, RawFileInfo, - ReplicationState, ReplicationStatusType, Result, VersionPurgeStatusType, is_restored_object_on_disk, + ReplicationState, ReplicationStatusType, Result, VersionPurgeStatusType, is_restored_object_on_disk, metadata_keys, replication_statuses_map, version_purge_statuses_map, }; use byteorder::ByteOrder; use bytes::Bytes; -use rustfs_utils::http::headers::{ - AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, - AMZ_STORAGE_CLASS, -}; +use rustfs_utils::http::headers::{AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5}; use rustfs_utils::http::{ - AMZ_BUCKET_REPLICATION_STATUS, MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING, - SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, + MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS, + SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, contains_key_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, }; -use s3s::header::X_AMZ_RESTORE; use serde::{Deserialize, Serialize}; #[cfg(test)] use std::cell::Cell; @@ -1114,34 +1110,6 @@ mod test { /// `parse_restore_obj_status` (fileinfo.rs). const RESTORED_ON_DISK: &str = "ongoing-request=\"false\", expiry-date=\"9999-01-01T00:00:00Z\""; - /// backlog#1733 (P9-01 §4.3/§7.6, g-key-001): pin the five `s3s::header` - /// constants that double as **persisted metadata map keys**. They are not - /// just HTTP header names — they are stored inside xl.meta (`meta_user`) - /// and read back by fail-open code, so a silent drift produces zero - /// HTTP-visible errors while: - /// - /// 1. **WORM silently dissolves** — `get_object_retention_meta` - /// (ecstore objectlock.rs) returns an empty retention when the lock keys - /// are unreadable, making every compliance-locked object deletable. - /// 2. **Live data dirs can be reclaimed** — `MetaObject::uses_data_dir` - /// falls back to `is_restored_object_on_disk`, which returns `false` - /// when `x-amz-restore` is unreadable, so a restored object's data dir - /// is judged unused. - /// - /// Any migration replacing these constants must keep the literals byte-stable. - #[test] - fn persisted_metadata_keys_are_byte_stable() { - use s3s::header::{ - X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, - X_AMZ_SERVER_SIDE_ENCRYPTION, - }; - assert_eq!(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(), "x-amz-object-lock-legal-hold"); - assert_eq!(X_AMZ_OBJECT_LOCK_MODE.as_str(), "x-amz-object-lock-mode"); - assert_eq!(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(), "x-amz-object-lock-retain-until-date"); - assert_eq!(X_AMZ_RESTORE.as_str(), "x-amz-restore"); - assert_eq!(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str(), "x-amz-server-side-encryption"); - } - /// backlog#1733 g-key-003: a restored-to-local object must keep its data /// dir. The restore marker lives under the pinned `x-amz-restore` key; if /// the key ever drifts this flips to `false` and the data dir becomes @@ -1178,11 +1146,11 @@ mod test { #[test] fn restore_marker_roundtrips_through_parser() { let mut meta = HashMap::new(); - meta.insert(X_AMZ_RESTORE.as_str().to_string(), RESTORED_ON_DISK.to_string()); + meta.insert(metadata_keys::RESTORE.to_string(), RESTORED_ON_DISK.to_string()); assert!(crate::is_restored_object_on_disk(&meta)); // An in-progress restore is not "on disk". - meta.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"true\"".to_string()); + meta.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string()); assert!(!crate::is_restored_object_on_disk(&meta)); } @@ -1900,7 +1868,7 @@ mod test { /// with its transition metadata (and thus the remote tier copy) intact. #[test] fn test_delete_version_expire_restored_keeps_transitioned_version() { - use rustfs_utils::http::headers::{AMZ_RESTORE, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE}; + use crate::metadata_keys::{RESTORE, RESTORE_EXPIRY_DAYS, RESTORE_REQUEST_DATE}; let mut fm = FileMeta::new(); let vid = Uuid::new_v4(); @@ -1914,12 +1882,12 @@ mod test { fi.transitioned_objname = "remote/obj".to_string(); fi.transition_tier = "COLDTIER".to_string(); fi.metadata.insert( - AMZ_RESTORE.to_string(), + RESTORE.to_string(), "ongoing-request=\"false\", expiry-date=\"Fri, 17 Jul 2026 00:00:00 GMT\"".to_string(), ); - fi.metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), "1".to_string()); + fi.metadata.insert(RESTORE_EXPIRY_DAYS.to_string(), "1".to_string()); fi.metadata - .insert(AMZ_RESTORE_REQUEST_DATE.to_string(), "Thu, 16 Jul 2026 00:00:00 GMT".to_string()); + .insert(RESTORE_REQUEST_DATE.to_string(), "Thu, 16 Jul 2026 00:00:00 GMT".to_string()); rustfs_utils::http::insert_str(&mut fi.metadata, SUFFIX_RESTORE_OPERATION_ID, Uuid::from_u128(1).to_string()); rustfs_utils::http::insert_str( &mut fi.metadata, @@ -1939,9 +1907,9 @@ mod test { assert_eq!(fm.versions.len(), 1, "the version must survive restored-copy expiry"); let after = fm.into_fileinfo("vol", "restored.bin", "", false, false, true).unwrap(); - assert!(!after.metadata.contains_key(AMZ_RESTORE), "x-amz-restore must be stripped"); - assert!(!after.metadata.contains_key(AMZ_RESTORE_EXPIRY_DAYS)); - assert!(!after.metadata.contains_key(AMZ_RESTORE_REQUEST_DATE)); + assert!(!after.metadata.contains_key(RESTORE), "x-amz-restore must be stripped"); + assert!(!after.metadata.contains_key(RESTORE_EXPIRY_DAYS)); + assert!(!after.metadata.contains_key(RESTORE_REQUEST_DATE)); assert!( rustfs_utils::http::get_str(&after.metadata, SUFFIX_RESTORE_OPERATION_ID).is_none(), "expired restore must not retain its operation generation" diff --git a/crates/filemeta/src/filemeta/inline_data.rs b/crates/filemeta/src/filemeta/inline_data.rs index 98d3bb28f..e1154e5c3 100644 --- a/crates/filemeta/src/filemeta/inline_data.rs +++ b/crates/filemeta/src/filemeta/inline_data.rs @@ -144,7 +144,6 @@ impl FileMeta { #[cfg(test)] mod tests { use super::*; - use s3s::header::X_AMZ_RESTORE; use time::format_description::well_known::Rfc3339; use time::{Duration, OffsetDateTime}; @@ -170,7 +169,7 @@ mod tests { data_dir, HashMap::from([ ("etag".to_string(), format!("etag-{version_id}")), - (X_AMZ_RESTORE.as_str().to_string(), restore_header), + (metadata_keys::RESTORE.to_string(), restore_header), ]), ) } diff --git a/crates/filemeta/src/filemeta/version.rs b/crates/filemeta/src/filemeta/version.rs index 42aa5a74d..0f2fd6626 100644 --- a/crates/filemeta/src/filemeta/version.rs +++ b/crates/filemeta/src/filemeta/version.rs @@ -2609,7 +2609,7 @@ impl MetaObject { continue; } - if k == AMZ_STORAGE_CLASS && v == "STANDARD" { + if k == metadata_keys::STORAGE_CLASS && v == "STANDARD" { continue; } @@ -2621,7 +2621,7 @@ impl MetaObject { continue; } - if k.eq_ignore_ascii_case(AMZ_STORAGE_CLASS) && v == b"STANDARD" { + if k.eq_ignore_ascii_case(metadata_keys::STORAGE_CLASS) && v == b"STANDARD" { continue; } @@ -2641,7 +2641,7 @@ impl MetaObject { let st = v.composite_replication_status(); if !st.is_empty() { - metadata.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), st.to_string()); + metadata.insert(metadata_keys::REPLICATION_STATUS.to_string(), st.to_string()); } } @@ -2734,9 +2734,9 @@ impl MetaObject { } pub fn remove_restore_hdrs(&mut self) { - self.meta_user.remove(X_AMZ_RESTORE.as_str()); - self.meta_user.remove(AMZ_RESTORE_EXPIRY_DAYS); - self.meta_user.remove(AMZ_RESTORE_REQUEST_DATE); + self.meta_user.remove(metadata_keys::RESTORE); + self.meta_user.remove(metadata_keys::RESTORE_EXPIRY_DAYS); + self.meta_user.remove(metadata_keys::RESTORE_REQUEST_DATE); remove_bytes(&mut self.meta_sys, SUFFIX_RESTORE_OPERATION_ID); remove_bytes(&mut self.meta_sys, SUFFIX_RESTORE_WORKER_LOCK); } diff --git a/crates/filemeta/src/lib.rs b/crates/filemeta/src/lib.rs index 3fca95ea6..4b79097f0 100644 --- a/crates/filemeta/src/lib.rs +++ b/crates/filemeta/src/lib.rs @@ -20,6 +20,7 @@ mod filemeta_inline; mod metacache; mod replication; +pub mod metadata_keys; pub mod test_data; pub use error::*; diff --git a/crates/filemeta/src/metadata_keys.rs b/crates/filemeta/src/metadata_keys.rs new file mode 100644 index 000000000..a627493d5 --- /dev/null +++ b/crates/filemeta/src/metadata_keys.rs @@ -0,0 +1,269 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Object-metadata map keys persisted inside xl.meta (`MetaObject::meta_user`). +//! +//! These strings are part of the on-disk format, not HTTP header names. They +//! happen to spell S3 header names, and some were historically taken from +//! `s3s::header` or duplicated in `rustfs_utils::http::headers`, which let an +//! HTTP-library or header-table edit silently change what is written to and +//! looked up in xl.meta. Map lookups are exact, so every byte (including ASCII +//! case) is load-bearing and existing xl.meta cannot be rewritten in bulk: +//! +//! - the object-lock keys unreadable => retention reads back as "no lock" and +//! WORM objects become deletable; +//! - [`RESTORE`] unreadable => `MetaObject::uses_data_dir` reports a restored +//! object's data dir as unused, making live data reclaimable. +//! +//! The literals are pinned by tests below against both a literal table and +//! xl.meta bytes captured from the code that predates this module. + +/// Object-lock legal hold (`ON`/`OFF`). +pub const OBJECT_LOCK_LEGAL_HOLD: &str = "x-amz-object-lock-legal-hold"; +/// Object-lock retention mode (`GOVERNANCE`/`COMPLIANCE`). +pub const OBJECT_LOCK_MODE: &str = "x-amz-object-lock-mode"; +/// Object-lock retain-until date. +pub const OBJECT_LOCK_RETAIN_UNTIL_DATE: &str = "x-amz-object-lock-retain-until-date"; +/// Restore status of a transitioned object restored to local storage. +pub const RESTORE: &str = "x-amz-restore"; +/// Requested restore duration. Persisted in this mixed case. +pub const RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days"; +/// Restore request date. Persisted in this mixed case. +pub const RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date"; +/// Server-side encryption algorithm. +pub const SERVER_SIDE_ENCRYPTION: &str = "x-amz-server-side-encryption"; +/// Storage class. +pub const STORAGE_CLASS: &str = "x-amz-storage-class"; +/// Composite replication status. Persisted in this mixed case. +pub const REPLICATION_STATUS: &str = "X-Amz-Replication-Status"; + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_data::create_pre_metadata_keys_xlmeta; + use crate::{FileMeta, FileMetaVersion, MetaObject, is_restored_object_on_disk}; + use std::collections::HashMap; + + /// Every persisted key with its exact on-disk spelling, written out as an + /// independent literal so a change to a constant cannot also change the + /// expectation. + const PINNED: [(&str, &str); 9] = [ + (OBJECT_LOCK_LEGAL_HOLD, "x-amz-object-lock-legal-hold"), + (OBJECT_LOCK_MODE, "x-amz-object-lock-mode"), + (OBJECT_LOCK_RETAIN_UNTIL_DATE, "x-amz-object-lock-retain-until-date"), + (RESTORE, "x-amz-restore"), + (RESTORE_EXPIRY_DAYS, "X-Amz-Restore-Expiry-Days"), + (RESTORE_REQUEST_DATE, "X-Amz-Restore-Request-Date"), + (SERVER_SIDE_ENCRYPTION, "x-amz-server-side-encryption"), + (STORAGE_CLASS, "x-amz-storage-class"), + (REPLICATION_STATUS, "X-Amz-Replication-Status"), + ]; + + /// Values stored under each key in the pre-module fixture. + const FIXTURE_VALUES: [(&str, &str); 9] = [ + ("x-amz-object-lock-legal-hold", "ON"), + ("x-amz-object-lock-mode", "COMPLIANCE"), + ("x-amz-object-lock-retain-until-date", "2099-01-01T00:00:00Z"), + ("x-amz-restore", "ongoing-request=\"false\", expiry-date=\"9999-01-01T00:00:00Z\""), + ("X-Amz-Restore-Expiry-Days", "7"), + ("X-Amz-Restore-Request-Date", "Thu, 16 Jul 2026 00:00:00 GMT"), + ("x-amz-server-side-encryption", "AES256"), + ("x-amz-storage-class", "GLACIER"), + ("X-Amz-Replication-Status", "COMPLETED"), + ]; + + const FIXTURE_VERSION_ID: &str = "0b1e5a3a-1735-4a3a-8000-00000000a3a0"; + + fn fixture_object() -> MetaObject { + let fm = FileMeta::load(&create_pre_metadata_keys_xlmeta().expect("decode fixture hex")).expect("load fixture xl.meta"); + assert_eq!(fm.versions.len(), 1); + FileMetaVersion::try_from(fm.versions[0].meta.as_slice()) + .expect("decode fixture version") + .object + .expect("fixture version is an object") + } + + fn fixture_metadata() -> HashMap { + let fm = FileMeta::load(&create_pre_metadata_keys_xlmeta().expect("decode fixture hex")).expect("load fixture xl.meta"); + fm.into_fileinfo("bucket", "object", FIXTURE_VERSION_ID, false, false, false) + .expect("fixture version to FileInfo") + .metadata + } + + /// Replaces the byte at `idx` with a different one of the same class: + /// ASCII letters flip case (lookups are case-sensitive), anything else + /// becomes `_`. + fn mutate_at(key: &str, idx: usize) -> String { + let mut bytes = key.as_bytes().to_vec(); + let b = bytes[idx]; + bytes[idx] = if b.is_ascii_lowercase() { + b.to_ascii_uppercase() + } else if b.is_ascii_uppercase() { + b.to_ascii_lowercase() + } else { + b'_' + }; + String::from_utf8(bytes).expect("ascii mutation stays utf-8") + } + + #[test] + fn persisted_metadata_keys_are_pinned_byte_for_byte() { + for (key, literal) in PINNED { + assert_eq!(key.as_bytes(), literal.as_bytes(), "persisted key {literal:?} drifted"); + } + let fixture_keys: Vec<&str> = FIXTURE_VALUES.iter().map(|(k, _)| *k).collect(); + let pinned_keys: Vec<&str> = PINNED.iter().map(|(_, l)| *l).collect(); + assert_eq!(fixture_keys, pinned_keys, "fixture table must cover every pinned key"); + } + + /// Migration-period cross-check: the constants must equal the historical + /// sources callers used before this module existed. Drop the `s3s` half + /// together with filemeta's `s3s` dependency. + #[test] + fn persisted_metadata_keys_match_their_historical_sources() { + use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS; + use rustfs_utils::http::headers::{ + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_RESTORE, + AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS, + }; + use s3s::header::{ + X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE, + X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_STORAGE_CLASS, + }; + + assert_eq!(OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str()); + assert_eq!(OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_MODE.as_str()); + assert_eq!(OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str()); + assert_eq!(RESTORE, X_AMZ_RESTORE.as_str()); + assert_eq!(SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION.as_str()); + assert_eq!(STORAGE_CLASS, X_AMZ_STORAGE_CLASS.as_str()); + + assert_eq!(OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER); + assert_eq!(OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER); + assert_eq!(OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER); + assert_eq!(RESTORE, AMZ_RESTORE); + assert_eq!(RESTORE_EXPIRY_DAYS, AMZ_RESTORE_EXPIRY_DAYS); + assert_eq!(RESTORE_REQUEST_DATE, AMZ_RESTORE_REQUEST_DATE); + assert_eq!(STORAGE_CLASS, AMZ_STORAGE_CLASS); + assert_eq!(REPLICATION_STATUS, AMZ_BUCKET_REPLICATION_STATUS); + } + + /// The fixture bytes were written by the code that predates this module, + /// with each key taken from its historical source. Every key must appear + /// verbatim in the raw bytes and read back through the new constants. + #[test] + fn pre_module_xlmeta_reads_back_every_persisted_key() { + let raw = create_pre_metadata_keys_xlmeta().expect("decode fixture hex"); + for (key, _) in PINNED { + assert!( + raw.windows(key.len()).any(|w| w == key.as_bytes()), + "fixture bytes must contain persisted key {key:?} verbatim" + ); + } + + let obj = fixture_object(); + let metadata = fixture_metadata(); + for ((key, _), (fixture_key, value)) in PINNED.iter().zip(FIXTURE_VALUES) { + assert_eq!(*key, fixture_key); + assert_eq!(obj.meta_user.get(*key).map(String::as_str), Some(value), "meta_user[{key:?}]"); + assert_eq!(metadata.get(*key).map(String::as_str), Some(value), "FileInfo.metadata[{key:?}]"); + } + + assert!(is_restored_object_on_disk(&metadata), "restore marker must read back as on disk"); + assert!(obj.uses_data_dir(), "restored object's data dir must stay in use"); + } + + /// Removing restore headers from the pre-module object must drop exactly + /// the restore keys and flip `uses_data_dir`, proving the remover and the + /// reader address the same persisted bytes. + #[test] + fn remove_restore_hdrs_strips_pre_module_restore_keys() { + let mut obj = fixture_object(); + obj.remove_restore_hdrs(); + for key in [RESTORE, RESTORE_EXPIRY_DAYS, RESTORE_REQUEST_DATE] { + assert!(!obj.meta_user.contains_key(key), "{key:?} must be removed"); + } + for key in [ + OBJECT_LOCK_LEGAL_HOLD, + OBJECT_LOCK_MODE, + OBJECT_LOCK_RETAIN_UNTIL_DATE, + SERVER_SIDE_ENCRYPTION, + STORAGE_CLASS, + REPLICATION_STATUS, + ] { + assert!(obj.meta_user.contains_key(key), "{key:?} must survive restore cleanup"); + } + assert!(!is_restored_object_on_disk(&obj.meta_user)); + assert!(!obj.uses_data_dir()); + } + + /// Writing through the new constants must produce the same persisted key + /// set as the pre-module bytes, so a rollback reads new objects too. + #[test] + fn new_writes_persist_the_same_keys_as_pre_module_bytes() { + let mut fi = FileMeta::load(&create_pre_metadata_keys_xlmeta().expect("decode fixture hex")) + .expect("load fixture xl.meta") + .into_fileinfo("bucket", "object", FIXTURE_VERSION_ID, false, false, false) + .expect("fixture version to FileInfo"); + fi.metadata = PINNED + .iter() + .zip(FIXTURE_VALUES) + .map(|((key, _), (_, value))| (key.to_string(), value.to_string())) + .collect(); + fi.metadata + .insert("etag".to_string(), "d41d8cd98f00b204e9800998ecf8427e".to_string()); + + let mut fm = FileMeta::new(); + fm.add_version(fi).expect("add version"); + let rewritten = FileMeta::load(&fm.marshal_msg().expect("marshal")).expect("reload"); + let rewritten = FileMetaVersion::try_from(rewritten.versions[0].meta.as_slice()) + .expect("decode rewritten version") + .object + .expect("rewritten version is an object"); + + assert_eq!(rewritten.meta_user, fixture_object().meta_user); + } + + /// Per-character mutation: a constant that differs from the persisted key + /// by any single byte (including ASCII case) finds nothing in pre-module + /// xl.meta, and a restore marker stored under such a key is not honored. + /// This is the drift the pins above turn into a test failure. + #[test] + fn single_character_key_mutation_misses_pre_module_data() { + let metadata = fixture_metadata(); + let mut mutations = 0usize; + for (key, _) in PINNED { + for idx in 0..key.len() { + let mutated = mutate_at(key, idx); + assert_ne!(mutated, key); + assert!( + !metadata.contains_key(&mutated), + "mutation {mutated:?} of {key:?} must not match pre-module data" + ); + mutations += 1; + } + } + assert_eq!(mutations, PINNED.iter().map(|(k, _)| k.len()).sum::()); + + let marker = metadata.get(RESTORE).expect("fixture restore marker").clone(); + for idx in 0..RESTORE.len() { + let meta = HashMap::from([(mutate_at(RESTORE, idx), marker.clone())]); + assert!( + !is_restored_object_on_disk(&meta), + "restore marker under {:?} must not count as on disk", + mutate_at(RESTORE, idx) + ); + } + } +} diff --git a/crates/filemeta/src/test_data.rs b/crates/filemeta/src/test_data.rs index 3ccb7b0d8..95a552597 100644 --- a/crates/filemeta/src/test_data.rs +++ b/crates/filemeta/src/test_data.rs @@ -171,6 +171,14 @@ pub fn create_minio_large_object_xlmeta() -> Result> { decode_hex_fixture(include_str!("../tests/fixtures/minio/object_large_bin.xlmeta.hex")) } +/// xl.meta for one restored, object-locked object, written by the code that +/// predates `metadata_keys` with every persisted key taken from its historical +/// source (`s3s::header` / `rustfs_utils::http`). Never regenerate it: it is +/// the old-bytes evidence that the key authority stayed byte-identical. +pub fn create_pre_metadata_keys_xlmeta() -> Result> { + decode_hex_fixture(include_str!("../tests/fixtures/persisted_metadata_keys_pre_a3a.hex")) +} + fn write_legacy_time(wr: &mut Vec, ts: OffsetDateTime) { wr.push(MSGPACK_EXT8); wr.push(12); diff --git a/crates/filemeta/tests/fixtures/persisted_metadata_keys_pre_a3a.hex b/crates/filemeta/tests/fixtures/persisted_metadata_keys_pre_a3a.hex new file mode 100644 index 000000000..52f4d9809 --- /dev/null +++ b/crates/filemeta/tests/fixtures/persisted_metadata_keys_pre_a3a.hex @@ -0,0 +1 @@ +584c322001000300c6000002aa030301c42697c4100b1e5a3a17354a3a800000000000a3a0d318c29c112e760000c40405541d2b01020204c5027c83a45479706501a556324f626ade0011a24944c4100b1e5a3a17354a3a800000000000a3a0a444446972c4100b1e5a3a17354a3a800000000000d1d0a64563416c676f01a345634d04a345634e02a745634253697a65ce00100000a74563496e64657801a645634469737496010203040506a84353756d416c676f01a8506172744e756d7390a9506172744554616773c0a95061727453697a657390aa506172744153697a6573c0a453697a65ce00010000a54d54696d65cf18c29c112e760000a74d657461537973c0a74d6574615573728aa465746167d9206434316438636439386630306232303465393830303939386563663834323765d923782d616d7a2d6f626a6563742d6c6f636b2d72657461696e2d756e74696c2d64617465b4323039392d30312d30315430303a30303a30305abc782d616d7a2d7365727665722d736964652d656e6372797074696f6ea6414553323536b8582d416d7a2d5265706c69636174696f6e2d537461747573a9434f4d504c45544544bc782d616d7a2d6f626a6563742d6c6f636b2d6c6567616c2d686f6c64a24f4eb9582d416d7a2d526573746f72652d4578706972792d44617973a137ad782d616d7a2d726573746f7265d93b6f6e676f696e672d726571756573743d2266616c7365222c206578706972792d646174653d22393939392d30312d30315430303a30303a30305a22ba582d416d7a2d526573746f72652d526571756573742d44617465bd5468752c203136204a756c20323032362030303a30303a303020474d54b6782d616d7a2d6f626a6563742d6c6f636b2d6d6f6465aa434f4d504c49414e4345b3782d616d7a2d73746f726167652d636c617373a7474c4143494552a17600ce35759b93