gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-24 11:26:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(filemeta): own persisted metadata key authority (#7760)

Browse Source 
Add rustfs_filemeta::metadata_keys as the single source of the nine xl.meta meta_user keys, byte-identical to current literals, and migrate filemeta consumers off s3s::header and rustfs_utils header constants. Pin the literals, cross-check historical sources, and prove old xl.meta bytes still decode via a captured fixture and a per-character mutation sweep.

Refs rustfs/backlog#1735
This commit is contained in:
Chris
2026-09-14 00:08:40 +08:00
committed by GitHub
parent f37e07cca3
commit 1a7db3ee12
8 changed files with 302 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+3 -3
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::metadata_keys;
use crate::{Error, ReplicationState, ReplicationStatusType, Result, TRANSITION_COMPLETE, VersionPurgeStatusType};
use bytes::Bytes;
use rmp_serde::Serializer;
@@ -23,7 +24,6 @@ use rustfs_utils::http::{
starts_with_ignore_ascii_case,
};
use s3s::dto::{RestoreStatus, Timestamp};
use s3s::header::X_AMZ_RESTORE;
use serde::de::{self, MapAccess, SeqAccess, Visitor, value::MapAccessDeserializer};
use serde::ser::SerializeMap;
use serde::{Deserialize, Serialize};
@@ -1482,7 +1482,7 @@ pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
}
pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool {
if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str())
if let Some(restore_hdr) = meta.get(metadata_keys::RESTORE)
&& let Ok(restore_status) = parse_restore_obj_status(restore_hdr)
{
return restore_status.on_disk();
@@ -2756,7 +2756,7 @@ mod tests {
fn minio_restored_object_is_recognised_as_on_disk() {
let mut meta = HashMap::new();
meta.insert(
X_AMZ_RESTORE.as_str().to_string(),
metadata_keys::RESTORE.to_string(),
"ongoing-request=\"false\", expiry-date=\"Fri, 01 Jan 9999 00:00:00 GMT\"".to_string(),
);
assert!(is_restored_object_on_disk(&meta));
crates/filemeta/src/filemeta.rs
+13 -45
View File
@@ -17,22 +17,18 @@ use crate::replication::{
};
use crate::{
ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, NULL_VERSION_ID, ObjectPartInfo, RawFileInfo,
ReplicationState, ReplicationStatusType, Result, VersionPurgeStatusType, is_restored_object_on_disk,
ReplicationState, ReplicationStatusType, Result, VersionPurgeStatusType, is_restored_object_on_disk, metadata_keys,
replication_statuses_map, version_purge_statuses_map,
};
use byteorder::ByteOrder;
use bytes::Bytes;
use rustfs_utils::http::headers::{
AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE,
AMZ_STORAGE_CLASS,
};
use rustfs_utils::http::headers::{AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING,
SUFFIX_PURGESTATUS, SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
MINIO_INTERNAL_PREFIX, RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_DATA_MOV, SUFFIX_HEALING, SUFFIX_PURGESTATUS,
SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_RESTORE_OPERATION_ID,
SUFFIX_RESTORE_WORKER_LOCK, contains_key_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
};
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
#[cfg(test)]
use std::cell::Cell;
@@ -1114,34 +1110,6 @@ mod test {
/// `parse_restore_obj_status` (fileinfo.rs).
const RESTORED_ON_DISK: &str = "ongoing-request=\"false\", expiry-date=\"9999-01-01T00:00:00Z\"";
/// backlog#1733 (P9-01 §4.3/§7.6, g-key-001): pin the five `s3s::header`
/// constants that double as **persisted metadata map keys**. They are not
/// just HTTP header names — they are stored inside xl.meta (`meta_user`)
/// and read back by fail-open code, so a silent drift produces zero
/// HTTP-visible errors while:
///
/// 1. **WORM silently dissolves** — `get_object_retention_meta`
/// (ecstore objectlock.rs) returns an empty retention when the lock keys
/// are unreadable, making every compliance-locked object deletable.
/// 2. **Live data dirs can be reclaimed** — `MetaObject::uses_data_dir`
/// falls back to `is_restored_object_on_disk`, which returns `false`
/// when `x-amz-restore` is unreadable, so a restored object's data dir
/// is judged unused.
///
/// Any migration replacing these constants must keep the literals byte-stable.
#[test]
fn persisted_metadata_keys_are_byte_stable() {
use s3s::header::{
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
X_AMZ_SERVER_SIDE_ENCRYPTION,
};
assert_eq!(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(), "x-amz-object-lock-legal-hold");
assert_eq!(X_AMZ_OBJECT_LOCK_MODE.as_str(), "x-amz-object-lock-mode");
assert_eq!(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(), "x-amz-object-lock-retain-until-date");
assert_eq!(X_AMZ_RESTORE.as_str(), "x-amz-restore");
assert_eq!(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str(), "x-amz-server-side-encryption");
}
/// backlog#1733 g-key-003: a restored-to-local object must keep its data
/// dir. The restore marker lives under the pinned `x-amz-restore` key; if
/// the key ever drifts this flips to `false` and the data dir becomes
@@ -1178,11 +1146,11 @@ mod test {
#[test]
fn restore_marker_roundtrips_through_parser() {
let mut meta = HashMap::new();
meta.insert(X_AMZ_RESTORE.as_str().to_string(), RESTORED_ON_DISK.to_string());
meta.insert(metadata_keys::RESTORE.to_string(), RESTORED_ON_DISK.to_string());
assert!(crate::is_restored_object_on_disk(&meta));
// An in-progress restore is not "on disk".
meta.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"true\"".to_string());
meta.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string());
assert!(!crate::is_restored_object_on_disk(&meta));
}
@@ -1900,7 +1868,7 @@ mod test {
/// with its transition metadata (and thus the remote tier copy) intact.
#[test]
fn test_delete_version_expire_restored_keeps_transitioned_version() {
use rustfs_utils::http::headers::{AMZ_RESTORE, AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
use crate::metadata_keys::{RESTORE, RESTORE_EXPIRY_DAYS, RESTORE_REQUEST_DATE};
let mut fm = FileMeta::new();
let vid = Uuid::new_v4();
@@ -1914,12 +1882,12 @@ mod test {
fi.transitioned_objname = "remote/obj".to_string();
fi.transition_tier = "COLDTIER".to_string();
fi.metadata.insert(
AMZ_RESTORE.to_string(),
RESTORE.to_string(),
"ongoing-request=\"false\", expiry-date=\"Fri, 17 Jul 2026 00:00:00 GMT\"".to_string(),
);
fi.metadata.insert(AMZ_RESTORE_EXPIRY_DAYS.to_string(), "1".to_string());
fi.metadata.insert(RESTORE_EXPIRY_DAYS.to_string(), "1".to_string());
fi.metadata
.insert(AMZ_RESTORE_REQUEST_DATE.to_string(), "Thu, 16 Jul 2026 00:00:00 GMT".to_string());
.insert(RESTORE_REQUEST_DATE.to_string(), "Thu, 16 Jul 2026 00:00:00 GMT".to_string());
rustfs_utils::http::insert_str(&mut fi.metadata, SUFFIX_RESTORE_OPERATION_ID, Uuid::from_u128(1).to_string());
rustfs_utils::http::insert_str(
&mut fi.metadata,
@@ -1939,9 +1907,9 @@ mod test {
assert_eq!(fm.versions.len(), 1, "the version must survive restored-copy expiry");
let after = fm.into_fileinfo("vol", "restored.bin", "", false, false, true).unwrap();
assert!(!after.metadata.contains_key(AMZ_RESTORE), "x-amz-restore must be stripped");
assert!(!after.metadata.contains_key(AMZ_RESTORE_EXPIRY_DAYS));
assert!(!after.metadata.contains_key(AMZ_RESTORE_REQUEST_DATE));
assert!(!after.metadata.contains_key(RESTORE), "x-amz-restore must be stripped");
assert!(!after.metadata.contains_key(RESTORE_EXPIRY_DAYS));
assert!(!after.metadata.contains_key(RESTORE_REQUEST_DATE));
assert!(
rustfs_utils::http::get_str(&after.metadata, SUFFIX_RESTORE_OPERATION_ID).is_none(),
"expired restore must not retain its operation generation"
crates/filemeta/src/filemeta/inline_data.rs
+1 -2
View File
@@ -144,7 +144,6 @@ impl FileMeta {
#[cfg(test)]
mod tests {
use super::*;
use s3s::header::X_AMZ_RESTORE;
use time::format_description::well_known::Rfc3339;
use time::{Duration, OffsetDateTime};
@@ -170,7 +169,7 @@ mod tests {
data_dir,
HashMap::from([
("etag".to_string(), format!("etag-{version_id}")),
(X_AMZ_RESTORE.as_str().to_string(), restore_header),
(metadata_keys::RESTORE.to_string(), restore_header),
]),
)
}
crates/filemeta/src/filemeta/version.rs
+6 -6
View File
@@ -2609,7 +2609,7 @@ impl MetaObject {
continue;
}
if k == AMZ_STORAGE_CLASS && v == "STANDARD" {
if k == metadata_keys::STORAGE_CLASS && v == "STANDARD" {
continue;
}
@@ -2621,7 +2621,7 @@ impl MetaObject {
continue;
}
if k.eq_ignore_ascii_case(AMZ_STORAGE_CLASS) && v == b"STANDARD" {
if k.eq_ignore_ascii_case(metadata_keys::STORAGE_CLASS) && v == b"STANDARD" {
continue;
}
@@ -2641,7 +2641,7 @@ impl MetaObject {
let st = v.composite_replication_status();
if !st.is_empty() {
metadata.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), st.to_string());
metadata.insert(metadata_keys::REPLICATION_STATUS.to_string(), st.to_string());
}
}
@@ -2734,9 +2734,9 @@ impl MetaObject {
}
pub fn remove_restore_hdrs(&mut self) {
self.meta_user.remove(X_AMZ_RESTORE.as_str());
self.meta_user.remove(AMZ_RESTORE_EXPIRY_DAYS);
self.meta_user.remove(AMZ_RESTORE_REQUEST_DATE);
self.meta_user.remove(metadata_keys::RESTORE);
self.meta_user.remove(metadata_keys::RESTORE_EXPIRY_DAYS);
self.meta_user.remove(metadata_keys::RESTORE_REQUEST_DATE);
remove_bytes(&mut self.meta_sys, SUFFIX_RESTORE_OPERATION_ID);
remove_bytes(&mut self.meta_sys, SUFFIX_RESTORE_WORKER_LOCK);
}
crates/filemeta/src/lib.rs
+1
View File
@@ -20,6 +20,7 @@ mod filemeta_inline;
mod metacache;
mod replication;
pub mod metadata_keys;
pub mod test_data;
pub use error::*;
crates/filemeta/src/metadata_keys.rs
+269
View File
@@ -0,0 +1,269 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Object-metadata map keys persisted inside xl.meta (`MetaObject::meta_user`).
//!
//! These strings are part of the on-disk format, not HTTP header names. They
//! happen to spell S3 header names, and some were historically taken from
//! `s3s::header` or duplicated in `rustfs_utils::http::headers`, which let an
//! HTTP-library or header-table edit silently change what is written to and
//! looked up in xl.meta. Map lookups are exact, so every byte (including ASCII
//! case) is load-bearing and existing xl.meta cannot be rewritten in bulk:
//!
//! - the object-lock keys unreadable => retention reads back as "no lock" and
//! WORM objects become deletable;
//! - [`RESTORE`] unreadable => `MetaObject::uses_data_dir` reports a restored
//! object's data dir as unused, making live data reclaimable.
//!
//! The literals are pinned by tests below against both a literal table and
//! xl.meta bytes captured from the code that predates this module.
/// Object-lock legal hold (`ON`/`OFF`).
pub const OBJECT_LOCK_LEGAL_HOLD: &str = "x-amz-object-lock-legal-hold";
/// Object-lock retention mode (`GOVERNANCE`/`COMPLIANCE`).
pub const OBJECT_LOCK_MODE: &str = "x-amz-object-lock-mode";
/// Object-lock retain-until date.
pub const OBJECT_LOCK_RETAIN_UNTIL_DATE: &str = "x-amz-object-lock-retain-until-date";
/// Restore status of a transitioned object restored to local storage.
pub const RESTORE: &str = "x-amz-restore";
/// Requested restore duration. Persisted in this mixed case.
pub const RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days";
/// Restore request date. Persisted in this mixed case.
pub const RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date";
/// Server-side encryption algorithm.
pub const SERVER_SIDE_ENCRYPTION: &str = "x-amz-server-side-encryption";
/// Storage class.
pub const STORAGE_CLASS: &str = "x-amz-storage-class";
/// Composite replication status. Persisted in this mixed case.
pub const REPLICATION_STATUS: &str = "X-Amz-Replication-Status";
#[cfg(test)]
mod tests {
use super::*;
use crate::test_data::create_pre_metadata_keys_xlmeta;
use crate::{FileMeta, FileMetaVersion, MetaObject, is_restored_object_on_disk};
use std::collections::HashMap;
/// Every persisted key with its exact on-disk spelling, written out as an
/// independent literal so a change to a constant cannot also change the
/// expectation.
const PINNED: [(&str, &str); 9] = [
(OBJECT_LOCK_LEGAL_HOLD, "x-amz-object-lock-legal-hold"),
(OBJECT_LOCK_MODE, "x-amz-object-lock-mode"),
(OBJECT_LOCK_RETAIN_UNTIL_DATE, "x-amz-object-lock-retain-until-date"),
(RESTORE, "x-amz-restore"),
(RESTORE_EXPIRY_DAYS, "X-Amz-Restore-Expiry-Days"),
(RESTORE_REQUEST_DATE, "X-Amz-Restore-Request-Date"),
(SERVER_SIDE_ENCRYPTION, "x-amz-server-side-encryption"),
(STORAGE_CLASS, "x-amz-storage-class"),
(REPLICATION_STATUS, "X-Amz-Replication-Status"),
];
/// Values stored under each key in the pre-module fixture.
const FIXTURE_VALUES: [(&str, &str); 9] = [
("x-amz-object-lock-legal-hold", "ON"),
("x-amz-object-lock-mode", "COMPLIANCE"),
("x-amz-object-lock-retain-until-date", "2099-01-01T00:00:00Z"),
("x-amz-restore", "ongoing-request=\"false\", expiry-date=\"9999-01-01T00:00:00Z\""),
("X-Amz-Restore-Expiry-Days", "7"),
("X-Amz-Restore-Request-Date", "Thu, 16 Jul 2026 00:00:00 GMT"),
("x-amz-server-side-encryption", "AES256"),
("x-amz-storage-class", "GLACIER"),
("X-Amz-Replication-Status", "COMPLETED"),
];
const FIXTURE_VERSION_ID: &str = "0b1e5a3a-1735-4a3a-8000-00000000a3a0";
fn fixture_object() -> MetaObject {
let fm = FileMeta::load(&create_pre_metadata_keys_xlmeta().expect("decode fixture hex")).expect("load fixture xl.meta");
assert_eq!(fm.versions.len(), 1);
FileMetaVersion::try_from(fm.versions[0].meta.as_slice())
.expect("decode fixture version")
.object
.expect("fixture version is an object")
}
fn fixture_metadata() -> HashMap<String, String> {
let fm = FileMeta::load(&create_pre_metadata_keys_xlmeta().expect("decode fixture hex")).expect("load fixture xl.meta");
fm.into_fileinfo("bucket", "object", FIXTURE_VERSION_ID, false, false, false)
.expect("fixture version to FileInfo")
.metadata
}
/// Replaces the byte at `idx` with a different one of the same class:
/// ASCII letters flip case (lookups are case-sensitive), anything else
/// becomes `_`.
fn mutate_at(key: &str, idx: usize) -> String {
let mut bytes = key.as_bytes().to_vec();
let b = bytes[idx];
bytes[idx] = if b.is_ascii_lowercase() {
b.to_ascii_uppercase()
} else if b.is_ascii_uppercase() {
b.to_ascii_lowercase()
} else {
b'_'
};
String::from_utf8(bytes).expect("ascii mutation stays utf-8")
}
#[test]
fn persisted_metadata_keys_are_pinned_byte_for_byte() {
for (key, literal) in PINNED {
assert_eq!(key.as_bytes(), literal.as_bytes(), "persisted key {literal:?} drifted");
}
let fixture_keys: Vec<&str> = FIXTURE_VALUES.iter().map(|(k, _)| *k).collect();
let pinned_keys: Vec<&str> = PINNED.iter().map(|(_, l)| *l).collect();
assert_eq!(fixture_keys, pinned_keys, "fixture table must cover every pinned key");
}
/// Migration-period cross-check: the constants must equal the historical
/// sources callers used before this module existed. Drop the `s3s` half
/// together with filemeta's `s3s` dependency.
#[test]
fn persisted_metadata_keys_match_their_historical_sources() {
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_RESTORE,
AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE, AMZ_STORAGE_CLASS,
};
use s3s::header::{
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE,
X_AMZ_SERVER_SIDE_ENCRYPTION, X_AMZ_STORAGE_CLASS,
};
assert_eq!(OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str());
assert_eq!(OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_MODE.as_str());
assert_eq!(OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str());
assert_eq!(RESTORE, X_AMZ_RESTORE.as_str());
assert_eq!(SERVER_SIDE_ENCRYPTION, X_AMZ_SERVER_SIDE_ENCRYPTION.as_str());
assert_eq!(STORAGE_CLASS, X_AMZ_STORAGE_CLASS.as_str());
assert_eq!(OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER);
assert_eq!(OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_MODE_LOWER);
assert_eq!(OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER);
assert_eq!(RESTORE, AMZ_RESTORE);
assert_eq!(RESTORE_EXPIRY_DAYS, AMZ_RESTORE_EXPIRY_DAYS);
assert_eq!(RESTORE_REQUEST_DATE, AMZ_RESTORE_REQUEST_DATE);
assert_eq!(STORAGE_CLASS, AMZ_STORAGE_CLASS);
assert_eq!(REPLICATION_STATUS, AMZ_BUCKET_REPLICATION_STATUS);
}
/// The fixture bytes were written by the code that predates this module,
/// with each key taken from its historical source. Every key must appear
/// verbatim in the raw bytes and read back through the new constants.
#[test]
fn pre_module_xlmeta_reads_back_every_persisted_key() {
let raw = create_pre_metadata_keys_xlmeta().expect("decode fixture hex");
for (key, _) in PINNED {
assert!(
raw.windows(key.len()).any(|w| w == key.as_bytes()),
"fixture bytes must contain persisted key {key:?} verbatim"
);
}
let obj = fixture_object();
let metadata = fixture_metadata();
for ((key, _), (fixture_key, value)) in PINNED.iter().zip(FIXTURE_VALUES) {
assert_eq!(*key, fixture_key);
assert_eq!(obj.meta_user.get(*key).map(String::as_str), Some(value), "meta_user[{key:?}]");
assert_eq!(metadata.get(*key).map(String::as_str), Some(value), "FileInfo.metadata[{key:?}]");
}
assert!(is_restored_object_on_disk(&metadata), "restore marker must read back as on disk");
assert!(obj.uses_data_dir(), "restored object's data dir must stay in use");
}
/// Removing restore headers from the pre-module object must drop exactly
/// the restore keys and flip `uses_data_dir`, proving the remover and the
/// reader address the same persisted bytes.
#[test]
fn remove_restore_hdrs_strips_pre_module_restore_keys() {
let mut obj = fixture_object();
obj.remove_restore_hdrs();
for key in [RESTORE, RESTORE_EXPIRY_DAYS, RESTORE_REQUEST_DATE] {
assert!(!obj.meta_user.contains_key(key), "{key:?} must be removed");
}
for key in [
OBJECT_LOCK_LEGAL_HOLD,
OBJECT_LOCK_MODE,
OBJECT_LOCK_RETAIN_UNTIL_DATE,
SERVER_SIDE_ENCRYPTION,
STORAGE_CLASS,
REPLICATION_STATUS,
] {
assert!(obj.meta_user.contains_key(key), "{key:?} must survive restore cleanup");
}
assert!(!is_restored_object_on_disk(&obj.meta_user));
assert!(!obj.uses_data_dir());
}
/// Writing through the new constants must produce the same persisted key
/// set as the pre-module bytes, so a rollback reads new objects too.
#[test]
fn new_writes_persist_the_same_keys_as_pre_module_bytes() {
let mut fi = FileMeta::load(&create_pre_metadata_keys_xlmeta().expect("decode fixture hex"))
.expect("load fixture xl.meta")
.into_fileinfo("bucket", "object", FIXTURE_VERSION_ID, false, false, false)
.expect("fixture version to FileInfo");
fi.metadata = PINNED
.iter()
.zip(FIXTURE_VALUES)
.map(|((key, _), (_, value))| (key.to_string(), value.to_string()))
.collect();
fi.metadata
.insert("etag".to_string(), "d41d8cd98f00b204e9800998ecf8427e".to_string());
let mut fm = FileMeta::new();
fm.add_version(fi).expect("add version");
let rewritten = FileMeta::load(&fm.marshal_msg().expect("marshal")).expect("reload");
let rewritten = FileMetaVersion::try_from(rewritten.versions[0].meta.as_slice())
.expect("decode rewritten version")
.object
.expect("rewritten version is an object");
assert_eq!(rewritten.meta_user, fixture_object().meta_user);
}
/// Per-character mutation: a constant that differs from the persisted key
/// by any single byte (including ASCII case) finds nothing in pre-module
/// xl.meta, and a restore marker stored under such a key is not honored.
/// This is the drift the pins above turn into a test failure.
#[test]
fn single_character_key_mutation_misses_pre_module_data() {
let metadata = fixture_metadata();
let mut mutations = 0usize;
for (key, _) in PINNED {
for idx in 0..key.len() {
let mutated = mutate_at(key, idx);
assert_ne!(mutated, key);
assert!(
!metadata.contains_key(&mutated),
"mutation {mutated:?} of {key:?} must not match pre-module data"
);
mutations += 1;
}
}
assert_eq!(mutations, PINNED.iter().map(|(k, _)| k.len()).sum::<usize>());
let marker = metadata.get(RESTORE).expect("fixture restore marker").clone();
for idx in 0..RESTORE.len() {
let meta = HashMap::from([(mutate_at(RESTORE, idx), marker.clone())]);
assert!(
!is_restored_object_on_disk(&meta),
"restore marker under {:?} must not count as on disk",
mutate_at(RESTORE, idx)
);
}
}
}
crates/filemeta/src/test_data.rs
+8
View File
@@ -171,6 +171,14 @@ pub fn create_minio_large_object_xlmeta() -> Result<Vec<u8>> {
decode_hex_fixture(include_str!("../tests/fixtures/minio/object_large_bin.xlmeta.hex"))
}
/// xl.meta for one restored, object-locked object, written by the code that
/// predates `metadata_keys` with every persisted key taken from its historical
/// source (`s3s::header` / `rustfs_utils::http`). Never regenerate it: it is
/// the old-bytes evidence that the key authority stayed byte-identical.
pub fn create_pre_metadata_keys_xlmeta() -> Result<Vec<u8>> {
decode_hex_fixture(include_str!("../tests/fixtures/persisted_metadata_keys_pre_a3a.hex"))
}
fn write_legacy_time(wr: &mut Vec<u8>, ts: OffsetDateTime) {
wr.push(MSGPACK_EXT8);
wr.push(12);
crates/filemeta/tests/fixtures/persisted_metadata_keys_pre_a3a.hex
Vendored
+1
View File
@@ -0,0 +1 @@
584c322001000300c6000002aa030301c42697c4100b1e5a3a17354a3a800000000000a3a0d318c29c112e760000c40405541d2b01020204c5027c83a45479706501a556324f626ade0011a24944c4100b1e5a3a17354a3a800000000000a3a0a444446972c4100b1e5a3a17354a3a800000000000d1d0a64563416c676f01a345634d04a345634e02a745634253697a65ce00100000a74563496e64657801a645634469737496010203040506a84353756d416c676f01a8506172744e756d7390a9506172744554616773c0a95061727453697a657390aa506172744153697a6573c0a453697a65ce00010000a54d54696d65cf18c29c112e760000a74d657461537973c0a74d6574615573728aa465746167d9206434316438636439386630306232303465393830303939386563663834323765d923782d616d7a2d6f626a6563742d6c6f636b2d72657461696e2d756e74696c2d64617465b4323039392d30312d30315430303a30303a30305abc782d616d7a2d7365727665722d736964652d656e6372797074696f6ea6414553323536b8582d416d7a2d5265706c69636174696f6e2d537461747573a9434f4d504c45544544bc782d616d7a2d6f626a6563742d6c6f636b2d6c6567616c2d686f6c64a24f4eb9582d416d7a2d526573746f72652d4578706972792d44617973a137ad782d616d7a2d726573746f7265d93b6f6e676f696e672d726571756573743d2266616c7365222c206578706972792d646174653d22393939392d30312d30315430303a30303a30305a22ba582d416d7a2d526573746f72652d526571756573742d44617465bd5468752c203136204a756c20323032362030303a30303a303020474d54b6782d616d7a2d6f626a6563742d6c6f636b2d6d6f6465aa434f4d504c49414e4345b3782d616d7a2d73746f726167652d636c617373a7474c4143494552a17600ce35759b93