test(ecstore): pin persisted metadata key literals and bucket config goldens (#5904)

crates/ecstore/src/api/mod.rs

@@ -453,6 +453,11 @@ pub mod rpc {
 pub mod set_disk {
     pub use crate::set_disk::{DEFAULT_READ_BUFFER_SIZE, SetDisks, get_lock_acquire_timeout, is_valid_storage_class};
 
+    /// Return the canonical object-metadata identity used for read-quorum grouping.
+    pub fn file_info_quorum_hash(meta: &rustfs_filemeta::FileInfo) -> [u8; 32] {
+        crate::set_disk::SetDisks::file_info_quorum_hash(meta)
+    }
+
     #[cfg(feature = "test-util")]
     pub mod test_util {
         pub use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};

crates/ecstore/src/bucket/object_lock/objectlock.rs

@@ -200,6 +200,29 @@ mod tests {
         assert!(retention.retain_until_date.is_some());
     }
 
+    /// backlog#1733 g-key-002: the persisted literal keys must still be read
+    /// through the current header constants, or WORM metadata fails open.
+    #[test]
+    fn persisted_compliance_lock_metadata_remains_effective() {
+        let mut meta = HashMap::new();
+        meta.insert("x-amz-object-lock-mode".to_string(), "COMPLIANCE".to_string());
+        meta.insert("x-amz-object-lock-retain-until-date".to_string(), "9999-01-01T00:00:00Z".to_string());
+        meta.insert("x-amz-object-lock-legal-hold".to_string(), "ON".to_string());
+
+        let retention = get_object_retention_meta(&meta);
+        assert_eq!(
+            retention.mode.as_ref().map(|mode| mode.as_str()),
+            Some(ObjectLockRetentionMode::COMPLIANCE)
+        );
+        assert!(retention.retain_until_date.is_some(), "persisted retention date must remain readable");
+
+        let legal_hold = get_object_legalhold_meta(&meta);
+        assert_eq!(
+            legal_hold.status.as_ref().map(|status| status.as_str()),
+            Some(ObjectLockLegalHoldStatus::ON)
+        );
+    }
+
     #[test]
     fn test_get_object_legalhold_meta_empty() {
         let meta = HashMap::new();

crates/ecstore/src/set_disk/metadata.rs

@@ -580,7 +580,7 @@ impl SetDisks {
         }
     }
 
-    pub(super) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
+    pub(crate) fn file_info_quorum_hash(meta: &FileInfo) -> [u8; 32] {
         let mut hasher = Sha256::new();
         Self::update_file_info_quorum_hash(&mut hasher, meta);
         let digest = hasher.finalize();

crates/filemeta/src/filemeta.rs

@@ -1016,6 +1016,83 @@ mod test {
     use proptest::collection::vec;
     use proptest::prelude::*;
 
+    /// A restore header meaning "restored copy is on disk until far in the future".
+    /// Format produced by `RestoreStatusOps::to_string` and consumed by
+    /// `parse_restore_obj_status` (fileinfo.rs).
+    const RESTORED_ON_DISK: &str = "ongoing-request=\"false\", expiry-date=\"9999-01-01T00:00:00Z\"";
+
+    /// backlog#1733 (P9-01 §4.3/§7.6, g-key-001): pin the five `s3s::header`
+    /// constants that double as **persisted metadata map keys**. They are not
+    /// just HTTP header names — they are stored inside xl.meta (`meta_user`)
+    /// and read back by fail-open code, so a silent drift produces zero
+    /// HTTP-visible errors while:
+    ///
+    /// 1. **WORM silently dissolves** — `get_object_retention_meta`
+    ///    (ecstore objectlock.rs) returns an empty retention when the lock keys
+    ///    are unreadable, making every compliance-locked object deletable.
+    /// 2. **Live data dirs can be reclaimed** — `MetaObject::uses_data_dir`
+    ///    falls back to `is_restored_object_on_disk`, which returns `false`
+    ///    when `x-amz-restore` is unreadable, so a restored object's data dir
+    ///    is judged unused.
+    ///
+    /// Any migration replacing these constants must keep the literals byte-stable.
+    #[test]
+    fn persisted_metadata_keys_are_byte_stable() {
+        use s3s::header::{
+            X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
+            X_AMZ_SERVER_SIDE_ENCRYPTION,
+        };
+        assert_eq!(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str(), "x-amz-object-lock-legal-hold");
+        assert_eq!(X_AMZ_OBJECT_LOCK_MODE.as_str(), "x-amz-object-lock-mode");
+        assert_eq!(X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str(), "x-amz-object-lock-retain-until-date");
+        assert_eq!(X_AMZ_RESTORE.as_str(), "x-amz-restore");
+        assert_eq!(X_AMZ_SERVER_SIDE_ENCRYPTION.as_str(), "x-amz-server-side-encryption");
+    }
+
+    /// backlog#1733 g-key-003: a restored-to-local object must keep its data
+    /// dir. The restore marker lives under the pinned `x-amz-restore` key; if
+    /// the key ever drifts this flips to `false` and the data dir becomes
+    /// eligible for reclamation while the restored copy is still being served.
+    #[test]
+    fn restored_object_keeps_using_data_dir() {
+        let mut obj = MetaObject::default();
+        obj.meta_user
+            .insert("x-amz-restore".to_string(), RESTORED_ON_DISK.to_string());
+        assert!(obj.uses_data_dir(), "restored object's data dir must be considered in use");
+
+        // The same fail-open shape the pin protects against: without the marker
+        // the data dir is judged unused — exactly what a key drift would cause.
+        let bare = MetaObject::default();
+        assert!(!bare.uses_data_dir(), "object without restore marker reports data dir unused");
+    }
+
+    /// backlog#1733 g-key-004: a transition-complete object short-circuits to
+    /// `false` even when the restore marker is present — the existing
+    /// precedence must not change.
+    #[test]
+    fn transition_complete_object_does_not_use_data_dir() {
+        use rustfs_utils::http::{SUFFIX_TRANSITION_STATUS, insert_bytes};
+        let mut obj = MetaObject::default();
+        obj.meta_user
+            .insert("x-amz-restore".to_string(), RESTORED_ON_DISK.to_string());
+        insert_bytes(&mut obj.meta_sys, SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.as_bytes().to_vec());
+        assert!(!obj.uses_data_dir(), "transition-complete short-circuit must win over the restore marker");
+    }
+
+    /// The restore-header parser and the pinned key literal must agree: the
+    /// marker written under `x-amz-restore` is only meaningful if the parser
+    /// accepts it.
+    #[test]
+    fn restore_marker_roundtrips_through_parser() {
+        let mut meta = HashMap::new();
+        meta.insert(X_AMZ_RESTORE.as_str().to_string(), RESTORED_ON_DISK.to_string());
+        assert!(crate::is_restored_object_on_disk(&meta));
+
+        // An in-progress restore is not "on disk".
+        meta.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"true\"".to_string());
+        assert!(!crate::is_restored_object_on_disk(&meta));
+    }
+
     /// backlog#580: RustFS parses real MinIO-written object xl.meta (inline,
     /// versioned, and multipart) into equivalent `FileInfo`. Object metadata is
     /// the strong part of MinIO interop; this pins it against real fixtures.

crates/filemeta/src/filemeta/version.rs

@@ -1855,6 +1855,7 @@ impl From<MetaObjectV1ChecksumInfo> for ChecksumInfo {
                 "highwayhash256" => HashAlgorithm::HighwayHash256,
                 "highwayhash256S" => HashAlgorithm::HighwayHash256S,
                 "blake2b" | "blake2b512" => HashAlgorithm::BLAKE2b512,
+                "md5" => HashAlgorithm::Md5,
                 _ => HashAlgorithm::HighwayHash256S,
             },
             hash: Bytes::from(value.hash),

crates/filemeta/src/test_data.rs

@@ -189,7 +189,14 @@ fn encode_legacy_v1_header(version_id: Uuid, mod_time: OffsetDateTime) -> Vec<u8
     wr
 }
 
-fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateTime) -> Vec<u8> {
+fn encode_legacy_v1_body(
+    version_id: Uuid,
+    data_dir: Uuid,
+    mod_time: OffsetDateTime,
+    erasure_index: usize,
+    checksum: Option<(&str, &[u8])>,
+    object_size: usize,
+) -> Vec<u8> {
     let mut wr = Vec::new();
 
     rmp::encode::write_map_len(&mut wr, 3).unwrap();
@@ -208,7 +215,7 @@ fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateT
     rmp::encode::write_str(&mut wr, "Stat").unwrap();
     rmp::encode::write_map_len(&mut wr, 5).unwrap();
     rmp::encode::write_str(&mut wr, "Size").unwrap();
-    rmp::encode::write_sint(&mut wr, 11).unwrap();
+    rmp::encode::write_sint(&mut wr, object_size as i64).unwrap();
     rmp::encode::write_str(&mut wr, "ModTime").unwrap();
     write_legacy_time(&mut wr, mod_time);
     rmp::encode::write_str(&mut wr, "Name").unwrap();
@@ -229,14 +236,23 @@ fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateT
     rmp::encode::write_str(&mut wr, "BlockSize").unwrap();
     rmp::encode::write_sint(&mut wr, 1_048_576).unwrap();
     rmp::encode::write_str(&mut wr, "Index").unwrap();
-    rmp::encode::write_sint(&mut wr, 1).unwrap();
+    rmp::encode::write_sint(&mut wr, erasure_index as i64).unwrap();
     rmp::encode::write_str(&mut wr, "Distribution").unwrap();
     rmp::encode::write_array_len(&mut wr, 6).unwrap();
     for value in 1..=6 {
         rmp::encode::write_sint(&mut wr, value).unwrap();
     }
     rmp::encode::write_str(&mut wr, "Checksums").unwrap();
-    rmp::encode::write_array_len(&mut wr, 0).unwrap();
+    rmp::encode::write_array_len(&mut wr, u32::from(checksum.is_some())).unwrap();
+    if let Some((algorithm, hash)) = checksum {
+        rmp::encode::write_map_len(&mut wr, 3).unwrap();
+        rmp::encode::write_str(&mut wr, "PartNumber").unwrap();
+        rmp::encode::write_sint(&mut wr, 1).unwrap();
+        rmp::encode::write_str(&mut wr, "Algorithm").unwrap();
+        rmp::encode::write_str(&mut wr, algorithm).unwrap();
+        rmp::encode::write_str(&mut wr, "Hash").unwrap();
+        rmp::encode::write_bin(&mut wr, hash).unwrap();
+    }
 
     rmp::encode::write_str(&mut wr, "Meta").unwrap();
     rmp::encode::write_map_len(&mut wr, 1).unwrap();
@@ -251,9 +267,9 @@ fn encode_legacy_v1_body(version_id: Uuid, data_dir: Uuid, mod_time: OffsetDateT
     rmp::encode::write_str(&mut wr, "n").unwrap();
     rmp::encode::write_sint(&mut wr, 1).unwrap();
     rmp::encode::write_str(&mut wr, "s").unwrap();
-    rmp::encode::write_sint(&mut wr, 11).unwrap();
+    rmp::encode::write_sint(&mut wr, object_size as i64).unwrap();
     rmp::encode::write_str(&mut wr, "as").unwrap();
-    rmp::encode::write_sint(&mut wr, 11).unwrap();
+    rmp::encode::write_sint(&mut wr, object_size as i64).unwrap();
     rmp::encode::write_str(&mut wr, "mt").unwrap();
     write_legacy_time(&mut wr, mod_time);
 
@@ -275,8 +291,29 @@ pub fn create_legacy_v1_object_xlmeta() -> Result<Vec<u8>> {
     let mod_time = OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789)?;
 
     let header = encode_legacy_v1_header(version_id, mod_time);
-    let body = encode_legacy_v1_body(version_id, data_dir, mod_time);
+    let body = encode_legacy_v1_body(version_id, data_dir, mod_time, 1, None, 11);
 
+    encode_legacy_v1_xlmeta(header, body)
+}
+
+/// Legacy V1 xl.meta fixture with a per-drive whole-file bitrot checksum.
+pub fn create_legacy_v1_object_xlmeta_with_checksum(
+    erasure_index: usize,
+    algorithm: &str,
+    hash: &[u8],
+    object_size: usize,
+) -> Result<Vec<u8>> {
+    let version_id = Uuid::parse_str("01234567-89ab-cdef-0123-456789abcdef")?;
+    let data_dir = Uuid::parse_str("fedcba98-7654-3210-fedc-ba9876543210")?;
+    let mod_time = OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789)?;
+
+    let header = encode_legacy_v1_header(version_id, mod_time);
+    let body = encode_legacy_v1_body(version_id, data_dir, mod_time, erasure_index, Some((algorithm, hash)), object_size);
+
+    encode_legacy_v1_xlmeta(header, body)
+}
+
+fn encode_legacy_v1_xlmeta(header: Vec<u8>, body: Vec<u8>) -> Result<Vec<u8>> {
     let mut wr = Vec::new();
     wr.extend_from_slice(b"XL2 ");
     wr.extend_from_slice(&1u16.to_le_bytes());