fix(object-lock): require source timestamps before replication passes WORM gate

Adversarial review of the replication WORM bypass found a silent unlock:
an authorized replication write that carries no source timestamp for a
category (the source never held the object, only its tags changed) is
not judged by receiver-side LWW, so the metadata replace would drop the
destination's legal hold or retention unjudged. Before the bypass that
write was merely rejected.

Gate the bypass on `replication_write_may_pass_worm_gate`: the write
passes only when it carries the source timestamp of every category that
currently locks the version, so LWW decides each one; otherwise it stays
WORM-rejected. The set layer evaluates the lock gate first so malformed
persisted lock metadata still fails closed, and the app-layer pre-check
applies the same rule.

Refs rustfs/backlog#1953

crates/ecstore/src/api/mod.rs

@@ -161,7 +161,7 @@ pub mod bucket {
         pub mod objectlock_sys {
             pub use crate::bucket::object_lock::objectlock_sys::{
                 BucketObjectLockSys, ObjectLockBlockReason, add_years, check_object_lock_for_deletion,
-                check_retention_for_modification, is_retention_active,
+                check_retention_for_modification, is_retention_active, replication_write_may_pass_worm_gate,
             };
         }
     }

crates/ecstore/src/bucket/object_lock/objectlock_sys.rs

@@ -15,7 +15,7 @@
 use crate::bucket::metadata_sys::{ObjectLockConfigState, get_object_lock_config, get_object_lock_config_state};
 use crate::bucket::object_lock::objectlock;
 use crate::error::{Error, Result, StorageError};
-use crate::object_api::ObjectInfo;
+use crate::object_api::{ObjectInfo, ObjectOptions};
 use s3s::dto::{Date, DefaultRetention, ObjectLockConfiguration, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
 use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE};
 use std::sync::Arc;
@@ -136,12 +136,41 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
 
 /// Check if an object has legal hold enabled.
 /// Returns true if legal hold is ON.
-#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
 fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
     let lhold = objectlock::get_object_legalhold_meta(user_defined);
     matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
 }
 
+/// Whether an authorized replication write (`ObjectOptions::replication_request`)
+/// may overwrite a locked destination version.
+///
+/// The source's lock state governs a replica (MinIO `checkPutObjectLockAllowed`
+/// skips the existing-version check for replicas), and a source-side hold
+/// release or retention change reaches this site only through this write. The
+/// overwrite is allowed only when the write carries the source timestamp of
+/// every category that currently locks the version, so receiver-side LWW
+/// (`merge_replication_metadata_lww`) judges each of them: a category locked
+/// more recently here is kept, otherwise the source's newer state wins. A write
+/// without that timestamp carries no source decision for the category — the
+/// metadata replace would lift the lock unjudged — so it stays WORM-rejected.
+pub fn replication_write_may_pass_worm_gate(
+    user_defined: &std::collections::HashMap<String, String>,
+    opts: &ObjectOptions,
+) -> bool {
+    if !opts.replication_request {
+        return false;
+    }
+    if has_legal_hold(user_defined) && opts.replication_legalhold_timestamp.is_none() {
+        return false;
+    }
+    let ret = objectlock::get_object_retention_meta(user_defined);
+    let retention_locked = ret
+        .mode
+        .as_ref()
+        .is_some_and(|mode| is_retention_active(mode.as_str(), ret.retain_until_date.as_ref()));
+    !(retention_locked && opts.replication_retention_timestamp.is_none())
+}
+
 /// Check if an object is locked based on its metadata.
 /// This is a common function used by both lifecycle evaluation and deletion checks.
 ///
@@ -491,6 +520,59 @@ mod tests {
         }
     }
 
+    /// A replication write passes the WORM gate only when it carries the
+    /// source timestamp of every category that currently locks the version.
+    #[test]
+    fn replication_write_passes_worm_gate_only_with_every_locking_category_timestamp() {
+        use rustfs_utils::http::headers::{
+            AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
+        };
+
+        let hold = [(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "ON")];
+        let retention = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "GOVERNANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2099-01-01T00:00:00Z"),
+        ];
+        let expired = [
+            (AMZ_OBJECT_LOCK_MODE_LOWER, "COMPLIANCE"),
+            (AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, "2000-01-01T00:00:00Z"),
+        ];
+        let metadata = |entries: &[&[(&str, &str)]]| -> std::collections::HashMap<String, String> {
+            entries
+                .iter()
+                .flat_map(|entries| entries.iter())
+                .map(|(key, value)| (key.to_string(), value.to_string()))
+                .collect()
+        };
+        let opts = |hold_ts: bool, retention_ts: bool| ObjectOptions {
+            replication_request: true,
+            replication_legalhold_timestamp: hold_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            replication_retention_timestamp: retention_ts.then_some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+
+        let locked_by_both = metadata(&[&hold, &retention]);
+        assert!(replication_write_may_pass_worm_gate(&locked_by_both, &opts(true, true)));
+        assert!(!replication_write_may_pass_worm_gate(&locked_by_both, &opts(true, false)));
+        assert!(!replication_write_may_pass_worm_gate(&locked_by_both, &opts(false, true)));
+
+        assert!(replication_write_may_pass_worm_gate(&metadata(&[&hold]), &opts(true, false)));
+        assert!(!replication_write_may_pass_worm_gate(&metadata(&[&hold]), &opts(false, true)));
+        assert!(replication_write_may_pass_worm_gate(&metadata(&[&retention]), &opts(false, true)));
+        assert!(!replication_write_may_pass_worm_gate(&metadata(&[&retention]), &opts(true, false)));
+
+        // Expired retention and a released hold no longer lock anything.
+        let unlocked = metadata(&[&expired, &[(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, "OFF")]]);
+        assert!(replication_write_may_pass_worm_gate(&unlocked, &opts(false, false)));
+
+        // Never for a non-replication write, whatever it carries.
+        let local = ObjectOptions {
+            replication_request: false,
+            ..opts(true, true)
+        };
+        assert!(!replication_write_may_pass_worm_gate(&metadata(&[&hold]), &local));
+    }
+
     /// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
     /// lock keys as empty strings (the MinIO on-disk shape, see
     /// `parse_object_lock_retention`); that is "no lock", not corruption, and

crates/ecstore/src/set_disk/mod.rs

@@ -47,6 +47,7 @@ use crate::bucket::metadata_sys;
 use crate::bucket::metadata_sys::ObjectLockConfigState;
 use crate::bucket::object_lock::objectlock_sys::{
     check_object_lock_for_deletion_with_config, check_object_lock_for_deletion_with_state, check_retention_for_modification,
+    replication_write_may_pass_worm_gate,
 };
 use crate::bucket::replication::{
     ReplicateDecision, ReplicationObjectBridge, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,

crates/ecstore/src/set_disk/ops/object.rs

@@ -2662,16 +2662,13 @@ impl SetDisks {
                             Error::other("explicit-version PUT is missing its Object Lock configuration snapshot")
                         })?;
                         // The WORM gate protects the locked version from local
-                        // overwrites only. For an authorized replication write
-                        // (ReplicateObjectAction, `replication_request`) the
-                        // source's lock state governs the replica, as in MinIO's
-                        // `checkPutObjectLockAllowed` (`!replica` guard): a
-                        // legal-hold release or retention change reaches this
-                        // site only through this write, and rejecting it loops
-                        // through MRF forever. Receiver-side LWW below still
-                        // keeps a category locked more recently here.
-                        if !opts.replication_request
-                            && check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some()
+                        // overwrites; an authorized replication write passes it
+                        // only when the LWW merge below will judge every
+                        // locking category (see
+                        // `replication_write_may_pass_worm_gate`). Gate first so
+                        // malformed lock metadata still fails closed.
+                        if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some()
+                            && !replication_write_may_pass_worm_gate(&existing.user_defined, opts)
                         {
                             return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
                         }
@@ -8379,15 +8376,20 @@ mod replication_lww_tests {
         put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, local)).await;
     }
 
+    /// Inbound legal-hold release from a source that also carries the (same)
+    /// COMPLIANCE retention; the sender stamps a source timestamp for every
+    /// category the source version has.
     fn inbound_legal_hold_release_opts(version_id: &str, timestamp: &str) -> ObjectOptions {
         let mut inbound = HashMap::new();
         inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
         insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, timestamp.to_string());
         inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
         inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
+        insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
         ObjectOptions {
             replication_request: true,
             replication_legalhold_timestamp: Some(parse_ts(timestamp)),
+            replication_retention_timestamp: Some(parse_ts(T_OLD)),
             ..versioned_opts(version_id, inbound)
         }
     }
@@ -8460,6 +8462,66 @@ mod replication_lww_tests {
         );
     }
 
+    /// A replication write that carries no source decision for a locking
+    /// category (here: tags changed at a source that never held the object)
+    /// must not lift the destination's hold by replacing the metadata
+    /// unjudged; it stays WORM-rejected like a local overwrite.
+    #[tokio::test]
+    async fn inbound_without_legal_hold_timestamp_stays_rejected_on_held_version() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-locked-unjudged-category";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        seed_locked_version(&set_disks, bucket, object, &version_id, T_OLD).await;
+
+        let mut inbound = HashMap::new();
+        inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "k=v".to_string());
+        insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_NEW.to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
+        inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2099-01-01T00:00:00Z".to_string());
+        let opts = ObjectOptions {
+            replication_request: true,
+            replication_tagging_timestamp: Some(parse_ts(T_NEW)),
+            replication_retention_timestamp: Some(parse_ts(T_NEW)),
+            replication_legalhold_timestamp: None,
+            ..versioned_opts(&version_id, inbound)
+        };
+        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
+        let err = set_disks
+            .put_object(bucket, object, &mut reader, &opts)
+            .await
+            .expect_err("a replication write without the legal-hold source timestamp must stay rejected");
+        assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("ON"));
+    }
+
+    /// The gate runs before the replication bypass, so malformed persisted
+    /// lock metadata still fails closed for an authorized replication write.
+    #[tokio::test]
+    async fn replication_write_on_malformed_lock_metadata_still_fails_closed() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "lww-locked-malformed";
+        let object = "object";
+        let version_id = Uuid::new_v4().to_string();
+        make_bucket(&disk_stores, bucket).await;
+        let mut local = HashMap::new();
+        local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "MAYBE".to_string());
+        put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
+
+        let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
+        let err = set_disks
+            .put_object(bucket, object, &mut reader, &inbound_legal_hold_release_opts(&version_id, T_NEW))
+            .await
+            .expect_err("malformed persisted lock metadata must fail the replication write closed");
+        assert!(!matches!(err, StorageError::PrefixAccessDenied(_, _)), "unexpected error: {err}");
+
+        let info = version_info(&set_disks, bucket, object, &version_id).await;
+        assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), Some("MAYBE"));
+    }
+
     /// The bypass is scoped to authorized replication writes: the same
     /// explicit-version PUT without `replication_request` stays WORM-rejected.
     #[tokio::test]