feat(replication): replicate managed-SSE objects via target re-encryption (#5885)

Open the managed-SSE replication gate (backlog#1783, PR-B of 3, after
#5872): the replication reader already decrypts through the injected
object-encryption resolver, so the source sends plaintext plus an
encryption intent header (AES256 / aws:kms, never the source key id) and
the target re-encrypts on its normal PUT path with its own KMS. No DEK
crosses sites.

- replication_put_object_options: fail closed only on Unsupported;
  insert the SSE intent after the strip loop.
- TargetClient::create_multipart_upload sends the full opts.header()
  set, fixing multipart replicas losing content-type/user metadata
  (plaintext included).
- Preserve source ETag and mtime on replicas (authorized replication
  only): receiver wires x-rustfs-source-etag into preserve_etag for PUT
  and CompleteMultipartUpload, resolve_complete_etag consumes it, and
  complete options carry source_etag/source_mtime (absent mtime
  degrades to epoch, not now_utc). Without this every replication HEAD
  comparison re-drives re-encrypted objects forever.
- e2e: managed SSE contracts flip to success on an independent-KMS
  dual-process pair (byte-identical plain GET proves target-owned
  envelopes; ETag/mtime preserved; version stable across scanner
  cycles; resync converges; multipart keeps structure and metadata);
  new target-without-KMS fail-closed contract; SSE-C stays FAILED.

Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -1922,14 +1922,11 @@ impl TargetClient {
         object: &str,
         opts: &PutObjectOptions,
     ) -> Result<String, S3ClientError> {
-        let mut headers = HeaderMap::new();
+        // Object metadata belongs to CreateMultipartUpload in S3 semantics;
+        // building only the source-version headers here used to drop user
+        // metadata, content-type, and the SSE intent for multipart replicas.
+        let headers = opts.header();
         let version_id = opts.internal.source_version_id.clone();
-        if !version_id.is_empty() {
-            insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, &version_id);
-        }
-        if opts.internal.replication_request {
-            insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
-        }
         // The remote version of a multipart replication is decided at initiate
         // time; CompleteMultipartUpload does not read a versionId.
         let api_version_id = resolve_put_api_version_id(&version_id).map(ToOwned::to_owned);

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2756,8 +2756,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
             }
         };
 
-        // TODO: SSE
-
         if tgt_client.bucket.is_empty() {
             debug!(
                 event = EVENT_RESYNC_RUNTIME_SKIPPED,
@@ -3204,7 +3202,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
         object,
         &upload_id,
         uploaded_parts,
-        &replication_complete_multipart_options(actual_size),
+        &replication_complete_multipart_options(actual_size, object_info.etag.clone().unwrap_or_default(), object_info.mod_time),
     )
     .await
     .map_err(|e| std::io::Error::other(e.to_string()))?;

crates/ecstore/src/bucket/replication/replication_target_boundary.rs

@@ -62,7 +62,6 @@ static STANDARD_HEADERS: &[&str] = &[
     AMZ_SERVER_SIDE_ENCRYPTION,
 ];
 
-const ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED: &str = "managed SSE replication requires target encryption support";
 const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -160,14 +159,8 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
     let source_encryption = classify_replication_source_encryption(&object_info.user_defined);
     let is_ssec = matches!(source_encryption, ReplicationSourceEncryption::SseC);
 
-    match source_encryption {
-        ReplicationSourceEncryption::Plaintext | ReplicationSourceEncryption::SseC => {}
-        ReplicationSourceEncryption::SseS3 | ReplicationSourceEncryption::SseKms => {
-            return Err(Error::other(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
-        }
-        ReplicationSourceEncryption::Unsupported => {
-            return Err(Error::other(ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED));
-        }
+    if matches!(source_encryption, ReplicationSourceEncryption::Unsupported) {
+        return Err(Error::other(ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED));
     }
 
     for (key, value) in object_info.user_defined.iter() {
@@ -190,6 +183,16 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
         meta.insert(key.to_string(), value.to_string());
     }
 
+    // Managed SSE replicates as plaintext (the replication reader decrypts via
+    // the object-encryption resolver) and re-encrypts on the target with the
+    // target's own KMS. Send only the encryption intent — never the source
+    // key id, whose meaning is local to the source site's KMS.
+    if matches!(source_encryption, ReplicationSourceEncryption::SseS3) {
+        meta.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string());
+    } else if matches!(source_encryption, ReplicationSourceEncryption::SseKms) {
+        meta.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string());
+    }
+
     let mut is_multipart = object_info.is_multipart();
 
     if let Some(checksum_data) = &object_info.checksum
@@ -402,13 +405,22 @@ pub(crate) fn replication_force_delete_remove_options() -> RemoveObjectOptions {
     }
 }
 
-pub(crate) fn replication_complete_multipart_options(actual_size: String) -> PutObjectOptions {
+pub(crate) fn replication_complete_multipart_options(
+    actual_size: String,
+    source_etag: String,
+    source_mtime: Option<OffsetDateTime>,
+) -> PutObjectOptions {
     let mut user_metadata = HashMap::new();
     insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
 
     PutObjectOptions {
         user_metadata,
         internal: AdvancedPutOptions {
+            source_etag,
+            // AdvancedPutOptions::default() stamps now_utc(); an absent source
+            // mtime must degrade to epoch so header() suppresses the header
+            // instead of asserting the replication time as the object's mtime.
+            source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
             replication_status: ReplicationStatusType::Replica,
             replication_request: true,
             ..Default::default()
@@ -573,7 +585,21 @@ mod tests {
 
     #[test]
     fn replication_complete_multipart_options_sets_actual_size() {
-        let options = replication_complete_multipart_options("1024".to_string());
+        let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
+        let options = replication_complete_multipart_options(
+            "1024".to_string(),
+            "0123456789abcdef0123456789abcdef-3".to_string(),
+            Some(source_mtime),
+        );
+        assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
+        assert_eq!(options.internal.source_mtime, source_mtime);
+
+        // Absent source mtime must degrade to epoch (header suppressed), not
+        // the AdvancedPutOptions default of now_utc() — that default would
+        // stamp the replication time as the replica's mtime and break the
+        // multipart HEAD convergence.
+        let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
+        assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
 
         assert_eq!(
             get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
@@ -809,36 +835,75 @@ mod tests {
     }
 
     #[test]
-    fn replication_put_options_rejects_sse_s3_until_target_encryption_is_supported() {
-        let object_info = ObjectInfo {
-            user_defined: Arc::new(HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string())])),
-            ..Default::default()
+    fn replication_put_options_sends_sse_s3_intent_without_source_material() {
+        use rustfs_utils::http::object_encryption_keys::{
+            INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
+            INTERNAL_ENCRYPTION_KEY_ID_HEADER, INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
         };
 
-        let err = match replication_put_object_options("", &object_info) {
-            Ok(_) => panic!("SSE-S3 replication should fail closed until target encryption headers are supported"),
-            Err(err) => err,
-        };
-
-        assert!(err.to_string().contains(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
-    }
-
-    #[test]
-    fn replication_put_options_rejects_sse_kms_until_target_encryption_is_supported() {
+        // The stored shape of a managed SSE-S3 object per
+        // encryption_material_to_metadata: SSE marker plus envelope material.
         let object_info = ObjectInfo {
             user_defined: Arc::new(HashMap::from([
-                (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
-                (AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "key-1".to_string()),
+                (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string()),
+                (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()),
+                (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "sealed-envelope".to_string()),
+                (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "iv".to_string()),
+                (INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), "AES256-GCM".to_string()),
+                (INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), "1024".to_string()),
+                ("x-user-meta".to_string(), "value".to_string()),
             ])),
             ..Default::default()
         };
 
-        let err = match replication_put_object_options("", &object_info) {
-            Ok(_) => panic!("SSE-KMS replication should fail closed until target encryption headers are supported"),
-            Err(err) => err,
+        let (options, _) = replication_put_object_options("", &object_info).expect("managed SSE-S3 must build put options");
+
+        assert_eq!(options.user_metadata.get(AMZ_SERVER_SIDE_ENCRYPTION), Some(&"AES256".to_string()));
+        assert_eq!(options.user_metadata.get("x-user-meta"), Some(&"value".to_string()));
+        // No envelope material and no key id may leave the source.
+        assert!(!options.user_metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER));
+        assert!(!options.user_metadata.contains_key(INTERNAL_ENCRYPTION_KEY_ID_HEADER));
+        assert!(!options.user_metadata.contains_key(INTERNAL_ENCRYPTION_IV_HEADER));
+        assert!(
+            !options.user_metadata.values().any(|value| value.contains("sealed-envelope")),
+            "source envelope material must never leave the source site"
+        );
+    }
+
+    #[test]
+    fn replication_put_options_sends_sse_kms_intent_without_source_key_id() {
+        use rustfs_utils::http::object_encryption_keys::{
+            INTERNAL_ENCRYPTION_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
         };
 
-        assert!(err.to_string().contains(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
+        let object_info = ObjectInfo {
+            user_defined: Arc::new(HashMap::from([
+                (AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
+                (AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "source-key-1".to_string()),
+                (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "sealed-envelope".to_string()),
+                (MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), "ctx".to_string()),
+            ])),
+            ..Default::default()
+        };
+
+        let (options, _) = replication_put_object_options("", &object_info).expect("managed SSE-KMS must build put options");
+
+        // Intent only: the target encrypts with its own default KMS key.
+        assert_eq!(options.user_metadata.get(AMZ_SERVER_SIDE_ENCRYPTION), Some(&"aws:kms".to_string()));
+        assert!(!options.user_metadata.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID));
+        assert!(!options.user_metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER));
+        assert!(
+            !options
+                .user_metadata
+                .contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
+        );
+        assert!(
+            !options
+                .user_metadata
+                .values()
+                .any(|value| value.contains("sealed-envelope") || value.contains("source-key-1")),
+            "source KMS identifiers and envelopes must never leave the source site"
+        );
     }
 
     #[test]

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -1919,13 +1919,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         }
 
         // etag
-        let etag = {
-            if let Some(etag) = opts.user_defined.get("etag") {
-                etag.clone()
-            } else {
-                get_complete_multipart_md5(&uploaded_parts)
-            }
-        };
+        let etag = resolve_complete_etag(opts, &uploaded_parts);
 
         fi.metadata.insert("etag".to_owned(), etag);
 
@@ -2167,6 +2161,21 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
     }
 }
 
+/// Final ETag for a completed multipart object. An authorized replication
+/// request preserves the source ETag so the replication HEAD comparison
+/// converges even when the source ETag is not derivable from the uploaded
+/// parts (foreign-origin objects, ciphertext-derived ETags); the internal
+/// metadata override comes next; otherwise the ETag is computed from parts.
+fn resolve_complete_etag(opts: &ObjectOptions, uploaded_parts: &[CompletePart]) -> String {
+    if let Some(etag) = opts.preserve_etag.as_ref().filter(|etag| !etag.is_empty()) {
+        return etag.clone();
+    }
+    if let Some(etag) = opts.user_defined.get("etag") {
+        return etag.clone();
+    }
+    get_complete_multipart_md5(uploaded_parts)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -5190,4 +5199,28 @@ mod tests {
             assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
         }
     }
+
+    #[test]
+    fn resolve_complete_etag_prefers_preserved_source_etag() {
+        // A replication-preserved ETag that no part combination can derive
+        // (foreign-origin object) must win over the computed md5-of-parts.
+        let foreign_etag = "11111111111111111111111111111111-7".to_string();
+        let opts = ObjectOptions {
+            preserve_etag: Some(foreign_etag.clone()),
+            ..Default::default()
+        };
+        assert_eq!(resolve_complete_etag(&opts, &[]), foreign_etag);
+
+        // Empty preserve value degrades to the next source.
+        let opts_empty = ObjectOptions {
+            preserve_etag: Some(String::new()),
+            user_defined: std::collections::HashMap::from([("etag".to_string(), "override-etag".to_string())]),
+            ..Default::default()
+        };
+        assert_eq!(resolve_complete_etag(&opts_empty, &[]), "override-etag");
+
+        // Without either source the ETag is computed from the parts.
+        let computed = resolve_complete_etag(&ObjectOptions::default(), &[]);
+        assert_eq!(computed, get_complete_multipart_md5(&[]));
+    }
 }