diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index d47a027c1..f04530a32 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -65,9 +65,9 @@ use super::replication_target_boundary::{ SsecPassthroughGate, TargetClient, VersionIdentityCapability, is_replication_target_offline_error, replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted, - replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version, - resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_capability_from_put, - version_identity_drifted, + replication_put_object_header_size, replication_put_object_options, replication_source_etag_is_comparable, + replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present, + ssec_passthrough_gate, version_identity_capability_from_put, version_identity_drifted, }; use super::replication_versioning_boundary::ReplicationVersioningStore; use super::runtime_boundary as runtime_sources; @@ -349,15 +349,22 @@ fn replica_etag_verification_enabled() -> bool { /// target that stores a transformed payload (e.g. undecoded `aws-chunked` /// frames, #6853) returns the ETag of what it actually wrote. Reporting /// COMPLETED over such a replica is silent corruption, so a decidable -/// mismatch fails the replication instead. An SSE-C ciphertext passthrough -/// transfer is exempt: the wire bytes are ciphertext while the source ETag is -/// the plaintext MD5, and that path has its own HEAD-back audit. +/// mismatch fails the replication instead. Encrypted sources are exempt: an +/// SSE-C ciphertext passthrough sends ciphertext while the source ETag is the +/// plaintext MD5 (that path has its own HEAD-back audit), and a managed-SSE +/// (SSE-S3 / SSE-KMS) source stores a keyed fingerprint, or a legacy plaintext +/// MD5, that the ETag of the target's re-encrypted replica can neither confirm +/// nor refute. The checksum header on the replication PUT stays the integrity +/// check for those objects. fn verify_single_part_replica( object_info: &ObjectInfo, response: &RemotePutObjectResponse, ciphertext_passthrough: bool, ) -> std::result::Result<(), std::io::Error> { - if ciphertext_passthrough || !replica_etag_verification_enabled() { + if ciphertext_passthrough + || !replication_source_etag_is_comparable(&object_info.user_defined) + || !replica_etag_verification_enabled() + { return Ok(()); } if single_part_replica_etag_mismatch(object_info.etag.as_deref(), response.etag.as_deref()) { @@ -5870,6 +5877,57 @@ mod tests { use super::super::replication_filemeta_boundary::ReplicateTargetDecision; use super::super::replication_object_decision_boundary::ReplicationMultipartPlanError; + #[test] + fn verify_single_part_replica_exempts_managed_sse_sources() { + // A new encrypted write stores a keyed fingerprint; a target that + // computes its own ETag answers with the MD5 of the plaintext it + // received. Both are 32 hex characters, so without the exemption the + // audit would fail every SSE-S3 / SSE-KMS replica (rustfs/backlog). + let fingerprint = "00112233445566778899aabbccddeeff"; + let replica_md5 = "ffeeddccbbaa99887766554433221100"; + let response = RemotePutObjectResponse { + version_id: None, + etag: Some(replica_md5.to_string()), + }; + for sse in ["AES256", "aws:kms"] { + let object_info = ObjectInfo { + bucket: "source".to_string(), + name: "encrypted".to_string(), + etag: Some(fingerprint.to_string()), + user_defined: HashMap::from([("x-amz-server-side-encryption".to_string(), sse.to_string())]).into(), + ..Default::default() + }; + verify_single_part_replica(&object_info, &response, false) + .unwrap_or_else(|err| panic!("{sse}: a managed-SSE source must not be audited against the replica MD5: {err}")); + } + } + + #[test] + fn verify_single_part_replica_still_fails_plaintext_mismatch() { + if !replica_etag_verification_enabled() { + return; + } + let object_info = ObjectInfo { + bucket: "source".to_string(), + name: "plain".to_string(), + etag: Some("00112233445566778899aabbccddeeff".to_string()), + ..Default::default() + }; + let mismatch = RemotePutObjectResponse { + version_id: None, + etag: Some("ffeeddccbbaa99887766554433221100".to_string()), + }; + let err = verify_single_part_replica(&object_info, &mismatch, false) + .expect_err("a plaintext source whose replica MD5 differs must fail the replication"); + assert_eq!(err.to_string(), REPLICA_ETAG_MISMATCH_ERROR); + + let matching = RemotePutObjectResponse { + version_id: None, + etag: object_info.etag.clone(), + }; + verify_single_part_replica(&object_info, &matching, false).expect("matching plaintext MD5s pass the audit"); + } + #[test] fn multipart_read_plan_preserves_legacy_plain_part_ranges() { const MIB: usize = 1024 * 1024; diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index e71ae9f19..e425bad99 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -157,6 +157,19 @@ pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap) -> bool { + matches!( + classify_replication_source_encryption(user_defined), + ReplicationSourceEncryption::Plaintext + ) +} + /// HeadObjectOutput adapter over the pure SSE-C passthrough evidence /// judgment owned by `rustfs-replication`: extract the echoed /// customer-algorithm header and let the crate-owned policy decide. @@ -578,6 +591,22 @@ mod tests { use time::Duration; use uuid::Uuid; + #[test] + fn replication_source_etag_is_comparable_only_for_plaintext_sources() { + assert!(replication_source_etag_is_comparable(&HashMap::new())); + for (key, value) in [ + (metadata_keys::SERVER_SIDE_ENCRYPTION, "AES256"), + (metadata_keys::SERVER_SIDE_ENCRYPTION, "aws:kms"), + (SSEC_ALGORITHM_HEADER, "AES256"), + ] { + let metadata = HashMap::from([(key.to_string(), value.to_string())]); + assert!( + !replication_source_etag_is_comparable(&metadata), + "{key}={value} must not expose its stored ETag to the replica audit" + ); + } + } + /// backlog#1735 A3b SSE key-case finding. Every RustFS writer persists the /// SSE intent as lowercase `x-amz-server-side-encryption` /// (`encryption_material_to_metadata`; header-derived keys come from a diff --git a/crates/replication/src/object.rs b/crates/replication/src/object.rs index 5d12f8c5a..c33348694 100644 --- a/crates/replication/src/object.rs +++ b/crates/replication/src/object.rs @@ -120,8 +120,11 @@ pub fn object_lock_put_integrity( /// stored bytes differ from what the source sent — e.g. a target that does not /// decode `aws-chunked` framing stores the frames verbatim and returns their /// ETag. Only a plain single-part MD5 ETag on both sides is decidable; a -/// multipart or opaque (encrypted) ETag, or a withheld replica ETag, returns -/// `false` because no corruption can be concluded from it. +/// multipart ETag, or a withheld replica ETag, returns `false` because no +/// corruption can be concluded from it. An encrypted source's keyed +/// fingerprint is also 32 hex characters and cannot be told apart from an MD5 +/// here, so callers must exempt encrypted sources before asking (the ecstore +/// resyncer does so through its source-encryption classifier). pub fn single_part_replica_etag_mismatch(source_etag: Option<&str>, replica_etag: Option<&str>) -> bool { let Some(source) = source_etag.map(trim_etag) else { return false; diff --git a/docs/operations/encrypted-etags.md b/docs/operations/encrypted-etags.md index 463ea42f6..5d2df8b12 100644 --- a/docs/operations/encrypted-etags.md +++ b/docs/operations/encrypted-etags.md @@ -24,6 +24,17 @@ Clients that require an ETag for these older or multipart objects must issue an authorized HeadObject request, including the SSE-C key when required. GET, HEAD, and conditional requests continue to use the object's stored ETag. +## Replication + +Bucket and site replication forward the stored ETag on `x-rustfs-source-etag` +(with its `x-minio-source-*` twin). A target that adopts it (RustFS, MinIO) +reports the fingerprint for the replica; a target that computes its own ETag +(AWS S3 and most compatible stores) reports the MD5 of the plaintext it +received, which never equals the fingerprint. The post-PUT replica ETag audit +therefore skips encrypted sources, and the checksum header on the replication +PUT remains their integrity check. See +`docs/operations/replication-outbound-transport.md`. + ## Existing data and migration The upgrade does not rewrite existing object metadata or erase fingerprints diff --git a/docs/operations/replication-outbound-transport.md b/docs/operations/replication-outbound-transport.md index 125492735..1173e039c 100644 --- a/docs/operations/replication-outbound-transport.md +++ b/docs/operations/replication-outbound-transport.md @@ -9,7 +9,7 @@ - For a single-part object, the checksum the source object was uploaded with, forwarded as its `x-amz-checksum-` header (the value the source verified on upload). A multipart replica is rebuilt through CreateMultipartUpload/UploadPart and carries no object-level checksum header. Managed-SSE objects forward none. - On a PUT that carries Object Lock parameters and no forwarded checksum: `Content-MD5` derived from the source ETag, or an SDK CRC32 checksum when the ETag is not the MD5 of the wire bytes (rustfs#7082). - The source ETag, mtime and version id on `x-rustfs-source-*` headers (with `x-minio-source-*` twins), and the Object Lock mode, retain-until date and legal hold of the source version when present. -- After the PUT, the target's ETag is compared with the source ETag when both are plain single-part MD5s; a mismatch fails the replication instead of reporting a corrupted replica as COMPLETED. +- After the PUT, the target's ETag is compared with the source ETag when both are plain single-part MD5s and the source is unencrypted; a mismatch fails the replication instead of reporting a corrupted replica as COMPLETED. A managed-SSE (SSE-S3 / SSE-KMS) or SSE-C source is not audited this way: its stored ETag is a keyed fingerprint (new writes, see `docs/operations/encrypted-etags.md`) or a legacy plaintext MD5, and the target re-encrypts what it receives, so the two ETags are never comparable. The checksum header on the replication PUT remains the integrity check for those objects. ## Target classes and their known requirements