From 53cabe927453e0f2d05053e6281c410c85b359f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Thu, 3 Sep 2026 20:26:44 +0800 Subject: [PATCH] fix(replication): send an integrity header on Object Lock PUTs (#7097) * fix(replication): send an integrity header on Object Lock replication PUTs AWS S3, MinIO and most compatible targets reject a PutObject that carries x-amz-object-lock-* headers unless it also carries Content-MD5 or an x-amz-checksum-* header. Since rustfs#6895 the replication client sends plain signed payloads with no SDK checksum, so every replicated object with a retention period or legal hold failed against such targets. TargetClient::put_object now decides per request through the pure rustfs_replication::object_lock_put_integrity: a plaintext single-part object whose source ETag is its MD5 gets Content-MD5 derived from the ETag (no body pass, framing unchanged); a multipart-layout ETag, managed SSE or SSE-C passthrough falls back to an SDK CRC32; a forwarded source checksum or an unlocked PUT is left alone. The outbound target matrix flips its two KnownFailing(rustfs#7082) cells to Completed and every Completed cell now asserts that a locked PutObject carried an integrity header. Fixes rustfs#7082. * test(e2e): keep the matrix expectation table clippy-clean under -D warnings The CI lint runs cargo clippy --all-targets -- -D warnings. With every cell green the single-arm match tripped match_single_binding and the unused KnownFailing variant tripped dead_code, and the target-client tests tripped field_reassign_with_default. Drive the expectation table from a KNOWN_FAILING_CELLS constant (so the variant stays live and adding a red cell is a one-line entry), build the test options as struct literals, and refresh the e2e-repl-nightly selection digest for the renamed table test. --- .config/e2e-repl-nightly-selection.txt | 2 +- .../src/replication_target_matrix_test.rs | 70 ++++--- .../ecstore/src/bucket/bucket_target_sys.rs | 193 +++++++++++++++++- crates/ecstore/src/bucket/replication/mod.rs | 1 + .../replication_target_boundary.rs | 1 + crates/replication/src/lib.rs | 6 +- crates/replication/src/object.rs | 79 +++++++ .../replication-outbound-transport.md | 3 +- ...replication-checksum-default-regression.md | 4 +- 9 files changed, 321 insertions(+), 38 deletions(-) diff --git a/.config/e2e-repl-nightly-selection.txt b/.config/e2e-repl-nightly-selection.txt index b95fe0eaf..67c105f50 100644 --- a/.config/e2e-repl-nightly-selection.txt +++ b/.config/e2e-repl-nightly-selection.txt @@ -1 +1 @@ -sha256=6b4c126c0b590e5768bcb9928e9bce025ddc2efeddcc8b6cda5ee4855730c469 +sha256=95c8adc016bbc0df9fb2afa24a108bcdf6567ec4d0518725a6cae301593ab556 diff --git a/crates/e2e_test/src/replication_target_matrix_test.rs b/crates/e2e_test/src/replication_target_matrix_test.rs index 521e0c323..00b4471ee 100644 --- a/crates/e2e_test/src/replication_target_matrix_test.rs +++ b/crates/e2e_test/src/replication_target_matrix_test.rs @@ -202,19 +202,21 @@ enum Expectation { KnownFailing(&'static str), } -/// The single source of truth for what every cell must do today. A fix that -/// turns a `KnownFailing` cell green must flip it here in the same PR; the -/// test refuses an unexpected pass so the table cannot go stale silently. +/// The cells that are red today, each pinned to the open issue that owns it. +/// This is the single source of truth: a fix that turns a cell green must +/// remove its entry in the same PR, and [`check_known_failing_cell`] refuses +/// an unexpected pass so the table cannot go stale silently. rustfs#7082 +/// (Retention and LegalHold against the checksum-requiring target) lived +/// here until the replication PUT started carrying a Content-MD5 derived +/// from the source ETag. +const KNOWN_FAILING_CELLS: &[(TargetMode, ObjectShape, &str)] = &[]; + fn expectation(mode: TargetMode, shape: ObjectShape) -> Expectation { - match (mode, shape) { - // rustfs#7082: the replication PUT carries the lock headers but no - // Content-MD5 / x-amz-checksum-* since rustfs#6895 switched the SDK - // to plain payloads; AWS-compatible Object Lock targets reject it. - (TargetMode::RequireChecksumWithObjectLock, ObjectShape::Retention | ObjectShape::LegalHold) => { - Expectation::KnownFailing("rustfs#7082") - } - _ => Expectation::Completed, - } + KNOWN_FAILING_CELLS + .iter() + .find(|(known_mode, known_shape, _)| *known_mode == mode && *known_shape == shape) + .map(|(_, _, issue)| Expectation::KnownFailing(issue)) + .unwrap_or(Expectation::Completed) } #[tokio::test] @@ -237,28 +239,27 @@ async fn matrix_mint_own_version_ids_target() -> TestResult { run_row(TargetMode::MintOwnVersionIds).await } -/// The expectation table must name every mode and shape exactly once, so a -/// new row or column cannot be added without deciding what it does. +/// Every known-red entry must name a real cell and an issue, and the lookup +/// must round-trip, so a stale or mistyped entry cannot silently pin nothing. #[test] -fn expectation_table_covers_every_cell() { - for mode in TargetMode::ALL { - for shape in ObjectShape::ALL { - let _ = expectation(mode, shape); - } +fn known_failing_table_names_real_cells() { + for (mode, shape, issue) in KNOWN_FAILING_CELLS { + assert!( + TargetMode::ALL.contains(mode) && ObjectShape::ALL.contains(shape), + "{mode:?}/{shape:?} is not a matrix cell" + ); + assert!( + issue.starts_with("rustfs#") || issue.starts_with("rustfs/backlog#"), + "{issue} must name an open issue" + ); + assert_eq!(expectation(*mode, *shape), Expectation::KnownFailing(issue)); } - let known_failing: Vec<_> = TargetMode::ALL + let red_cells = TargetMode::ALL .iter() .flat_map(|mode| ObjectShape::ALL.iter().map(move |shape| (*mode, *shape))) .filter(|(mode, shape)| matches!(expectation(*mode, *shape), Expectation::KnownFailing(_))) - .collect(); - assert_eq!( - known_failing, - vec![ - (TargetMode::RequireChecksumWithObjectLock, ObjectShape::Retention), - (TargetMode::RequireChecksumWithObjectLock, ObjectShape::LegalHold), - ], - "every known-red cell is listed here on purpose; update this list together with the expectation table" - ); + .count(); + assert_eq!(red_cells, KNOWN_FAILING_CELLS.len()); } async fn run_row(mode: TargetMode) -> TestResult { @@ -389,6 +390,17 @@ async fn check_completed_cell( ) .into()); } + // rustfs#7082 contract: every PutObject that carries Object Lock + // parameters also carries Content-MD5 or an x-amz-checksum-* header, + // whatever the target's own policy is. + if let Some(bare) = uploads.iter().find(|record| { + record.operation == FakeTargetOperation::PutObject + && record.transport.object_lock_params + && record.transport.content_md5.is_none() + && record.transport.checksum_headers.is_empty() + }) { + return Err(format!("a locked PutObject went out without any integrity header (rustfs#7082): {bare:?}").into()); + } Ok(()) } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 9ed77d446..6932784f9 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -16,6 +16,7 @@ use crate::bucket::metadata::BucketMetadata; use crate::bucket::metadata_sys::get_bucket_targets_config; use crate::bucket::metadata_sys::get_replication_config; use crate::bucket::remote_s3_client::{PathStyle, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client}; +use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity}; use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge}; use crate::bucket::target::ARN; use crate::bucket::target::BucketTargetType; @@ -36,7 +37,7 @@ use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::BucketVersioningStatus; use aws_sdk_s3::types::Tagging as SdkTagging; use aws_sdk_s3::types::{ - ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode, + ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ServerSideEncryption, }; use aws_sdk_s3::{Client as S3Client, operation::head_object::HeadObjectOutput}; @@ -1380,6 +1381,25 @@ impl Default for AdvancedPutOptions { } } +/// Decide how a replication PUT satisfies the Object Lock integrity rule from +/// the headers it is about to send (the pure decision lives in the replication crate, re-exported through the replication boundary). +fn object_lock_put_integrity_for(headers: &HeaderMap, opts: &PutObjectOptions) -> ObjectLockIntegrity { + let lock_params = opts.mode.is_some() || opts.retain_until_date.unix_timestamp() != 0 || opts.legalhold.is_some(); + let has_integrity_header = headers.keys().any(|name| { + let name = name.as_str(); + name.starts_with("x-amz-checksum-") || name == "x-amz-sdk-checksum-algorithm" || name == "content-md5" + }); + let plaintext_end_to_end = !headers.contains_key("x-amz-server-side-encryption") + && !headers.contains_key("x-amz-server-side-encryption-customer-algorithm") + && !rustfs_utils::http::has_ssec_transport_headers(headers); + object_lock_put_integrity( + lock_params, + has_integrity_header, + plaintext_end_to_end, + Some(opts.internal.source_etag.as_str()), + ) +} + /// The subset of the target's PutObject response replication audits. #[derive(Debug, Clone)] pub struct RemotePutObjectResponse { @@ -1949,7 +1969,7 @@ impl TargetClient { ) -> Result { let mut headers = opts.header(); - let builder = self.client.put_object(); + let mut builder = self.client.put_object(); let version_id = opts.internal.source_version_id.clone(); if !version_id.is_empty() { @@ -1957,6 +1977,27 @@ impl TargetClient { } let api_version_id = resolve_put_api_version_id(&version_id).map(ToOwned::to_owned); + // A PUT carrying Object Lock parameters must also carry Content-MD5 or + // an x-amz-checksum-* header on AWS-compatible targets (rustfs#7082). + // The plain-payload default (rustfs#6853) sends neither, so supply one + // here: the source ETag when it is the MD5 of the wire bytes, else an + // SDK-computed checksum. + match object_lock_put_integrity_for(&headers, opts) { + ObjectLockIntegrity::NotRequired => {} + ObjectLockIntegrity::ContentMd5Hex(md5_hex) => { + let digest = hex_simd::decode_to_vec(md5_hex.as_bytes()) + .map_err(|err| S3ClientError::new(format!("source etag is not hex: {err}")))?; + let encoded = base64_simd::STANDARD.encode_to_string(digest); + headers.insert( + http::header::HeaderName::from_static("content-md5"), + HeaderValue::from_str(&encoded).map_err(|err| S3ClientError::new(format!("invalid Content-MD5: {err}")))?, + ); + } + ObjectLockIntegrity::SdkChecksum => { + builder = builder.checksum_algorithm(ChecksumAlgorithm::Crc32); + } + } + match builder .bucket(bucket) .key(object) @@ -2324,6 +2365,7 @@ mod tests { use rcgen::generate_simple_self_signed; use rustfs_config::{RUSTFS_CA_CERT, RUSTFS_TLS_CERT}; use rustfs_utils::egress::OutboundUrlError; + use rustfs_utils::http::AMZ_SERVER_SIDE_ENCRYPTION; // The startup panic fix for hosts without a CA bundle (issue #6734) rests // on two properties: the health-check client constructor never panics, and @@ -2455,6 +2497,153 @@ mod tests { assert_eq!(header("content-length"), Some("4")); } + fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) + } + + fn locked_put_options(source_etag: &str) -> PutObjectOptions { + PutObjectOptions { + mode: Some(ObjectLockRetentionMode::Governance), + retain_until_date: OffsetDateTime::from_unix_timestamp(4_102_444_800).expect("valid timestamp"), + internal: AdvancedPutOptions { + source_etag: source_etag.to_string(), + ..Default::default() + }, + ..Default::default() + } + } + + /// rustfs#7082: a locked PUT of a plaintext single-part object carries a + /// Content-MD5 derived from the source ETag, still as a plain payload. + #[tokio::test] + async fn locked_put_object_carries_content_md5_from_the_source_etag() { + let (client, recorded) = header_recording_target_client(Vec::new()); + client + .put_object( + "target-bucket", + "object", + 4, + streaming_test_body(b"data"), + &locked_put_options("\"8d777f385d3dfec8815d20f7496026dc\""), + ) + .await + .expect("recorded put_object should succeed"); + + let recorded = recorded.lock().expect("recorded header lock should not be poisoned"); + let headers = &recorded[0]; + // base64 of the MD5 bytes of "data". + assert_eq!(recorded_header(headers, "content-md5"), Some("jXd/OF09/siBXSD3SWAm3A==")); + assert_eq!(recorded_header(headers, "x-amz-object-lock-mode"), Some("GOVERNANCE")); + assert_eq!( + recorded_header(headers, "x-amz-trailer"), + None, + "Content-MD5 must not change the payload framing" + ); + assert!( + recorded_header(headers, "content-encoding").is_none_or(|v| !v.contains("aws-chunked")), + "locked uploads stay plain signed payloads" + ); + assert_eq!(recorded_header(headers, "content-length"), Some("4")); + } + + /// A legal hold is an Object Lock parameter too. + #[tokio::test] + async fn legal_hold_put_object_carries_content_md5() { + let (client, recorded) = header_recording_target_client(Vec::new()); + let opts = PutObjectOptions { + legalhold: Some(ObjectLockLegalHoldStatus::On), + internal: AdvancedPutOptions { + source_etag: "8d777f385d3dfec8815d20f7496026dc".to_string(), + ..Default::default() + }, + ..Default::default() + }; + client + .put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &opts) + .await + .expect("recorded put_object should succeed"); + let recorded = recorded.lock().expect("recorded header lock should not be poisoned"); + assert_eq!(recorded_header(&recorded[0], "content-md5"), Some("jXd/OF09/siBXSD3SWAm3A==")); + assert_eq!(recorded_header(&recorded[0], "x-amz-object-lock-legal-hold"), Some("ON")); + } + + /// When the source ETag is not the MD5 of the wire bytes (multipart + /// layout, or an encrypted object) the SDK computes the checksum instead; + /// with a streaming body that is a CRC32 trailer. + #[tokio::test] + async fn locked_put_object_without_a_usable_etag_uses_an_sdk_checksum() { + for (label, opts) in [ + ("multipart etag", locked_put_options("8d777f385d3dfec8815d20f7496026dc-3")), + ("managed sse", { + let mut opts = locked_put_options("8d777f385d3dfec8815d20f7496026dc"); + opts.user_metadata + .insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string()); + opts + }), + ] { + let (client, recorded) = header_recording_target_client(Vec::new()); + client + .put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &opts) + .await + .expect("recorded put_object should succeed"); + let recorded = recorded.lock().expect("recorded header lock should not be poisoned"); + let headers = &recorded[0]; + assert_eq!( + recorded_header(headers, "content-md5"), + None, + "{label}: the source etag is not the wire MD5" + ); + assert!( + recorded_header(headers, "x-amz-sdk-checksum-algorithm").is_some_and(|v| v.eq_ignore_ascii_case("CRC32")) + || recorded_header(headers, "x-amz-checksum-crc32").is_some(), + "{label}: the SDK must announce a CRC32 checksum; headers: {headers:?}" + ); + } + } + + /// A forwarded source checksum already satisfies the rule; nothing is added. + #[tokio::test] + async fn locked_put_object_keeps_a_forwarded_source_checksum() { + let (client, recorded) = header_recording_target_client(Vec::new()); + let mut opts = locked_put_options("8d777f385d3dfec8815d20f7496026dc"); + opts.user_metadata + .insert("x-amz-checksum-crc32".to_string(), "rfPzYw==".to_string()); + client + .put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &opts) + .await + .expect("recorded put_object should succeed"); + let recorded = recorded.lock().expect("recorded header lock should not be poisoned"); + let headers = &recorded[0]; + assert_eq!(recorded_header(headers, "x-amz-checksum-crc32"), Some("rfPzYw==")); + assert_eq!(recorded_header(headers, "content-md5"), None); + assert_eq!(recorded_header(headers, "x-amz-trailer"), None); + } + + /// Without Object Lock parameters the plain-payload default is untouched. + #[tokio::test] + async fn unlocked_put_object_adds_no_integrity_header() { + let (client, recorded) = header_recording_target_client(Vec::new()); + let opts = PutObjectOptions { + internal: AdvancedPutOptions { + source_etag: "8d777f385d3dfec8815d20f7496026dc".to_string(), + ..Default::default() + }, + ..Default::default() + }; + client + .put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &opts) + .await + .expect("recorded put_object should succeed"); + let recorded = recorded.lock().expect("recorded header lock should not be poisoned"); + let headers = &recorded[0]; + assert_eq!(recorded_header(headers, "content-md5"), None); + assert_eq!(recorded_header(headers, "x-amz-trailer"), None); + assert_eq!(recorded_header(headers, "x-amz-sdk-checksum-algorithm"), None); + } + #[tokio::test] async fn put_object_returns_the_etag_the_target_stored() { let (client, _) = diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index 86271d419..df68abea5 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -88,4 +88,5 @@ pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog}; pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats}; pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage}; pub use replication_target_boundary::SsecPassthroughCapability; +pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity}; pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge; diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index 784122bbb..a5cbf6b34 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -43,6 +43,7 @@ pub(crate) use crate::bucket::bucket_target_sys::{ pub(crate) use crate::bucket::target::BucketTarget; pub(crate) use crate::bucket::target::BucketTargets; pub use rustfs_replication::SsecPassthroughCapability; +pub use rustfs_replication::{ObjectLockIntegrity, object_lock_put_integrity}; pub(crate) use rustfs_replication::{ SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted, }; diff --git a/crates/replication/src/lib.rs b/crates/replication/src/lib.rs index a8b9679de..1aa20daca 100644 --- a/crates/replication/src/lib.rs +++ b/crates/replication/src/lib.rs @@ -64,9 +64,9 @@ pub use multipart::{ replication_single_put_size_error, }; pub use object::{ - ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag, - is_replication_target_offline_error, replication_action_for_target, replication_etags_match, - single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate, + ObjectLockIntegrity, ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, + content_matches_by_etag, is_replication_target_offline_error, object_lock_put_integrity, replication_action_for_target, + replication_etags_match, single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_drifted, }; pub use operation::{ diff --git a/crates/replication/src/object.rs b/crates/replication/src/object.rs index 505907df4..490821706 100644 --- a/crates/replication/src/object.rs +++ b/crates/replication/src/object.rs @@ -75,6 +75,47 @@ fn is_plain_single_part_md5(etag: &str) -> bool { etag.len() == 32 && etag.bytes().all(|b| b.is_ascii_hexdigit()) } +/// How a replication PutObject that carries Object Lock parameters satisfies +/// the target-side rule that such a request must also carry `Content-MD5` or +/// an `x-amz-checksum-*` header (AWS S3, MinIO and most compatible stores +/// enforce it; rustfs#7082). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ObjectLockIntegrity { + /// No Object Lock parameters, or an integrity header is already present. + NotRequired, + /// Send `Content-MD5` computed from this hex MD5: the source ETag is the + /// MD5 of exactly the bytes going on the wire, so no body pass and no + /// change of payload framing is needed. + ContentMd5Hex(String), + /// The source ETag is not the MD5 of the wire bytes (multipart layout, + /// or an encrypted object whose ETag does not describe the plaintext); + /// let the SDK compute a checksum instead. + SdkChecksum, +} + +/// Decide the integrity header for a locked replication PUT. +/// +/// `plaintext_end_to_end` is false when the request announces any server-side +/// encryption or carries SSE-C ciphertext passthrough headers: the source +/// ETag then does not describe the bytes on the wire and must not be turned +/// into a `Content-MD5` the target would reject with `BadDigest`. +pub fn object_lock_put_integrity( + lock_params: bool, + has_integrity_header: bool, + plaintext_end_to_end: bool, + source_etag: Option<&str>, +) -> ObjectLockIntegrity { + if !lock_params || has_integrity_header { + return ObjectLockIntegrity::NotRequired; + } + match source_etag.map(trim_etag) { + Some(etag) if plaintext_end_to_end && is_plain_single_part_md5(&etag) => { + ObjectLockIntegrity::ContentMd5Hex(etag.to_ascii_lowercase()) + } + _ => ObjectLockIntegrity::SdkChecksum, + } +} + /// Whether the ETag the target returned for a single-part replica proves the /// stored bytes differ from what the source sent — e.g. a target that does not /// decode `aws-chunked` framing stores the frames verbatim and returns their @@ -302,6 +343,8 @@ pub fn ssec_passthrough_evidence_present(sse_customer_algorithm: Option<&str>) - #[cfg(test)] mod tests { + use super::{ObjectLockIntegrity, object_lock_put_integrity}; + const SOURCE_MD5: &str = "9a0364b9e99bb480dd25e1f0284c8555"; const FRAMED_MD5: &str = "0f343b0931126a20f133d67c2b018a3b"; @@ -556,4 +599,40 @@ mod tests { ReplicationAction::Metadata ); } + + #[test] + fn locked_put_uses_the_plain_source_md5_as_content_md5() { + assert_eq!( + object_lock_put_integrity(true, false, true, Some("\"9A0364B9E99BB480DD25E1F0284C8555\"")), + ObjectLockIntegrity::ContentMd5Hex("9a0364b9e99bb480dd25e1f0284c8555".to_string()) + ); + } + + #[test] + fn locked_put_without_a_usable_etag_falls_back_to_the_sdk_checksum() { + // Multipart layout: the ETag is not the MD5 of the body. + assert_eq!( + object_lock_put_integrity(true, false, true, Some("9a0364b9e99bb480dd25e1f0284c8555-2")), + ObjectLockIntegrity::SdkChecksum + ); + // Encrypted end to end: the ETag does not describe the wire bytes. + assert_eq!( + object_lock_put_integrity(true, false, false, Some("9a0364b9e99bb480dd25e1f0284c8555")), + ObjectLockIntegrity::SdkChecksum + ); + assert_eq!(object_lock_put_integrity(true, false, true, None), ObjectLockIntegrity::SdkChecksum); + assert_eq!(object_lock_put_integrity(true, false, true, Some("")), ObjectLockIntegrity::SdkChecksum); + } + + #[test] + fn integrity_is_not_added_without_lock_params_or_when_already_present() { + assert_eq!( + object_lock_put_integrity(false, false, true, Some("9a0364b9e99bb480dd25e1f0284c8555")), + ObjectLockIntegrity::NotRequired + ); + assert_eq!( + object_lock_put_integrity(true, true, true, Some("9a0364b9e99bb480dd25e1f0284c8555")), + ObjectLockIntegrity::NotRequired + ); + } } diff --git a/docs/operations/replication-outbound-transport.md b/docs/operations/replication-outbound-transport.md index 4d0c9ec36..8ff7f285a 100644 --- a/docs/operations/replication-outbound-transport.md +++ b/docs/operations/replication-outbound-transport.md @@ -7,6 +7,7 @@ - A plain signed body with an exact `Content-Length`. The SDK does not add a streaming trailer checksum, so the body is never wrapped in `aws-chunked` framing (rustfs#6853: a target that does not decode that framing stored the frames verbatim while RustFS recorded COMPLETED). - Any object-level checksum the source object was uploaded with, forwarded as its `x-amz-checksum-*` header. +- On a PUT that carries Object Lock parameters and no forwarded checksum: `Content-MD5` derived from the source ETag, or an SDK CRC32 checksum when the ETag is not the MD5 of the wire bytes (rustfs#7082). - The source ETag, mtime and version id on `x-rustfs-source-*` headers (with `x-minio-source-*` twins), and the Object Lock mode, retain-until date and legal hold of the source version when present. - After the PUT, the target's ETag is compared with the source ETag when both are plain single-part MD5s; a mismatch fails the replication instead of reporting a corrupted replica as COMPLETED. @@ -15,7 +16,7 @@ | Target behavior | Effect on RustFS replication | Detected by | | --- | --- | --- | | Rejects or mis-stores `aws-chunked` bodies (SeaweedFS 3.97) | Handled by the plain-payload default above. | Outbound target matrix, `RejectAwsChunked` mode | -| Requires `Content-MD5` or `x-amz-checksum-*` on a PutObject with Object Lock parameters (AWS S3, MinIO, Impossible Cloud, most compatible stores) | Objects with a retention period or legal hold fail until rustfs#7082 lands. Set `RUSTFS_REPLICATION_STREAMING_CHECKSUMS=true` as a workaround when the target also decodes `aws-chunked`. | Outbound target matrix, `RequireChecksumWithObjectLock` mode | +| Requires `Content-MD5` or `x-amz-checksum-*` on a PutObject with Object Lock parameters (AWS S3, MinIO, Impossible Cloud, most compatible stores) | Satisfied: a locked single PUT carries `Content-MD5` derived from the source ETag (plaintext objects whose ETag is the MD5 of the wire bytes) or an SDK CRC32 checksum (multipart-layout ETags, managed SSE, SSE-C passthrough — this one is an `aws-chunked` trailer, so a target that also rejects that framing cannot take such objects). Releases before this fix (`1.0.0-rc.5`) need `RUSTFS_REPLICATION_STREAMING_CHECKSUMS=true` as a workaround. | Outbound target matrix, `RequireChecksumWithObjectLock` mode | | Mints its own version ids (AWS S3, Wasabi, Impossible Cloud) | Data lands; version-addressed convergence does not. See rustfs/backlog#2085 and `docs/operations/replication-check.md` (VersionFidelity). | `replication-check`, outbound target matrix, `MintOwnVersionIds` mode | | Returns an ETag that is not the content MD5 without announcing SSE | Every single-part object fails ETag verification. Set `RUSTFS_REPLICATION_REPLICA_ETAG_VERIFY=false`. | Replication status FAILED with `replica etag mismatch` | diff --git a/docs/postmortems/2026-09-03-replication-checksum-default-regression.md b/docs/postmortems/2026-09-03-replication-checksum-default-regression.md index 17e4fcd65..03096ea39 100644 --- a/docs/postmortems/2026-09-03-replication-checksum-default-regression.md +++ b/docs/postmortems/2026-09-03-replication-checksum-default-regression.md @@ -42,8 +42,8 @@ The same thing was missing at every layer: an inventory of what remote targets m | Compatibility lens gains an outbound target section. | `.agents/skills/adversarial-validation/references/compatibility.md` | | `AGENTS.md` names outbound client defaults as high risk and requires the matrix plus documented escape hatches. | `AGENTS.md`, Adversarial Validation | | The two escape hatches from rustfs#6895 are documented. | `docs/operations/replication-outbound-transport.md` | -| Product fix for rustfs#7082: derive `Content-MD5` from the source ETag on locked PUTs. | tracked in rustfs#7082 | -| `replication-check` probe PUT carries retention headers when the target bucket has Object Lock. | tracked in rustfs#7082 | +| Product fix for rustfs#7082: a locked PUT carries `Content-MD5` derived from the source ETag when that ETag is the MD5 of the wire bytes, and an SDK CRC32 checksum otherwise; the two matrix cells flipped to `Completed` in the same change. | `crates/ecstore/src/bucket/bucket_target_sys.rs` (`object_lock_put_integrity_for`), `crates/replication/src/object.rs` (`object_lock_put_integrity`) | +| `replication-check` probe PUT with retention headers: deliberately not done. A retained probe object cannot be cleaned up on targets that deny `s3:BypassGovernanceRetention`, leaving locked residue; the matrix pins the contract instead. | `crates/e2e_test/src/replication_target_matrix_test.rs` | ## SOP: changing an outbound client default