From 9eaf5fc8e3982df9f069fc140a60cbd8cc4fadf2 Mon Sep 17 00:00:00 2001 From: cxymds Date: Fri, 24 Jul 2026 16:57:11 +0800 Subject: [PATCH] fix(s3): complete CopyObject checksum support (#5178) --- Cargo.lock | 5 +- Cargo.toml | 2 +- .../e2e_test/src/copy_object_checksum_test.rs | 411 +++++++++++++++++- crates/rio/src/checksum.rs | 6 +- docs/architecture/s3-compatibility-matrix.md | 1 + docs/testing/e2e-suite-inventory.md | 4 +- rustfs/src/app/multipart_usecase.rs | 5 +- rustfs/src/app/object_usecase.rs | 124 ++++-- 8 files changed, 495 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18625163e..4f1dbc2c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10264,7 +10264,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -10321,7 +10321,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.14.1" -source = "git+https://github.com/s3s-project/s3s.git?rev=a5471625975f5014f7b28eee7e4d801f1b32f529#a5471625975f5014f7b28eee7e4d801f1b32f529" +source = "git+https://github.com/s3s-project/s3s.git?rev=8136db4ac8253c0ccfd86f8216e00e0235747757#8136db4ac8253c0ccfd86f8216e00e0235747757" dependencies = [ "arc-swap", "arrayvec", @@ -10366,6 +10366,7 @@ dependencies = [ "transform-stream", "url", "urlencoding", + "xxhash-rust", "zeroize", ] diff --git a/Cargo.toml b/Cargo.toml index eb52973e4..4ee3c9dc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -286,7 +286,7 @@ redis = { version = "1.4.1" } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "a5471625975f5014f7b28eee7e4d801f1b32f529" } +s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "8136db4ac8253c0ccfd86f8216e00e0235747757" } serial_test = "3.5.0" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.3" diff --git a/crates/e2e_test/src/copy_object_checksum_test.rs b/crates/e2e_test/src/copy_object_checksum_test.rs index 70baf0b58..bdd6251a9 100644 --- a/crates/e2e_test/src/copy_object_checksum_test.rs +++ b/crates/e2e_test/src/copy_object_checksum_test.rs @@ -12,18 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Regression test for Issue #4996: CopyObject must return the destination object's -//! checksum in `CopyObjectResult` and persist it so a later checksum-mode HEAD/GET -//! returns the same value. Covers both the requested-algorithm case (compute fresh) -//! and the no-algorithm case (preserve the source object's existing checksum). +//! CopyObject checksum compatibility tests. Covers all supported algorithms, +//! source-checksum preservation, explicit override, and fail-closed handling of +//! unsupported algorithms before destination mutation. #[cfg(test)] mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; + use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation}; + use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; - use aws_sdk_s3::types::{BucketVersioningStatus, ChecksumAlgorithm, ChecksumMode, VersioningConfiguration}; + use aws_sdk_s3::types::{ + BucketVersioningStatus, ChecksumAlgorithm, ChecksumMode, ChecksumType, CompletedMultipartUpload, CompletedPart, + VersioningConfiguration, + }; + use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; + use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; use serial_test::serial; use sha2::{Digest, Sha256}; use tracing::info; @@ -48,6 +54,401 @@ mod tests { .expect("Failed to enable versioning"); } + fn create_s3_client_no_auto_checksum(env: &RustFSTestEnvironment) -> aws_sdk_s3::Client { + let credentials = Credentials::new(&env.access_key, &env.secret_key, None, None, "copy-checksum-e2e"); + let config = aws_sdk_s3::Config::builder() + .credentials_provider(credentials) + .region(Region::new("us-east-1")) + .endpoint_url(format!("http://{}", env.address)) + .force_path_style(true) + .behavior_version_latest() + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .http_client(SmithyHttpClientBuilder::new().build_http()) + .build(); + aws_sdk_s3::Client::from_conf(config) + } + + fn algorithms() -> [(ChecksumAlgorithm, RioChecksumType); 10] { + [ + (ChecksumAlgorithm::Crc32, RioChecksumType::CRC32), + (ChecksumAlgorithm::Crc32C, RioChecksumType::CRC32C), + (ChecksumAlgorithm::Crc64Nvme, RioChecksumType::CRC64_NVME), + (ChecksumAlgorithm::Sha1, RioChecksumType::SHA1), + (ChecksumAlgorithm::Sha256, RioChecksumType::SHA256), + (ChecksumAlgorithm::Md5, RioChecksumType::MD5), + (ChecksumAlgorithm::Sha512, RioChecksumType::SHA512), + (ChecksumAlgorithm::Xxhash3, RioChecksumType::XXHASH3), + (ChecksumAlgorithm::Xxhash64, RioChecksumType::XXHASH64), + (ChecksumAlgorithm::Xxhash128, RioChecksumType::XXHASH128), + ] + } + + fn result_checksums(result: &aws_sdk_s3::types::CopyObjectResult) -> [Option<&str>; 10] { + [ + result.checksum_crc32(), + result.checksum_crc32_c(), + result.checksum_crc64_nvme(), + result.checksum_sha1(), + result.checksum_sha256(), + result.checksum_md5(), + result.checksum_sha512(), + result.checksum_xxhash3(), + result.checksum_xxhash64(), + result.checksum_xxhash128(), + ] + } + + fn head_checksums(output: &aws_sdk_s3::operation::head_object::HeadObjectOutput) -> [Option<&str>; 10] { + [ + output.checksum_crc32(), + output.checksum_crc32_c(), + output.checksum_crc64_nvme(), + output.checksum_sha1(), + output.checksum_sha256(), + output.checksum_md5(), + output.checksum_sha512(), + output.checksum_xxhash3(), + output.checksum_xxhash64(), + output.checksum_xxhash128(), + ] + } + + #[tokio::test] + #[serial] + async fn test_copy_supports_all_checksum_algorithms() { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client_no_auto_checksum(&env); + let src_bucket = "copy-all-checksums-src"; + let dst_bucket = "copy-all-checksums-dst"; + let src_key = "objects/source.bin"; + let content = b"deterministic CopyObject payload for all ten checksum algorithms"; + + create_versioned_bucket(&client, src_bucket).await; + create_versioned_bucket(&client, dst_bucket).await; + client + .put_object() + .bucket(src_bucket) + .key(src_key) + .body(ByteStream::from_static(content)) + .send() + .await + .expect("PUT source failed"); + + for (index, (sdk_algorithm, rio_algorithm)) in algorithms().into_iter().enumerate() { + let expected = Checksum::new_from_data(rio_algorithm, content) + .expect("supported checksum must be computable") + .encoded; + let dst_key = format!("objects/destination-{index}.bin"); + let copy = client + .copy_object() + .bucket(dst_bucket) + .key(&dst_key) + .copy_source(format!("{src_bucket}/{src_key}")) + .checksum_algorithm(sdk_algorithm) + .send() + .await + .expect("CopyObject with supported checksum must succeed"); + let result = copy.copy_object_result().expect("CopyObject result"); + let checksums = result_checksums(result); + assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: response checksum"); + assert_eq!( + checksums.iter().filter(|checksum| checksum.is_some()).count(), + 1, + "{rio_algorithm}: only the requested checksum may be returned" + ); + + let head = client + .head_object() + .bucket(dst_bucket) + .key(&dst_key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("HEAD destination failed"); + let checksums = head_checksums(&head); + assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: persisted checksum"); + assert_eq!( + checksums.iter().filter(|checksum| checksum.is_some()).count(), + 1, + "{rio_algorithm}: destination must persist only the requested checksum" + ); + + let body = client + .get_object() + .bucket(dst_bucket) + .key(&dst_key) + .send() + .await + .expect("GET destination failed") + .body + .collect() + .await + .expect("collect destination body") + .into_bytes(); + assert_eq!(body.as_ref(), content, "{rio_algorithm}: full copied body"); + } + + env.stop_server(); + } + + #[tokio::test] + #[serial] + async fn test_copy_without_algorithm_preserves_every_supported_source_checksum() { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client_no_auto_checksum(&env); + let src_bucket = "copy-preserve-all-src"; + let dst_bucket = "copy-preserve-all-dst"; + let content = b"source checksum preservation payload for all ten algorithms"; + + create_versioned_bucket(&client, src_bucket).await; + create_versioned_bucket(&client, dst_bucket).await; + + for (index, (_sdk_algorithm, rio_algorithm)) in algorithms().into_iter().enumerate() { + let expected = Checksum::new_from_data(rio_algorithm, content) + .expect("supported checksum must be computable") + .encoded; + let checksum_header = rio_algorithm.key().expect("supported checksum header"); + let request_checksum = expected.clone(); + let src_key = format!("objects/source-{index}.bin"); + let dst_key = format!("objects/destination-{index}.bin"); + client + .put_object() + .bucket(src_bucket) + .key(&src_key) + .body(ByteStream::from_static(content)) + .customize() + .mutate_request(move |request| { + request.headers_mut().insert(checksum_header, request_checksum.clone()); + }) + .send() + .await + .expect("PUT checksummed source failed"); + + let copy = client + .copy_object() + .bucket(dst_bucket) + .key(&dst_key) + .copy_source(format!("{src_bucket}/{src_key}")) + .send() + .await + .expect("CopyObject without algorithm must succeed"); + let result = copy.copy_object_result().expect("CopyObject result"); + let checksums = result_checksums(result); + assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: preserved response checksum"); + assert_eq!(checksums.iter().filter(|checksum| checksum.is_some()).count(), 1); + + let head = client + .head_object() + .bucket(dst_bucket) + .key(&dst_key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("HEAD destination failed"); + let checksums = head_checksums(&head); + assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: preserved stored checksum"); + assert_eq!(checksums.iter().filter(|checksum| checksum.is_some()).count(), 1); + } + + env.stop_server(); + } + + #[tokio::test] + #[serial] + async fn test_copy_without_algorithm_preserves_composite_checksum_type() { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client_no_auto_checksum(&env); + let bucket = "copy-preserve-composite"; + let source_key = "objects/multipart-source.bin"; + let destination_key = "objects/copied-multipart.bin"; + let content = b"multipart source checksum must remain composite"; + + create_versioned_bucket(&client, bucket).await; + let created = client + .create_multipart_upload() + .bucket(bucket) + .key(source_key) + .checksum_algorithm(ChecksumAlgorithm::Sha256) + .send() + .await + .expect("CreateMultipartUpload failed"); + let upload_id = created.upload_id().expect("multipart upload ID"); + let checksum = Checksum::new_from_data(RioChecksumType::SHA256, content) + .expect("SHA256 checksum") + .encoded; + let uploaded = client + .upload_part() + .bucket(bucket) + .key(source_key) + .upload_id(upload_id) + .part_number(1) + .checksum_sha256(&checksum) + .body(ByteStream::from_static(content)) + .send() + .await + .expect("UploadPart failed"); + let completed_part = CompletedPart::builder() + .part_number(1) + .e_tag(uploaded.e_tag().expect("part ETag")) + .checksum_sha256(uploaded.checksum_sha256().expect("part checksum")) + .build(); + client + .complete_multipart_upload() + .bucket(bucket) + .key(source_key) + .upload_id(upload_id) + .multipart_upload(CompletedMultipartUpload::builder().parts(completed_part).build()) + .send() + .await + .expect("CompleteMultipartUpload failed"); + + let source_head = client + .head_object() + .bucket(bucket) + .key(source_key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("HEAD multipart source failed"); + let source_checksum = source_head.checksum_sha256().expect("multipart source checksum"); + assert_eq!(source_head.checksum_type(), Some(&ChecksumType::Composite)); + + let copied = client + .copy_object() + .bucket(bucket) + .key(destination_key) + .copy_source(format!("{bucket}/{source_key}")) + .send() + .await + .expect("CopyObject without algorithm failed"); + let result = copied.copy_object_result().expect("CopyObject result"); + assert_eq!(result.checksum_sha256(), Some(source_checksum)); + assert_eq!(result.checksum_type(), Some(&ChecksumType::Composite)); + + let destination_head = client + .head_object() + .bucket(bucket) + .key(destination_key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("HEAD copied multipart object failed"); + assert_eq!(destination_head.checksum_sha256(), Some(source_checksum)); + assert_eq!(destination_head.checksum_type(), Some(&ChecksumType::Composite)); + + env.stop_server(); + } + + #[tokio::test] + #[serial] + async fn test_copy_rejects_unknown_algorithm_without_destination_mutation() { + init_logging(); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = env.create_s3_client(); + let bucket = "copy-reject-unknown-checksum"; + let src_key = "objects/source.bin"; + let dst_key = "objects/destination.bin"; + let source = b"source must never replace destination"; + let destination = b"pre-existing destination must remain byte-for-byte unchanged"; + let expected = Checksum::new_from_data(RioChecksumType::SHA256, destination) + .expect("SHA256 checksum") + .encoded; + + create_versioned_bucket(&client, bucket).await; + client + .put_object() + .bucket(bucket) + .key(src_key) + .body(ByteStream::from_static(source)) + .send() + .await + .expect("PUT source failed"); + let original = client + .put_object() + .bucket(bucket) + .key(dst_key) + .metadata("state", "original") + .checksum_algorithm(ChecksumAlgorithm::Sha256) + .body(ByteStream::from_static(destination)) + .send() + .await + .expect("PUT destination failed"); + let original_version = original.version_id().expect("versioned PUT must return a version id"); + + let missing_source_error = client + .copy_object() + .bucket(bucket) + .key(dst_key) + .copy_source(format!("{bucket}/objects/missing-source.bin")) + .checksum_algorithm(ChecksumAlgorithm::from("BLAKE3")) + .send() + .await + .expect_err("checksum validation must precede source lookup"); + assert_eq!( + missing_source_error.as_service_error().and_then(|value| value.code()), + Some("InvalidArgument") + ); + assert_eq!(missing_source_error.raw_response().map(|response| response.status().as_u16()), Some(400)); + + let error = client + .copy_object() + .bucket(bucket) + .key(dst_key) + .copy_source(format!("{bucket}/{src_key}")) + .checksum_algorithm(ChecksumAlgorithm::from("BLAKE3")) + .send() + .await + .expect_err("unsupported checksum algorithm must fail"); + assert_eq!(error.as_service_error().and_then(|value| value.code()), Some("InvalidArgument")); + assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(400)); + + let head = client + .head_object() + .bucket(bucket) + .key(dst_key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("HEAD unchanged destination"); + assert_eq!(head.version_id(), Some(original_version)); + assert_eq!( + head.metadata().and_then(|metadata| metadata.get("state").map(String::as_str)), + Some("original") + ); + assert_eq!(head.checksum_sha256(), Some(expected.as_str())); + + let body = client + .get_object() + .bucket(bucket) + .key(dst_key) + .send() + .await + .expect("GET unchanged destination") + .body + .collect() + .await + .expect("collect unchanged destination") + .into_bytes(); + assert_eq!(body.as_ref(), destination); + + env.stop_server(); + } + /// Requested algorithm: a CopyObject asking for SHA256 must compute it over the copied /// bytes, return it in `CopyObjectResult.ChecksumSHA256`, and persist it so a checksum-mode /// HEAD on the destination returns the identical value. diff --git a/crates/rio/src/checksum.rs b/crates/rio/src/checksum.rs index cecefd7ae..dee358b0b 100644 --- a/crates/rio/src/checksum.rs +++ b/crates/rio/src/checksum.rs @@ -180,10 +180,8 @@ impl ChecksumType { (self.0 & Self::FULL_OBJECT.0) == Self::FULL_OBJECT.0 || self.is(Self::CRC64_NVME) } - /// True for the five algorithms s3s exposes as typed `*Output` fields - /// (CRC32/CRC32C/SHA1/SHA256/CRC64NVME). The AWS 2026-04 additional algorithms - /// (XXHash3/64/128, SHA-512, MD5) have no typed field and are carried as raw - /// response headers instead. Single source of truth for that split. + /// True for the five legacy algorithms currently handled by RustFS's named + /// response fields. Additional algorithms use the extended response path. pub fn is_s3s_typed(self) -> bool { matches!(self.base(), Self::CRC32 | Self::CRC32C | Self::SHA1 | Self::SHA256 | Self::CRC64_NVME) } diff --git a/docs/architecture/s3-compatibility-matrix.md b/docs/architecture/s3-compatibility-matrix.md index 786e00d00..588221188 100644 --- a/docs/architecture/s3-compatibility-matrix.md +++ b/docs/architecture/s3-compatibility-matrix.md @@ -38,6 +38,7 @@ The implemented test list currently covers the common object-storage surface: |---|---|---| | Bucket create/delete/list/head | Supported | `implemented_tests.txt` | | Object put/get/delete/copy/head | Supported | `implemented_tests.txt` | +| CopyObject checksums (CRC32, CRC32C, CRC64NVME, SHA1, SHA256, MD5, SHA512, XXHASH3, XXHASH64, XXHASH128), including source preservation and explicit override | Supported in the first RustFS release containing this change | `crates/e2e_test/src/copy_object_checksum_test.rs` | | ListObjects/ListObjectsV2 prefix, delimiter, marker, max-keys | Supported | `implemented_tests.txt` | | Multipart upload create/upload/complete/abort and selected multipart copy/checksum/object-attribute behavior | Supported | `implemented_tests.txt` | | Bucket and object tagging | Supported | `implemented_tests.txt` | diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index c84e34a02..97a74f5ff 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -34,7 +34,7 @@ | connection_cap_test | 2 | | | console_smoke_test | 1 | ✅ | | content_encoding_test | 3 | ✅ | -| copy_object_checksum_test | 3 | | +| copy_object_checksum_test | 7 | | | copy_object_metadata_test | 4 | ✅ | | copy_object_tagging_test | 2 | ✅ | | copy_object_version_restore_test | 2 | | @@ -86,4 +86,4 @@ `notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above. -**Total listed: 479 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24. +**Total listed: 483 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24. diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 67304deed..4049d8e0b 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -554,11 +554,11 @@ impl DefaultMultipartUsecase { // checksum: stored (decrypted) values take precedence over the request input; // additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed // CompleteMultipartUploadOutput field, are echoed as raw response headers (#1261). - let (checksums, _is_multipart) = obj_info + let (checksums, is_multipart) = obj_info .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) .map_err(ApiError::from)?; - let classified = crate::app::object_usecase::classify_response_checksums(checksums); + let classified = crate::app::object_usecase::classify_response_checksums(checksums, is_multipart); let checksum_crc32 = classified.crc32.or(input.checksum_crc32); let checksum_crc32c = classified.crc32c.or(input.checksum_crc32c); let checksum_sha1 = classified.sha1.or(input.checksum_sha1); @@ -1352,6 +1352,7 @@ impl DefaultMultipartUsecase { checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME), e_tag: part_info.etag.map(|etag| to_s3s_etag(&etag)), last_modified: part_info.last_mod.map(Timestamp::from), + ..Default::default() }; let output = UploadPartCopyOutput { diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 45cb1ab0f..e96200dc7 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -2368,11 +2368,8 @@ fn apply_trailing_checksums( } /// Checksums resolved from stored (decrypted) metadata for a response. The five -/// s3s-typed algorithms fill named fields; the AWS 2026-04 additional algorithms -/// (XXHash3/64/128, SHA-512, MD5), which s3s has no typed `*Output` field for, land -/// in `extra` and are emitted as raw response headers via -/// [`inject_additional_checksum_headers`]. Built by [`classify_response_checksums`] -/// so the typed/extra split lives in exactly one place. +/// legacy algorithms fill named fields; the additional algorithms land in `extra` +/// for raw-header response paths and DTOs that expose their newer typed fields. #[derive(Default)] pub(crate) struct ResponseChecksums { pub(crate) crc32: Option, @@ -2384,11 +2381,11 @@ pub(crate) struct ResponseChecksums { pub(crate) extra: Vec<(&'static str, String)>, } -/// Split decrypted checksum (key, value) pairs into the five s3s-typed fields and the -/// additional-algorithm `extra` headers. Single source of truth for every response +/// Split decrypted checksum pairs into the five legacy fields and the additional +/// algorithm values. Single source of truth for every response /// path (GetObject / HeadObject / GetObjectAttributes / CompleteMultipartUpload), /// replacing what used to be five copies of this match loop. -pub(crate) fn classify_response_checksums(pairs: I) -> ResponseChecksums +pub(crate) fn classify_response_checksums(pairs: I, is_multipart: bool) -> ResponseChecksums where I: IntoIterator, { @@ -2412,6 +2409,9 @@ where } } } + if is_multipart && c.checksum_type.is_none() { + c.checksum_type = Some(ChecksumType::from("COMPOSITE".to_string())); + } c } @@ -4109,13 +4109,12 @@ impl DefaultObjectUsecase { && checksum_mode.to_str().unwrap_or_default() == "ENABLED" && rs.is_none() { - let (decrypted_checksums, _is_multipart) = - info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| { - error!(error = %e, "GetObject checksum decryption failed"); - ApiError::from(e) - })?; + let (decrypted_checksums, is_multipart) = info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| { + error!(error = %e, "GetObject checksum decryption failed"); + ApiError::from(e) + })?; - return Ok(classify_response_checksums(decrypted_checksums)); + return Ok(classify_response_checksums(decrypted_checksums, is_multipart)); } Ok(ResponseChecksums::default()) @@ -5626,7 +5625,7 @@ impl DefaultObjectUsecase { }; let checksum = if requested(ObjectAttributes::CHECKSUM) { - let (checksums, _is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?; + let (checksums, is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?; // GetObjectAttributes returns checksums in the XML body, and s3s's Checksum // type has no field for the additional algorithms, so `extra` cannot be // surfaced here (unlike the header-based GET/HEAD paths) — an s3s limitation @@ -5639,7 +5638,7 @@ impl DefaultObjectUsecase { crc64nvme: checksum_crc64nvme, checksum_type, .. - } = classify_response_checksums(checksums); + } = classify_response_checksums(checksums, is_multipart); Some(Checksum { checksum_crc32, @@ -5648,6 +5647,7 @@ impl DefaultObjectUsecase { checksum_sha256, checksum_crc64nvme, checksum_type, + ..Default::default() }) } else { None @@ -5674,7 +5674,7 @@ impl DefaultObjectUsecase { let is_truncated = end < info.parts.len(); for part in &info.parts[start_at..end] { - let (checksums, _is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?; + let (checksums, is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?; // Additional algorithms cannot be surfaced in the ObjectPart XML body // (s3s has no field); same limitation as the object-level attributes above. let ResponseChecksums { @@ -5684,7 +5684,7 @@ impl DefaultObjectUsecase { sha256: checksum_sha256, crc64nvme: checksum_crc64nvme, .. - } = classify_response_checksums(checksums); + } = classify_response_checksums(checksums, is_multipart); let part_size = if part.actual_size > 0 { part.actual_size @@ -5702,6 +5702,7 @@ impl DefaultObjectUsecase { checksum_crc64nvme, part_number: i32::try_from(part.number).ok(), size: Some(part_size), + ..Default::default() }); } @@ -5793,6 +5794,12 @@ impl DefaultObjectUsecase { checksum_algorithm, .. } = req.input.clone(); + let requested_checksum_type = checksum_algorithm + .as_ref() + .map(|algorithm| rustfs_rio::ChecksumType::from_string(algorithm.as_str())); + if requested_checksum_type.is_some_and(|checksum_type| !checksum_type.is_set()) { + return Err(s3_error!(InvalidArgument, "Unsupported checksum algorithm")); + } let (src_bucket, src_key, version_id) = match copy_source { CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)), @@ -5996,12 +6003,9 @@ impl DefaultObjectUsecase { // rather than re-hashing every byte. let src_checksum = src_info.checksum.as_ref().and_then(|bytes| { let (pairs, _) = rustfs_rio::read_checksums(bytes.as_ref(), 0); - pairs.into_iter().find_map(|(k, v)| { - rustfs_rio::ChecksumType::from_string(&k) - .is_s3s_typed() - .then(|| rustfs_rio::Checksum::new_from_string(&k, &v)) - .flatten() - }) + pairs + .into_iter() + .find_map(|(k, v)| rustfs_rio::Checksum::new_from_string(&k, &v)) }); // Validate copy source conditions @@ -6125,12 +6129,9 @@ impl DefaultObjectUsecase { // none is requested, carry the source object's stored checksum over unchanged — the copy // does not alter the plaintext, so re-hashing would be wasted work and would flatten a // multipart composite value. - match checksum_algorithm.as_ref() { - Some(algo) => { - let ct = rustfs_rio::ChecksumType::from_string(algo.as_str()); - if ct.is_set() { - reader.add_calculated_checksum(ct).map_err(ApiError::from)?; - } + match requested_checksum_type { + Some(checksum_type) => { + reader.add_calculated_checksum(checksum_type).map_err(ApiError::from)?; } None => { if let Some(cs) = src_checksum { @@ -6221,11 +6222,26 @@ impl DefaultObjectUsecase { // / HeadObject do so the value is identical to a later checksum-mode HEAD/GET (#4996). let response_checksums = oi .decrypt_checksums(0, &req.headers) - .map(|(pairs, _)| classify_response_checksums(pairs)) + .map(|(pairs, is_multipart)| classify_response_checksums(pairs, is_multipart)) .unwrap_or_default(); // warn!("copy_object oi {:?}", &oi); let object_info = oi.clone(); + let mut checksum_md5 = None; + let mut checksum_sha512 = None; + let mut checksum_xxhash3 = None; + let mut checksum_xxhash64 = None; + let mut checksum_xxhash128 = None; + for (name, value) in response_checksums.extra { + match name { + "x-amz-checksum-md5" => checksum_md5 = Some(value), + "x-amz-checksum-sha512" => checksum_sha512 = Some(value), + "x-amz-checksum-xxhash3" => checksum_xxhash3 = Some(value), + "x-amz-checksum-xxhash64" => checksum_xxhash64 = Some(value), + "x-amz-checksum-xxhash128" => checksum_xxhash128 = Some(value), + _ => {} + } + } let copy_object_result = CopyObjectResult { e_tag: oi.etag.map(|etag| to_s3s_etag(&etag)), last_modified: oi.mod_time.map(Timestamp::from), @@ -6234,6 +6250,11 @@ impl DefaultObjectUsecase { checksum_sha1: response_checksums.sha1, checksum_sha256: response_checksums.sha256, checksum_crc64nvme: response_checksums.crc64nvme, + checksum_md5, + checksum_sha512, + checksum_xxhash3, + checksum_xxhash64, + checksum_xxhash128, checksum_type: response_checksums.checksum_type, }; @@ -7122,10 +7143,10 @@ impl DefaultObjectUsecase { && checksum_mode.to_str().unwrap_or_default() == "ENABLED" && rs.is_none() { - let (checksums, _is_multipart) = info + let (checksums, is_multipart) = info .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) .map_err(ApiError::from)?; - classify_response_checksums(checksums) + classify_response_checksums(checksums, is_multipart) } else { ResponseChecksums::default() }; @@ -8109,24 +8130,30 @@ mod tests { #[test] fn classify_response_checksums_splits_typed_and_extra() { // Typed algorithms fill named fields; nothing spills into extra. - let c = classify_response_checksums(vec![ - ("CRC32".to_string(), "AAAAAA==".to_string()), - ("SHA256".to_string(), "c2hhMjU2".to_string()), - ("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()), - ]); + let c = classify_response_checksums( + vec![ + ("CRC32".to_string(), "AAAAAA==".to_string()), + ("SHA256".to_string(), "c2hhMjU2".to_string()), + ("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()), + ], + false, + ); assert_eq!(c.crc32.as_deref(), Some("AAAAAA==")); assert_eq!(c.sha256.as_deref(), Some("c2hhMjU2")); assert_eq!(c.crc64nvme.as_deref(), Some("Zm9vYmFyCg==")); assert!(c.extra.is_empty(), "typed algorithms must not land in extra"); // Additional algorithms land in extra keyed by their response-header name. - let c = classify_response_checksums(vec![ - ("XXHASH3".to_string(), "eHhoMw==".to_string()), - ("XXHASH64".to_string(), "eHhoNjQ=".to_string()), - ("XXHASH128".to_string(), "eHhoMTI4".to_string()), - ("SHA512".to_string(), "c2hhNTEy".to_string()), - ("MD5".to_string(), "bWQ1".to_string()), - ]); + let c = classify_response_checksums( + vec![ + ("XXHASH3".to_string(), "eHhoMw==".to_string()), + ("XXHASH64".to_string(), "eHhoNjQ=".to_string()), + ("XXHASH128".to_string(), "eHhoMTI4".to_string()), + ("SHA512".to_string(), "c2hhNTEy".to_string()), + ("MD5".to_string(), "bWQ1".to_string()), + ], + false, + ); assert!(c.crc32.is_none() && c.sha256.is_none() && c.crc64nvme.is_none()); let names: Vec<&str> = c.extra.iter().map(|(n, _)| *n).collect(); for expected in [ @@ -8141,12 +8168,15 @@ mod tests { assert_eq!(c.extra.len(), 5); // The checksum-type marker is captured as the type, not mistaken for an algorithm. - let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())]); + let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())], false); assert!(c.checksum_type.is_some()); assert!(c.extra.is_empty() && c.crc32.is_none()); + let c = classify_response_checksums(vec![("CRC32".to_string(), "AAAAAA==-2".to_string())], true); + assert_eq!(c.checksum_type.as_ref().map(ChecksumType::as_str), Some("COMPOSITE")); + // Empty input yields an all-default result. - let c = classify_response_checksums(Vec::<(String, String)>::new()); + let c = classify_response_checksums(Vec::<(String, String)>::new(), false); assert!(c.crc32.is_none() && c.extra.is_empty() && c.checksum_type.is_none()); }