fix(s3): complete CopyObject checksum support (#5178)

This commit is contained in:
cxymds
2026-07-24 16:57:11 +08:00
committed by GitHub
parent 3132637294
commit 9eaf5fc8e3
8 changed files with 495 additions and 63 deletions
Generated
+3 -2
View File
@@ -10264,7 +10264,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -10321,7 +10321,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "s3s"
version = "0.14.1"
source = "git+https://github.com/s3s-project/s3s.git?rev=a5471625975f5014f7b28eee7e4d801f1b32f529#a5471625975f5014f7b28eee7e4d801f1b32f529"
source = "git+https://github.com/s3s-project/s3s.git?rev=8136db4ac8253c0ccfd86f8216e00e0235747757#8136db4ac8253c0ccfd86f8216e00e0235747757"
dependencies = [
"arc-swap",
"arrayvec",
@@ -10366,6 +10366,7 @@ dependencies = [
"transform-stream",
"url",
"urlencoding",
"xxhash-rust",
"zeroize",
]
+1 -1
View File
@@ -286,7 +286,7 @@ redis = { version = "1.4.1" }
rustix = { version = "1.1.4" }
rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "a5471625975f5014f7b28eee7e4d801f1b32f529" }
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "8136db4ac8253c0ccfd86f8216e00e0235747757" }
serial_test = "3.5.0"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.3"
@@ -12,18 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Regression test for Issue #4996: CopyObject must return the destination object's
//! checksum in `CopyObjectResult` and persist it so a later checksum-mode HEAD/GET
//! returns the same value. Covers both the requested-algorithm case (compute fresh)
//! and the no-algorithm case (preserve the source object's existing checksum).
//! CopyObject checksum compatibility tests. Covers all supported algorithms,
//! source-checksum preservation, explicit override, and fail-closed handling of
//! unsupported algorithms before destination mutation.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, ChecksumAlgorithm, ChecksumMode, VersioningConfiguration};
use aws_sdk_s3::types::{
BucketVersioningStatus, ChecksumAlgorithm, ChecksumMode, ChecksumType, CompletedMultipartUpload, CompletedPart,
VersioningConfiguration,
};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use serial_test::serial;
use sha2::{Digest, Sha256};
use tracing::info;
@@ -48,6 +54,401 @@ mod tests {
.expect("Failed to enable versioning");
}
fn create_s3_client_no_auto_checksum(env: &RustFSTestEnvironment) -> aws_sdk_s3::Client {
let credentials = Credentials::new(&env.access_key, &env.secret_key, None, None, "copy-checksum-e2e");
let config = aws_sdk_s3::Config::builder()
.credentials_provider(credentials)
.region(Region::new("us-east-1"))
.endpoint_url(format!("http://{}", env.address))
.force_path_style(true)
.behavior_version_latest()
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.http_client(SmithyHttpClientBuilder::new().build_http())
.build();
aws_sdk_s3::Client::from_conf(config)
}
fn algorithms() -> [(ChecksumAlgorithm, RioChecksumType); 10] {
[
(ChecksumAlgorithm::Crc32, RioChecksumType::CRC32),
(ChecksumAlgorithm::Crc32C, RioChecksumType::CRC32C),
(ChecksumAlgorithm::Crc64Nvme, RioChecksumType::CRC64_NVME),
(ChecksumAlgorithm::Sha1, RioChecksumType::SHA1),
(ChecksumAlgorithm::Sha256, RioChecksumType::SHA256),
(ChecksumAlgorithm::Md5, RioChecksumType::MD5),
(ChecksumAlgorithm::Sha512, RioChecksumType::SHA512),
(ChecksumAlgorithm::Xxhash3, RioChecksumType::XXHASH3),
(ChecksumAlgorithm::Xxhash64, RioChecksumType::XXHASH64),
(ChecksumAlgorithm::Xxhash128, RioChecksumType::XXHASH128),
]
}
fn result_checksums(result: &aws_sdk_s3::types::CopyObjectResult) -> [Option<&str>; 10] {
[
result.checksum_crc32(),
result.checksum_crc32_c(),
result.checksum_crc64_nvme(),
result.checksum_sha1(),
result.checksum_sha256(),
result.checksum_md5(),
result.checksum_sha512(),
result.checksum_xxhash3(),
result.checksum_xxhash64(),
result.checksum_xxhash128(),
]
}
fn head_checksums(output: &aws_sdk_s3::operation::head_object::HeadObjectOutput) -> [Option<&str>; 10] {
[
output.checksum_crc32(),
output.checksum_crc32_c(),
output.checksum_crc64_nvme(),
output.checksum_sha1(),
output.checksum_sha256(),
output.checksum_md5(),
output.checksum_sha512(),
output.checksum_xxhash3(),
output.checksum_xxhash64(),
output.checksum_xxhash128(),
]
}
#[tokio::test]
#[serial]
async fn test_copy_supports_all_checksum_algorithms() {
init_logging();
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client_no_auto_checksum(&env);
let src_bucket = "copy-all-checksums-src";
let dst_bucket = "copy-all-checksums-dst";
let src_key = "objects/source.bin";
let content = b"deterministic CopyObject payload for all ten checksum algorithms";
create_versioned_bucket(&client, src_bucket).await;
create_versioned_bucket(&client, dst_bucket).await;
client
.put_object()
.bucket(src_bucket)
.key(src_key)
.body(ByteStream::from_static(content))
.send()
.await
.expect("PUT source failed");
for (index, (sdk_algorithm, rio_algorithm)) in algorithms().into_iter().enumerate() {
let expected = Checksum::new_from_data(rio_algorithm, content)
.expect("supported checksum must be computable")
.encoded;
let dst_key = format!("objects/destination-{index}.bin");
let copy = client
.copy_object()
.bucket(dst_bucket)
.key(&dst_key)
.copy_source(format!("{src_bucket}/{src_key}"))
.checksum_algorithm(sdk_algorithm)
.send()
.await
.expect("CopyObject with supported checksum must succeed");
let result = copy.copy_object_result().expect("CopyObject result");
let checksums = result_checksums(result);
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: response checksum");
assert_eq!(
checksums.iter().filter(|checksum| checksum.is_some()).count(),
1,
"{rio_algorithm}: only the requested checksum may be returned"
);
let head = client
.head_object()
.bucket(dst_bucket)
.key(&dst_key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("HEAD destination failed");
let checksums = head_checksums(&head);
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: persisted checksum");
assert_eq!(
checksums.iter().filter(|checksum| checksum.is_some()).count(),
1,
"{rio_algorithm}: destination must persist only the requested checksum"
);
let body = client
.get_object()
.bucket(dst_bucket)
.key(&dst_key)
.send()
.await
.expect("GET destination failed")
.body
.collect()
.await
.expect("collect destination body")
.into_bytes();
assert_eq!(body.as_ref(), content, "{rio_algorithm}: full copied body");
}
env.stop_server();
}
#[tokio::test]
#[serial]
async fn test_copy_without_algorithm_preserves_every_supported_source_checksum() {
init_logging();
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client_no_auto_checksum(&env);
let src_bucket = "copy-preserve-all-src";
let dst_bucket = "copy-preserve-all-dst";
let content = b"source checksum preservation payload for all ten algorithms";
create_versioned_bucket(&client, src_bucket).await;
create_versioned_bucket(&client, dst_bucket).await;
for (index, (_sdk_algorithm, rio_algorithm)) in algorithms().into_iter().enumerate() {
let expected = Checksum::new_from_data(rio_algorithm, content)
.expect("supported checksum must be computable")
.encoded;
let checksum_header = rio_algorithm.key().expect("supported checksum header");
let request_checksum = expected.clone();
let src_key = format!("objects/source-{index}.bin");
let dst_key = format!("objects/destination-{index}.bin");
client
.put_object()
.bucket(src_bucket)
.key(&src_key)
.body(ByteStream::from_static(content))
.customize()
.mutate_request(move |request| {
request.headers_mut().insert(checksum_header, request_checksum.clone());
})
.send()
.await
.expect("PUT checksummed source failed");
let copy = client
.copy_object()
.bucket(dst_bucket)
.key(&dst_key)
.copy_source(format!("{src_bucket}/{src_key}"))
.send()
.await
.expect("CopyObject without algorithm must succeed");
let result = copy.copy_object_result().expect("CopyObject result");
let checksums = result_checksums(result);
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: preserved response checksum");
assert_eq!(checksums.iter().filter(|checksum| checksum.is_some()).count(), 1);
let head = client
.head_object()
.bucket(dst_bucket)
.key(&dst_key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("HEAD destination failed");
let checksums = head_checksums(&head);
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: preserved stored checksum");
assert_eq!(checksums.iter().filter(|checksum| checksum.is_some()).count(), 1);
}
env.stop_server();
}
#[tokio::test]
#[serial]
async fn test_copy_without_algorithm_preserves_composite_checksum_type() {
init_logging();
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client_no_auto_checksum(&env);
let bucket = "copy-preserve-composite";
let source_key = "objects/multipart-source.bin";
let destination_key = "objects/copied-multipart.bin";
let content = b"multipart source checksum must remain composite";
create_versioned_bucket(&client, bucket).await;
let created = client
.create_multipart_upload()
.bucket(bucket)
.key(source_key)
.checksum_algorithm(ChecksumAlgorithm::Sha256)
.send()
.await
.expect("CreateMultipartUpload failed");
let upload_id = created.upload_id().expect("multipart upload ID");
let checksum = Checksum::new_from_data(RioChecksumType::SHA256, content)
.expect("SHA256 checksum")
.encoded;
let uploaded = client
.upload_part()
.bucket(bucket)
.key(source_key)
.upload_id(upload_id)
.part_number(1)
.checksum_sha256(&checksum)
.body(ByteStream::from_static(content))
.send()
.await
.expect("UploadPart failed");
let completed_part = CompletedPart::builder()
.part_number(1)
.e_tag(uploaded.e_tag().expect("part ETag"))
.checksum_sha256(uploaded.checksum_sha256().expect("part checksum"))
.build();
client
.complete_multipart_upload()
.bucket(bucket)
.key(source_key)
.upload_id(upload_id)
.multipart_upload(CompletedMultipartUpload::builder().parts(completed_part).build())
.send()
.await
.expect("CompleteMultipartUpload failed");
let source_head = client
.head_object()
.bucket(bucket)
.key(source_key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("HEAD multipart source failed");
let source_checksum = source_head.checksum_sha256().expect("multipart source checksum");
assert_eq!(source_head.checksum_type(), Some(&ChecksumType::Composite));
let copied = client
.copy_object()
.bucket(bucket)
.key(destination_key)
.copy_source(format!("{bucket}/{source_key}"))
.send()
.await
.expect("CopyObject without algorithm failed");
let result = copied.copy_object_result().expect("CopyObject result");
assert_eq!(result.checksum_sha256(), Some(source_checksum));
assert_eq!(result.checksum_type(), Some(&ChecksumType::Composite));
let destination_head = client
.head_object()
.bucket(bucket)
.key(destination_key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("HEAD copied multipart object failed");
assert_eq!(destination_head.checksum_sha256(), Some(source_checksum));
assert_eq!(destination_head.checksum_type(), Some(&ChecksumType::Composite));
env.stop_server();
}
#[tokio::test]
#[serial]
async fn test_copy_rejects_unknown_algorithm_without_destination_mutation() {
init_logging();
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = env.create_s3_client();
let bucket = "copy-reject-unknown-checksum";
let src_key = "objects/source.bin";
let dst_key = "objects/destination.bin";
let source = b"source must never replace destination";
let destination = b"pre-existing destination must remain byte-for-byte unchanged";
let expected = Checksum::new_from_data(RioChecksumType::SHA256, destination)
.expect("SHA256 checksum")
.encoded;
create_versioned_bucket(&client, bucket).await;
client
.put_object()
.bucket(bucket)
.key(src_key)
.body(ByteStream::from_static(source))
.send()
.await
.expect("PUT source failed");
let original = client
.put_object()
.bucket(bucket)
.key(dst_key)
.metadata("state", "original")
.checksum_algorithm(ChecksumAlgorithm::Sha256)
.body(ByteStream::from_static(destination))
.send()
.await
.expect("PUT destination failed");
let original_version = original.version_id().expect("versioned PUT must return a version id");
let missing_source_error = client
.copy_object()
.bucket(bucket)
.key(dst_key)
.copy_source(format!("{bucket}/objects/missing-source.bin"))
.checksum_algorithm(ChecksumAlgorithm::from("BLAKE3"))
.send()
.await
.expect_err("checksum validation must precede source lookup");
assert_eq!(
missing_source_error.as_service_error().and_then(|value| value.code()),
Some("InvalidArgument")
);
assert_eq!(missing_source_error.raw_response().map(|response| response.status().as_u16()), Some(400));
let error = client
.copy_object()
.bucket(bucket)
.key(dst_key)
.copy_source(format!("{bucket}/{src_key}"))
.checksum_algorithm(ChecksumAlgorithm::from("BLAKE3"))
.send()
.await
.expect_err("unsupported checksum algorithm must fail");
assert_eq!(error.as_service_error().and_then(|value| value.code()), Some("InvalidArgument"));
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(400));
let head = client
.head_object()
.bucket(bucket)
.key(dst_key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("HEAD unchanged destination");
assert_eq!(head.version_id(), Some(original_version));
assert_eq!(
head.metadata().and_then(|metadata| metadata.get("state").map(String::as_str)),
Some("original")
);
assert_eq!(head.checksum_sha256(), Some(expected.as_str()));
let body = client
.get_object()
.bucket(bucket)
.key(dst_key)
.send()
.await
.expect("GET unchanged destination")
.body
.collect()
.await
.expect("collect unchanged destination")
.into_bytes();
assert_eq!(body.as_ref(), destination);
env.stop_server();
}
/// Requested algorithm: a CopyObject asking for SHA256 must compute it over the copied
/// bytes, return it in `CopyObjectResult.ChecksumSHA256`, and persist it so a checksum-mode
/// HEAD on the destination returns the identical value.
+2 -4
View File
@@ -180,10 +180,8 @@ impl ChecksumType {
(self.0 & Self::FULL_OBJECT.0) == Self::FULL_OBJECT.0 || self.is(Self::CRC64_NVME)
}
/// True for the five algorithms s3s exposes as typed `*Output` fields
/// (CRC32/CRC32C/SHA1/SHA256/CRC64NVME). The AWS 2026-04 additional algorithms
/// (XXHash3/64/128, SHA-512, MD5) have no typed field and are carried as raw
/// response headers instead. Single source of truth for that split.
/// True for the five legacy algorithms currently handled by RustFS's named
/// response fields. Additional algorithms use the extended response path.
pub fn is_s3s_typed(self) -> bool {
matches!(self.base(), Self::CRC32 | Self::CRC32C | Self::SHA1 | Self::SHA256 | Self::CRC64_NVME)
}
@@ -38,6 +38,7 @@ The implemented test list currently covers the common object-storage surface:
|---|---|---|
| Bucket create/delete/list/head | Supported | `implemented_tests.txt` |
| Object put/get/delete/copy/head | Supported | `implemented_tests.txt` |
| CopyObject checksums (CRC32, CRC32C, CRC64NVME, SHA1, SHA256, MD5, SHA512, XXHASH3, XXHASH64, XXHASH128), including source preservation and explicit override | Supported in the first RustFS release containing this change | `crates/e2e_test/src/copy_object_checksum_test.rs` |
| ListObjects/ListObjectsV2 prefix, delimiter, marker, max-keys | Supported | `implemented_tests.txt` |
| Multipart upload create/upload/complete/abort and selected multipart copy/checksum/object-attribute behavior | Supported | `implemented_tests.txt` |
| Bucket and object tagging | Supported | `implemented_tests.txt` |
+2 -2
View File
@@ -34,7 +34,7 @@
| connection_cap_test | 2 | |
| console_smoke_test | 1 | ✅ |
| content_encoding_test | 3 | ✅ |
| copy_object_checksum_test | 3 | |
| copy_object_checksum_test | 7 | |
| copy_object_metadata_test | 4 | ✅ |
| copy_object_tagging_test | 2 | ✅ |
| copy_object_version_restore_test | 2 | |
@@ -86,4 +86,4 @@
`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
**Total listed: 479 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24.
**Total listed: 483 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24.
+3 -2
View File
@@ -554,11 +554,11 @@ impl DefaultMultipartUsecase {
// checksum: stored (decrypted) values take precedence over the request input;
// additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed
// CompleteMultipartUploadOutput field, are echoed as raw response headers (#1261).
let (checksums, _is_multipart) = obj_info
let (checksums, is_multipart) = obj_info
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
.map_err(ApiError::from)?;
let classified = crate::app::object_usecase::classify_response_checksums(checksums);
let classified = crate::app::object_usecase::classify_response_checksums(checksums, is_multipart);
let checksum_crc32 = classified.crc32.or(input.checksum_crc32);
let checksum_crc32c = classified.crc32c.or(input.checksum_crc32c);
let checksum_sha1 = classified.sha1.or(input.checksum_sha1);
@@ -1352,6 +1352,7 @@ impl DefaultMultipartUsecase {
checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME),
e_tag: part_info.etag.map(|etag| to_s3s_etag(&etag)),
last_modified: part_info.last_mod.map(Timestamp::from),
..Default::default()
};
let output = UploadPartCopyOutput {
+77 -47
View File
@@ -2368,11 +2368,8 @@ fn apply_trailing_checksums(
}
/// Checksums resolved from stored (decrypted) metadata for a response. The five
/// s3s-typed algorithms fill named fields; the AWS 2026-04 additional algorithms
/// (XXHash3/64/128, SHA-512, MD5), which s3s has no typed `*Output` field for, land
/// in `extra` and are emitted as raw response headers via
/// [`inject_additional_checksum_headers`]. Built by [`classify_response_checksums`]
/// so the typed/extra split lives in exactly one place.
/// legacy algorithms fill named fields; the additional algorithms land in `extra`
/// for raw-header response paths and DTOs that expose their newer typed fields.
#[derive(Default)]
pub(crate) struct ResponseChecksums {
pub(crate) crc32: Option<String>,
@@ -2384,11 +2381,11 @@ pub(crate) struct ResponseChecksums {
pub(crate) extra: Vec<(&'static str, String)>,
}
/// Split decrypted checksum (key, value) pairs into the five s3s-typed fields and the
/// additional-algorithm `extra` headers. Single source of truth for every response
/// Split decrypted checksum pairs into the five legacy fields and the additional
/// algorithm values. Single source of truth for every response
/// path (GetObject / HeadObject / GetObjectAttributes / CompleteMultipartUpload),
/// replacing what used to be five copies of this match loop.
pub(crate) fn classify_response_checksums<I>(pairs: I) -> ResponseChecksums
pub(crate) fn classify_response_checksums<I>(pairs: I, is_multipart: bool) -> ResponseChecksums
where
I: IntoIterator<Item = (String, String)>,
{
@@ -2412,6 +2409,9 @@ where
}
}
}
if is_multipart && c.checksum_type.is_none() {
c.checksum_type = Some(ChecksumType::from("COMPOSITE".to_string()));
}
c
}
@@ -4109,13 +4109,12 @@ impl DefaultObjectUsecase {
&& checksum_mode.to_str().unwrap_or_default() == "ENABLED"
&& rs.is_none()
{
let (decrypted_checksums, _is_multipart) =
info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
error!(error = %e, "GetObject checksum decryption failed");
ApiError::from(e)
})?;
let (decrypted_checksums, is_multipart) = info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
error!(error = %e, "GetObject checksum decryption failed");
ApiError::from(e)
})?;
return Ok(classify_response_checksums(decrypted_checksums));
return Ok(classify_response_checksums(decrypted_checksums, is_multipart));
}
Ok(ResponseChecksums::default())
@@ -5626,7 +5625,7 @@ impl DefaultObjectUsecase {
};
let checksum = if requested(ObjectAttributes::CHECKSUM) {
let (checksums, _is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?;
let (checksums, is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?;
// GetObjectAttributes returns checksums in the XML body, and s3s's Checksum
// type has no field for the additional algorithms, so `extra` cannot be
// surfaced here (unlike the header-based GET/HEAD paths) — an s3s limitation
@@ -5639,7 +5638,7 @@ impl DefaultObjectUsecase {
crc64nvme: checksum_crc64nvme,
checksum_type,
..
} = classify_response_checksums(checksums);
} = classify_response_checksums(checksums, is_multipart);
Some(Checksum {
checksum_crc32,
@@ -5648,6 +5647,7 @@ impl DefaultObjectUsecase {
checksum_sha256,
checksum_crc64nvme,
checksum_type,
..Default::default()
})
} else {
None
@@ -5674,7 +5674,7 @@ impl DefaultObjectUsecase {
let is_truncated = end < info.parts.len();
for part in &info.parts[start_at..end] {
let (checksums, _is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?;
let (checksums, is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?;
// Additional algorithms cannot be surfaced in the ObjectPart XML body
// (s3s has no field); same limitation as the object-level attributes above.
let ResponseChecksums {
@@ -5684,7 +5684,7 @@ impl DefaultObjectUsecase {
sha256: checksum_sha256,
crc64nvme: checksum_crc64nvme,
..
} = classify_response_checksums(checksums);
} = classify_response_checksums(checksums, is_multipart);
let part_size = if part.actual_size > 0 {
part.actual_size
@@ -5702,6 +5702,7 @@ impl DefaultObjectUsecase {
checksum_crc64nvme,
part_number: i32::try_from(part.number).ok(),
size: Some(part_size),
..Default::default()
});
}
@@ -5793,6 +5794,12 @@ impl DefaultObjectUsecase {
checksum_algorithm,
..
} = req.input.clone();
let requested_checksum_type = checksum_algorithm
.as_ref()
.map(|algorithm| rustfs_rio::ChecksumType::from_string(algorithm.as_str()));
if requested_checksum_type.is_some_and(|checksum_type| !checksum_type.is_set()) {
return Err(s3_error!(InvalidArgument, "Unsupported checksum algorithm"));
}
let (src_bucket, src_key, version_id) = match copy_source {
CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)),
CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)),
@@ -5996,12 +6003,9 @@ impl DefaultObjectUsecase {
// rather than re-hashing every byte.
let src_checksum = src_info.checksum.as_ref().and_then(|bytes| {
let (pairs, _) = rustfs_rio::read_checksums(bytes.as_ref(), 0);
pairs.into_iter().find_map(|(k, v)| {
rustfs_rio::ChecksumType::from_string(&k)
.is_s3s_typed()
.then(|| rustfs_rio::Checksum::new_from_string(&k, &v))
.flatten()
})
pairs
.into_iter()
.find_map(|(k, v)| rustfs_rio::Checksum::new_from_string(&k, &v))
});
// Validate copy source conditions
@@ -6125,12 +6129,9 @@ impl DefaultObjectUsecase {
// none is requested, carry the source object's stored checksum over unchanged — the copy
// does not alter the plaintext, so re-hashing would be wasted work and would flatten a
// multipart composite value.
match checksum_algorithm.as_ref() {
Some(algo) => {
let ct = rustfs_rio::ChecksumType::from_string(algo.as_str());
if ct.is_set() {
reader.add_calculated_checksum(ct).map_err(ApiError::from)?;
}
match requested_checksum_type {
Some(checksum_type) => {
reader.add_calculated_checksum(checksum_type).map_err(ApiError::from)?;
}
None => {
if let Some(cs) = src_checksum {
@@ -6221,11 +6222,26 @@ impl DefaultObjectUsecase {
// / HeadObject do so the value is identical to a later checksum-mode HEAD/GET (#4996).
let response_checksums = oi
.decrypt_checksums(0, &req.headers)
.map(|(pairs, _)| classify_response_checksums(pairs))
.map(|(pairs, is_multipart)| classify_response_checksums(pairs, is_multipart))
.unwrap_or_default();
// warn!("copy_object oi {:?}", &oi);
let object_info = oi.clone();
let mut checksum_md5 = None;
let mut checksum_sha512 = None;
let mut checksum_xxhash3 = None;
let mut checksum_xxhash64 = None;
let mut checksum_xxhash128 = None;
for (name, value) in response_checksums.extra {
match name {
"x-amz-checksum-md5" => checksum_md5 = Some(value),
"x-amz-checksum-sha512" => checksum_sha512 = Some(value),
"x-amz-checksum-xxhash3" => checksum_xxhash3 = Some(value),
"x-amz-checksum-xxhash64" => checksum_xxhash64 = Some(value),
"x-amz-checksum-xxhash128" => checksum_xxhash128 = Some(value),
_ => {}
}
}
let copy_object_result = CopyObjectResult {
e_tag: oi.etag.map(|etag| to_s3s_etag(&etag)),
last_modified: oi.mod_time.map(Timestamp::from),
@@ -6234,6 +6250,11 @@ impl DefaultObjectUsecase {
checksum_sha1: response_checksums.sha1,
checksum_sha256: response_checksums.sha256,
checksum_crc64nvme: response_checksums.crc64nvme,
checksum_md5,
checksum_sha512,
checksum_xxhash3,
checksum_xxhash64,
checksum_xxhash128,
checksum_type: response_checksums.checksum_type,
};
@@ -7122,10 +7143,10 @@ impl DefaultObjectUsecase {
&& checksum_mode.to_str().unwrap_or_default() == "ENABLED"
&& rs.is_none()
{
let (checksums, _is_multipart) = info
let (checksums, is_multipart) = info
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
.map_err(ApiError::from)?;
classify_response_checksums(checksums)
classify_response_checksums(checksums, is_multipart)
} else {
ResponseChecksums::default()
};
@@ -8109,24 +8130,30 @@ mod tests {
#[test]
fn classify_response_checksums_splits_typed_and_extra() {
// Typed algorithms fill named fields; nothing spills into extra.
let c = classify_response_checksums(vec![
("CRC32".to_string(), "AAAAAA==".to_string()),
("SHA256".to_string(), "c2hhMjU2".to_string()),
("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()),
]);
let c = classify_response_checksums(
vec![
("CRC32".to_string(), "AAAAAA==".to_string()),
("SHA256".to_string(), "c2hhMjU2".to_string()),
("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()),
],
false,
);
assert_eq!(c.crc32.as_deref(), Some("AAAAAA=="));
assert_eq!(c.sha256.as_deref(), Some("c2hhMjU2"));
assert_eq!(c.crc64nvme.as_deref(), Some("Zm9vYmFyCg=="));
assert!(c.extra.is_empty(), "typed algorithms must not land in extra");
// Additional algorithms land in extra keyed by their response-header name.
let c = classify_response_checksums(vec![
("XXHASH3".to_string(), "eHhoMw==".to_string()),
("XXHASH64".to_string(), "eHhoNjQ=".to_string()),
("XXHASH128".to_string(), "eHhoMTI4".to_string()),
("SHA512".to_string(), "c2hhNTEy".to_string()),
("MD5".to_string(), "bWQ1".to_string()),
]);
let c = classify_response_checksums(
vec![
("XXHASH3".to_string(), "eHhoMw==".to_string()),
("XXHASH64".to_string(), "eHhoNjQ=".to_string()),
("XXHASH128".to_string(), "eHhoMTI4".to_string()),
("SHA512".to_string(), "c2hhNTEy".to_string()),
("MD5".to_string(), "bWQ1".to_string()),
],
false,
);
assert!(c.crc32.is_none() && c.sha256.is_none() && c.crc64nvme.is_none());
let names: Vec<&str> = c.extra.iter().map(|(n, _)| *n).collect();
for expected in [
@@ -8141,12 +8168,15 @@ mod tests {
assert_eq!(c.extra.len(), 5);
// The checksum-type marker is captured as the type, not mistaken for an algorithm.
let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())]);
let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())], false);
assert!(c.checksum_type.is_some());
assert!(c.extra.is_empty() && c.crc32.is_none());
let c = classify_response_checksums(vec![("CRC32".to_string(), "AAAAAA==-2".to_string())], true);
assert_eq!(c.checksum_type.as_ref().map(ChecksumType::as_str), Some("COMPOSITE"));
// Empty input yields an all-default result.
let c = classify_response_checksums(Vec::<(String, String)>::new());
let c = classify_response_checksums(Vec::<(String, String)>::new(), false);
assert!(c.crc32.is_none() && c.extra.is_empty() && c.checksum_type.is_none());
}