mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
fix(s3): complete CopyObject checksum support (#5178)
This commit is contained in:
Generated
+3
-2
@@ -10264,7 +10264,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10321,7 +10321,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.14.1"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=a5471625975f5014f7b28eee7e4d801f1b32f529#a5471625975f5014f7b28eee7e4d801f1b32f529"
|
||||
source = "git+https://github.com/s3s-project/s3s.git?rev=8136db4ac8253c0ccfd86f8216e00e0235747757#8136db4ac8253c0ccfd86f8216e00e0235747757"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -10366,6 +10366,7 @@ dependencies = [
|
||||
"transform-stream",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"xxhash-rust",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
|
||||
+1
-1
@@ -286,7 +286,7 @@ redis = { version = "1.4.1" }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "a5471625975f5014f7b28eee7e4d801f1b32f529" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s.git", rev = "8136db4ac8253c0ccfd86f8216e00e0235747757" }
|
||||
serial_test = "3.5.0"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
|
||||
@@ -12,18 +12,24 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression test for Issue #4996: CopyObject must return the destination object's
|
||||
//! checksum in `CopyObjectResult` and persist it so a later checksum-mode HEAD/GET
|
||||
//! returns the same value. Covers both the requested-algorithm case (compute fresh)
|
||||
//! and the no-algorithm case (preserve the source object's existing checksum).
|
||||
//! CopyObject checksum compatibility tests. Covers all supported algorithms,
|
||||
//! source-checksum preservation, explicit override, and fail-closed handling of
|
||||
//! unsupported algorithms before destination mutation.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, ChecksumAlgorithm, ChecksumMode, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, ChecksumAlgorithm, ChecksumMode, ChecksumType, CompletedMultipartUpload, CompletedPart,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::info;
|
||||
@@ -48,6 +54,401 @@ mod tests {
|
||||
.expect("Failed to enable versioning");
|
||||
}
|
||||
|
||||
fn create_s3_client_no_auto_checksum(env: &RustFSTestEnvironment) -> aws_sdk_s3::Client {
|
||||
let credentials = Credentials::new(&env.access_key, &env.secret_key, None, None, "copy-checksum-e2e");
|
||||
let config = aws_sdk_s3::Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(format!("http://{}", env.address))
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||
.http_client(SmithyHttpClientBuilder::new().build_http())
|
||||
.build();
|
||||
aws_sdk_s3::Client::from_conf(config)
|
||||
}
|
||||
|
||||
fn algorithms() -> [(ChecksumAlgorithm, RioChecksumType); 10] {
|
||||
[
|
||||
(ChecksumAlgorithm::Crc32, RioChecksumType::CRC32),
|
||||
(ChecksumAlgorithm::Crc32C, RioChecksumType::CRC32C),
|
||||
(ChecksumAlgorithm::Crc64Nvme, RioChecksumType::CRC64_NVME),
|
||||
(ChecksumAlgorithm::Sha1, RioChecksumType::SHA1),
|
||||
(ChecksumAlgorithm::Sha256, RioChecksumType::SHA256),
|
||||
(ChecksumAlgorithm::Md5, RioChecksumType::MD5),
|
||||
(ChecksumAlgorithm::Sha512, RioChecksumType::SHA512),
|
||||
(ChecksumAlgorithm::Xxhash3, RioChecksumType::XXHASH3),
|
||||
(ChecksumAlgorithm::Xxhash64, RioChecksumType::XXHASH64),
|
||||
(ChecksumAlgorithm::Xxhash128, RioChecksumType::XXHASH128),
|
||||
]
|
||||
}
|
||||
|
||||
fn result_checksums(result: &aws_sdk_s3::types::CopyObjectResult) -> [Option<&str>; 10] {
|
||||
[
|
||||
result.checksum_crc32(),
|
||||
result.checksum_crc32_c(),
|
||||
result.checksum_crc64_nvme(),
|
||||
result.checksum_sha1(),
|
||||
result.checksum_sha256(),
|
||||
result.checksum_md5(),
|
||||
result.checksum_sha512(),
|
||||
result.checksum_xxhash3(),
|
||||
result.checksum_xxhash64(),
|
||||
result.checksum_xxhash128(),
|
||||
]
|
||||
}
|
||||
|
||||
fn head_checksums(output: &aws_sdk_s3::operation::head_object::HeadObjectOutput) -> [Option<&str>; 10] {
|
||||
[
|
||||
output.checksum_crc32(),
|
||||
output.checksum_crc32_c(),
|
||||
output.checksum_crc64_nvme(),
|
||||
output.checksum_sha1(),
|
||||
output.checksum_sha256(),
|
||||
output.checksum_md5(),
|
||||
output.checksum_sha512(),
|
||||
output.checksum_xxhash3(),
|
||||
output.checksum_xxhash64(),
|
||||
output.checksum_xxhash128(),
|
||||
]
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_supports_all_checksum_algorithms() {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client_no_auto_checksum(&env);
|
||||
let src_bucket = "copy-all-checksums-src";
|
||||
let dst_bucket = "copy-all-checksums-dst";
|
||||
let src_key = "objects/source.bin";
|
||||
let content = b"deterministic CopyObject payload for all ten checksum algorithms";
|
||||
|
||||
create_versioned_bucket(&client, src_bucket).await;
|
||||
create_versioned_bucket(&client, dst_bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(src_bucket)
|
||||
.key(src_key)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT source failed");
|
||||
|
||||
for (index, (sdk_algorithm, rio_algorithm)) in algorithms().into_iter().enumerate() {
|
||||
let expected = Checksum::new_from_data(rio_algorithm, content)
|
||||
.expect("supported checksum must be computable")
|
||||
.encoded;
|
||||
let dst_key = format!("objects/destination-{index}.bin");
|
||||
let copy = client
|
||||
.copy_object()
|
||||
.bucket(dst_bucket)
|
||||
.key(&dst_key)
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
.checksum_algorithm(sdk_algorithm)
|
||||
.send()
|
||||
.await
|
||||
.expect("CopyObject with supported checksum must succeed");
|
||||
let result = copy.copy_object_result().expect("CopyObject result");
|
||||
let checksums = result_checksums(result);
|
||||
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: response checksum");
|
||||
assert_eq!(
|
||||
checksums.iter().filter(|checksum| checksum.is_some()).count(),
|
||||
1,
|
||||
"{rio_algorithm}: only the requested checksum may be returned"
|
||||
);
|
||||
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(dst_bucket)
|
||||
.key(&dst_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("HEAD destination failed");
|
||||
let checksums = head_checksums(&head);
|
||||
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: persisted checksum");
|
||||
assert_eq!(
|
||||
checksums.iter().filter(|checksum| checksum.is_some()).count(),
|
||||
1,
|
||||
"{rio_algorithm}: destination must persist only the requested checksum"
|
||||
);
|
||||
|
||||
let body = client
|
||||
.get_object()
|
||||
.bucket(dst_bucket)
|
||||
.key(&dst_key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET destination failed")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect destination body")
|
||||
.into_bytes();
|
||||
assert_eq!(body.as_ref(), content, "{rio_algorithm}: full copied body");
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_without_algorithm_preserves_every_supported_source_checksum() {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client_no_auto_checksum(&env);
|
||||
let src_bucket = "copy-preserve-all-src";
|
||||
let dst_bucket = "copy-preserve-all-dst";
|
||||
let content = b"source checksum preservation payload for all ten algorithms";
|
||||
|
||||
create_versioned_bucket(&client, src_bucket).await;
|
||||
create_versioned_bucket(&client, dst_bucket).await;
|
||||
|
||||
for (index, (_sdk_algorithm, rio_algorithm)) in algorithms().into_iter().enumerate() {
|
||||
let expected = Checksum::new_from_data(rio_algorithm, content)
|
||||
.expect("supported checksum must be computable")
|
||||
.encoded;
|
||||
let checksum_header = rio_algorithm.key().expect("supported checksum header");
|
||||
let request_checksum = expected.clone();
|
||||
let src_key = format!("objects/source-{index}.bin");
|
||||
let dst_key = format!("objects/destination-{index}.bin");
|
||||
client
|
||||
.put_object()
|
||||
.bucket(src_bucket)
|
||||
.key(&src_key)
|
||||
.body(ByteStream::from_static(content))
|
||||
.customize()
|
||||
.mutate_request(move |request| {
|
||||
request.headers_mut().insert(checksum_header, request_checksum.clone());
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT checksummed source failed");
|
||||
|
||||
let copy = client
|
||||
.copy_object()
|
||||
.bucket(dst_bucket)
|
||||
.key(&dst_key)
|
||||
.copy_source(format!("{src_bucket}/{src_key}"))
|
||||
.send()
|
||||
.await
|
||||
.expect("CopyObject without algorithm must succeed");
|
||||
let result = copy.copy_object_result().expect("CopyObject result");
|
||||
let checksums = result_checksums(result);
|
||||
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: preserved response checksum");
|
||||
assert_eq!(checksums.iter().filter(|checksum| checksum.is_some()).count(), 1);
|
||||
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(dst_bucket)
|
||||
.key(&dst_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("HEAD destination failed");
|
||||
let checksums = head_checksums(&head);
|
||||
assert_eq!(checksums[index], Some(expected.as_str()), "{rio_algorithm}: preserved stored checksum");
|
||||
assert_eq!(checksums.iter().filter(|checksum| checksum.is_some()).count(), 1);
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_without_algorithm_preserves_composite_checksum_type() {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = create_s3_client_no_auto_checksum(&env);
|
||||
let bucket = "copy-preserve-composite";
|
||||
let source_key = "objects/multipart-source.bin";
|
||||
let destination_key = "objects/copied-multipart.bin";
|
||||
let content = b"multipart source checksum must remain composite";
|
||||
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
let created = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(source_key)
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateMultipartUpload failed");
|
||||
let upload_id = created.upload_id().expect("multipart upload ID");
|
||||
let checksum = Checksum::new_from_data(RioChecksumType::SHA256, content)
|
||||
.expect("SHA256 checksum")
|
||||
.encoded;
|
||||
let uploaded = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(source_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.checksum_sha256(&checksum)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("UploadPart failed");
|
||||
let completed_part = CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(uploaded.e_tag().expect("part ETag"))
|
||||
.checksum_sha256(uploaded.checksum_sha256().expect("part checksum"))
|
||||
.build();
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(source_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().parts(completed_part).build())
|
||||
.send()
|
||||
.await
|
||||
.expect("CompleteMultipartUpload failed");
|
||||
|
||||
let source_head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(source_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("HEAD multipart source failed");
|
||||
let source_checksum = source_head.checksum_sha256().expect("multipart source checksum");
|
||||
assert_eq!(source_head.checksum_type(), Some(&ChecksumType::Composite));
|
||||
|
||||
let copied = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(destination_key)
|
||||
.copy_source(format!("{bucket}/{source_key}"))
|
||||
.send()
|
||||
.await
|
||||
.expect("CopyObject without algorithm failed");
|
||||
let result = copied.copy_object_result().expect("CopyObject result");
|
||||
assert_eq!(result.checksum_sha256(), Some(source_checksum));
|
||||
assert_eq!(result.checksum_type(), Some(&ChecksumType::Composite));
|
||||
|
||||
let destination_head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(destination_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("HEAD copied multipart object failed");
|
||||
assert_eq!(destination_head.checksum_sha256(), Some(source_checksum));
|
||||
assert_eq!(destination_head.checksum_type(), Some(&ChecksumType::Composite));
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_copy_rejects_unknown_algorithm_without_destination_mutation() {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "copy-reject-unknown-checksum";
|
||||
let src_key = "objects/source.bin";
|
||||
let dst_key = "objects/destination.bin";
|
||||
let source = b"source must never replace destination";
|
||||
let destination = b"pre-existing destination must remain byte-for-byte unchanged";
|
||||
let expected = Checksum::new_from_data(RioChecksumType::SHA256, destination)
|
||||
.expect("SHA256 checksum")
|
||||
.encoded;
|
||||
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(src_key)
|
||||
.body(ByteStream::from_static(source))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT source failed");
|
||||
let original = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(dst_key)
|
||||
.metadata("state", "original")
|
||||
.checksum_algorithm(ChecksumAlgorithm::Sha256)
|
||||
.body(ByteStream::from_static(destination))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT destination failed");
|
||||
let original_version = original.version_id().expect("versioned PUT must return a version id");
|
||||
|
||||
let missing_source_error = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(dst_key)
|
||||
.copy_source(format!("{bucket}/objects/missing-source.bin"))
|
||||
.checksum_algorithm(ChecksumAlgorithm::from("BLAKE3"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("checksum validation must precede source lookup");
|
||||
assert_eq!(
|
||||
missing_source_error.as_service_error().and_then(|value| value.code()),
|
||||
Some("InvalidArgument")
|
||||
);
|
||||
assert_eq!(missing_source_error.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(dst_key)
|
||||
.copy_source(format!("{bucket}/{src_key}"))
|
||||
.checksum_algorithm(ChecksumAlgorithm::from("BLAKE3"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported checksum algorithm must fail");
|
||||
assert_eq!(error.as_service_error().and_then(|value| value.code()), Some("InvalidArgument"));
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(dst_key)
|
||||
.checksum_mode(ChecksumMode::Enabled)
|
||||
.send()
|
||||
.await
|
||||
.expect("HEAD unchanged destination");
|
||||
assert_eq!(head.version_id(), Some(original_version));
|
||||
assert_eq!(
|
||||
head.metadata().and_then(|metadata| metadata.get("state").map(String::as_str)),
|
||||
Some("original")
|
||||
);
|
||||
assert_eq!(head.checksum_sha256(), Some(expected.as_str()));
|
||||
|
||||
let body = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(dst_key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET unchanged destination")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("collect unchanged destination")
|
||||
.into_bytes();
|
||||
assert_eq!(body.as_ref(), destination);
|
||||
|
||||
env.stop_server();
|
||||
}
|
||||
|
||||
/// Requested algorithm: a CopyObject asking for SHA256 must compute it over the copied
|
||||
/// bytes, return it in `CopyObjectResult.ChecksumSHA256`, and persist it so a checksum-mode
|
||||
/// HEAD on the destination returns the identical value.
|
||||
|
||||
@@ -180,10 +180,8 @@ impl ChecksumType {
|
||||
(self.0 & Self::FULL_OBJECT.0) == Self::FULL_OBJECT.0 || self.is(Self::CRC64_NVME)
|
||||
}
|
||||
|
||||
/// True for the five algorithms s3s exposes as typed `*Output` fields
|
||||
/// (CRC32/CRC32C/SHA1/SHA256/CRC64NVME). The AWS 2026-04 additional algorithms
|
||||
/// (XXHash3/64/128, SHA-512, MD5) have no typed field and are carried as raw
|
||||
/// response headers instead. Single source of truth for that split.
|
||||
/// True for the five legacy algorithms currently handled by RustFS's named
|
||||
/// response fields. Additional algorithms use the extended response path.
|
||||
pub fn is_s3s_typed(self) -> bool {
|
||||
matches!(self.base(), Self::CRC32 | Self::CRC32C | Self::SHA1 | Self::SHA256 | Self::CRC64_NVME)
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ The implemented test list currently covers the common object-storage surface:
|
||||
|---|---|---|
|
||||
| Bucket create/delete/list/head | Supported | `implemented_tests.txt` |
|
||||
| Object put/get/delete/copy/head | Supported | `implemented_tests.txt` |
|
||||
| CopyObject checksums (CRC32, CRC32C, CRC64NVME, SHA1, SHA256, MD5, SHA512, XXHASH3, XXHASH64, XXHASH128), including source preservation and explicit override | Supported in the first RustFS release containing this change | `crates/e2e_test/src/copy_object_checksum_test.rs` |
|
||||
| ListObjects/ListObjectsV2 prefix, delimiter, marker, max-keys | Supported | `implemented_tests.txt` |
|
||||
| Multipart upload create/upload/complete/abort and selected multipart copy/checksum/object-attribute behavior | Supported | `implemented_tests.txt` |
|
||||
| Bucket and object tagging | Supported | `implemented_tests.txt` |
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
| connection_cap_test | 2 | |
|
||||
| console_smoke_test | 1 | ✅ |
|
||||
| content_encoding_test | 3 | ✅ |
|
||||
| copy_object_checksum_test | 3 | |
|
||||
| copy_object_checksum_test | 7 | |
|
||||
| copy_object_metadata_test | 4 | ✅ |
|
||||
| copy_object_tagging_test | 2 | ✅ |
|
||||
| copy_object_version_restore_test | 2 | |
|
||||
@@ -86,4 +86,4 @@
|
||||
|
||||
`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
|
||||
|
||||
**Total listed: 479 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24.
|
||||
**Total listed: 483 tests across 66 modules · PR smoke subset: 126 tests / 31 modules** (29 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-24.
|
||||
|
||||
@@ -554,11 +554,11 @@ impl DefaultMultipartUsecase {
|
||||
// checksum: stored (decrypted) values take precedence over the request input;
|
||||
// additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed
|
||||
// CompleteMultipartUploadOutput field, are echoed as raw response headers (#1261).
|
||||
let (checksums, _is_multipart) = obj_info
|
||||
let (checksums, is_multipart) = obj_info
|
||||
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let classified = crate::app::object_usecase::classify_response_checksums(checksums);
|
||||
let classified = crate::app::object_usecase::classify_response_checksums(checksums, is_multipart);
|
||||
let checksum_crc32 = classified.crc32.or(input.checksum_crc32);
|
||||
let checksum_crc32c = classified.crc32c.or(input.checksum_crc32c);
|
||||
let checksum_sha1 = classified.sha1.or(input.checksum_sha1);
|
||||
@@ -1352,6 +1352,7 @@ impl DefaultMultipartUsecase {
|
||||
checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME),
|
||||
e_tag: part_info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
last_modified: part_info.last_mod.map(Timestamp::from),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let output = UploadPartCopyOutput {
|
||||
|
||||
@@ -2368,11 +2368,8 @@ fn apply_trailing_checksums(
|
||||
}
|
||||
|
||||
/// Checksums resolved from stored (decrypted) metadata for a response. The five
|
||||
/// s3s-typed algorithms fill named fields; the AWS 2026-04 additional algorithms
|
||||
/// (XXHash3/64/128, SHA-512, MD5), which s3s has no typed `*Output` field for, land
|
||||
/// in `extra` and are emitted as raw response headers via
|
||||
/// [`inject_additional_checksum_headers`]. Built by [`classify_response_checksums`]
|
||||
/// so the typed/extra split lives in exactly one place.
|
||||
/// legacy algorithms fill named fields; the additional algorithms land in `extra`
|
||||
/// for raw-header response paths and DTOs that expose their newer typed fields.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct ResponseChecksums {
|
||||
pub(crate) crc32: Option<String>,
|
||||
@@ -2384,11 +2381,11 @@ pub(crate) struct ResponseChecksums {
|
||||
pub(crate) extra: Vec<(&'static str, String)>,
|
||||
}
|
||||
|
||||
/// Split decrypted checksum (key, value) pairs into the five s3s-typed fields and the
|
||||
/// additional-algorithm `extra` headers. Single source of truth for every response
|
||||
/// Split decrypted checksum pairs into the five legacy fields and the additional
|
||||
/// algorithm values. Single source of truth for every response
|
||||
/// path (GetObject / HeadObject / GetObjectAttributes / CompleteMultipartUpload),
|
||||
/// replacing what used to be five copies of this match loop.
|
||||
pub(crate) fn classify_response_checksums<I>(pairs: I) -> ResponseChecksums
|
||||
pub(crate) fn classify_response_checksums<I>(pairs: I, is_multipart: bool) -> ResponseChecksums
|
||||
where
|
||||
I: IntoIterator<Item = (String, String)>,
|
||||
{
|
||||
@@ -2412,6 +2409,9 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
if is_multipart && c.checksum_type.is_none() {
|
||||
c.checksum_type = Some(ChecksumType::from("COMPOSITE".to_string()));
|
||||
}
|
||||
c
|
||||
}
|
||||
|
||||
@@ -4109,13 +4109,12 @@ impl DefaultObjectUsecase {
|
||||
&& checksum_mode.to_str().unwrap_or_default() == "ENABLED"
|
||||
&& rs.is_none()
|
||||
{
|
||||
let (decrypted_checksums, _is_multipart) =
|
||||
info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
|
||||
error!(error = %e, "GetObject checksum decryption failed");
|
||||
ApiError::from(e)
|
||||
})?;
|
||||
let (decrypted_checksums, is_multipart) = info.decrypt_checksums(part_number.unwrap_or(0), headers).map_err(|e| {
|
||||
error!(error = %e, "GetObject checksum decryption failed");
|
||||
ApiError::from(e)
|
||||
})?;
|
||||
|
||||
return Ok(classify_response_checksums(decrypted_checksums));
|
||||
return Ok(classify_response_checksums(decrypted_checksums, is_multipart));
|
||||
}
|
||||
|
||||
Ok(ResponseChecksums::default())
|
||||
@@ -5626,7 +5625,7 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
|
||||
let checksum = if requested(ObjectAttributes::CHECKSUM) {
|
||||
let (checksums, _is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?;
|
||||
let (checksums, is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?;
|
||||
// GetObjectAttributes returns checksums in the XML body, and s3s's Checksum
|
||||
// type has no field for the additional algorithms, so `extra` cannot be
|
||||
// surfaced here (unlike the header-based GET/HEAD paths) — an s3s limitation
|
||||
@@ -5639,7 +5638,7 @@ impl DefaultObjectUsecase {
|
||||
crc64nvme: checksum_crc64nvme,
|
||||
checksum_type,
|
||||
..
|
||||
} = classify_response_checksums(checksums);
|
||||
} = classify_response_checksums(checksums, is_multipart);
|
||||
|
||||
Some(Checksum {
|
||||
checksum_crc32,
|
||||
@@ -5648,6 +5647,7 @@ impl DefaultObjectUsecase {
|
||||
checksum_sha256,
|
||||
checksum_crc64nvme,
|
||||
checksum_type,
|
||||
..Default::default()
|
||||
})
|
||||
} else {
|
||||
None
|
||||
@@ -5674,7 +5674,7 @@ impl DefaultObjectUsecase {
|
||||
let is_truncated = end < info.parts.len();
|
||||
|
||||
for part in &info.parts[start_at..end] {
|
||||
let (checksums, _is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?;
|
||||
let (checksums, is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?;
|
||||
// Additional algorithms cannot be surfaced in the ObjectPart XML body
|
||||
// (s3s has no field); same limitation as the object-level attributes above.
|
||||
let ResponseChecksums {
|
||||
@@ -5684,7 +5684,7 @@ impl DefaultObjectUsecase {
|
||||
sha256: checksum_sha256,
|
||||
crc64nvme: checksum_crc64nvme,
|
||||
..
|
||||
} = classify_response_checksums(checksums);
|
||||
} = classify_response_checksums(checksums, is_multipart);
|
||||
|
||||
let part_size = if part.actual_size > 0 {
|
||||
part.actual_size
|
||||
@@ -5702,6 +5702,7 @@ impl DefaultObjectUsecase {
|
||||
checksum_crc64nvme,
|
||||
part_number: i32::try_from(part.number).ok(),
|
||||
size: Some(part_size),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
|
||||
@@ -5793,6 +5794,12 @@ impl DefaultObjectUsecase {
|
||||
checksum_algorithm,
|
||||
..
|
||||
} = req.input.clone();
|
||||
let requested_checksum_type = checksum_algorithm
|
||||
.as_ref()
|
||||
.map(|algorithm| rustfs_rio::ChecksumType::from_string(algorithm.as_str()));
|
||||
if requested_checksum_type.is_some_and(|checksum_type| !checksum_type.is_set()) {
|
||||
return Err(s3_error!(InvalidArgument, "Unsupported checksum algorithm"));
|
||||
}
|
||||
let (src_bucket, src_key, version_id) = match copy_source {
|
||||
CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)),
|
||||
CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)),
|
||||
@@ -5996,12 +6003,9 @@ impl DefaultObjectUsecase {
|
||||
// rather than re-hashing every byte.
|
||||
let src_checksum = src_info.checksum.as_ref().and_then(|bytes| {
|
||||
let (pairs, _) = rustfs_rio::read_checksums(bytes.as_ref(), 0);
|
||||
pairs.into_iter().find_map(|(k, v)| {
|
||||
rustfs_rio::ChecksumType::from_string(&k)
|
||||
.is_s3s_typed()
|
||||
.then(|| rustfs_rio::Checksum::new_from_string(&k, &v))
|
||||
.flatten()
|
||||
})
|
||||
pairs
|
||||
.into_iter()
|
||||
.find_map(|(k, v)| rustfs_rio::Checksum::new_from_string(&k, &v))
|
||||
});
|
||||
|
||||
// Validate copy source conditions
|
||||
@@ -6125,12 +6129,9 @@ impl DefaultObjectUsecase {
|
||||
// none is requested, carry the source object's stored checksum over unchanged — the copy
|
||||
// does not alter the plaintext, so re-hashing would be wasted work and would flatten a
|
||||
// multipart composite value.
|
||||
match checksum_algorithm.as_ref() {
|
||||
Some(algo) => {
|
||||
let ct = rustfs_rio::ChecksumType::from_string(algo.as_str());
|
||||
if ct.is_set() {
|
||||
reader.add_calculated_checksum(ct).map_err(ApiError::from)?;
|
||||
}
|
||||
match requested_checksum_type {
|
||||
Some(checksum_type) => {
|
||||
reader.add_calculated_checksum(checksum_type).map_err(ApiError::from)?;
|
||||
}
|
||||
None => {
|
||||
if let Some(cs) = src_checksum {
|
||||
@@ -6221,11 +6222,26 @@ impl DefaultObjectUsecase {
|
||||
// / HeadObject do so the value is identical to a later checksum-mode HEAD/GET (#4996).
|
||||
let response_checksums = oi
|
||||
.decrypt_checksums(0, &req.headers)
|
||||
.map(|(pairs, _)| classify_response_checksums(pairs))
|
||||
.map(|(pairs, is_multipart)| classify_response_checksums(pairs, is_multipart))
|
||||
.unwrap_or_default();
|
||||
|
||||
// warn!("copy_object oi {:?}", &oi);
|
||||
let object_info = oi.clone();
|
||||
let mut checksum_md5 = None;
|
||||
let mut checksum_sha512 = None;
|
||||
let mut checksum_xxhash3 = None;
|
||||
let mut checksum_xxhash64 = None;
|
||||
let mut checksum_xxhash128 = None;
|
||||
for (name, value) in response_checksums.extra {
|
||||
match name {
|
||||
"x-amz-checksum-md5" => checksum_md5 = Some(value),
|
||||
"x-amz-checksum-sha512" => checksum_sha512 = Some(value),
|
||||
"x-amz-checksum-xxhash3" => checksum_xxhash3 = Some(value),
|
||||
"x-amz-checksum-xxhash64" => checksum_xxhash64 = Some(value),
|
||||
"x-amz-checksum-xxhash128" => checksum_xxhash128 = Some(value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let copy_object_result = CopyObjectResult {
|
||||
e_tag: oi.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
last_modified: oi.mod_time.map(Timestamp::from),
|
||||
@@ -6234,6 +6250,11 @@ impl DefaultObjectUsecase {
|
||||
checksum_sha1: response_checksums.sha1,
|
||||
checksum_sha256: response_checksums.sha256,
|
||||
checksum_crc64nvme: response_checksums.crc64nvme,
|
||||
checksum_md5,
|
||||
checksum_sha512,
|
||||
checksum_xxhash3,
|
||||
checksum_xxhash64,
|
||||
checksum_xxhash128,
|
||||
checksum_type: response_checksums.checksum_type,
|
||||
};
|
||||
|
||||
@@ -7122,10 +7143,10 @@ impl DefaultObjectUsecase {
|
||||
&& checksum_mode.to_str().unwrap_or_default() == "ENABLED"
|
||||
&& rs.is_none()
|
||||
{
|
||||
let (checksums, _is_multipart) = info
|
||||
let (checksums, is_multipart) = info
|
||||
.decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers)
|
||||
.map_err(ApiError::from)?;
|
||||
classify_response_checksums(checksums)
|
||||
classify_response_checksums(checksums, is_multipart)
|
||||
} else {
|
||||
ResponseChecksums::default()
|
||||
};
|
||||
@@ -8109,24 +8130,30 @@ mod tests {
|
||||
#[test]
|
||||
fn classify_response_checksums_splits_typed_and_extra() {
|
||||
// Typed algorithms fill named fields; nothing spills into extra.
|
||||
let c = classify_response_checksums(vec![
|
||||
("CRC32".to_string(), "AAAAAA==".to_string()),
|
||||
("SHA256".to_string(), "c2hhMjU2".to_string()),
|
||||
("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()),
|
||||
]);
|
||||
let c = classify_response_checksums(
|
||||
vec![
|
||||
("CRC32".to_string(), "AAAAAA==".to_string()),
|
||||
("SHA256".to_string(), "c2hhMjU2".to_string()),
|
||||
("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()),
|
||||
],
|
||||
false,
|
||||
);
|
||||
assert_eq!(c.crc32.as_deref(), Some("AAAAAA=="));
|
||||
assert_eq!(c.sha256.as_deref(), Some("c2hhMjU2"));
|
||||
assert_eq!(c.crc64nvme.as_deref(), Some("Zm9vYmFyCg=="));
|
||||
assert!(c.extra.is_empty(), "typed algorithms must not land in extra");
|
||||
|
||||
// Additional algorithms land in extra keyed by their response-header name.
|
||||
let c = classify_response_checksums(vec![
|
||||
("XXHASH3".to_string(), "eHhoMw==".to_string()),
|
||||
("XXHASH64".to_string(), "eHhoNjQ=".to_string()),
|
||||
("XXHASH128".to_string(), "eHhoMTI4".to_string()),
|
||||
("SHA512".to_string(), "c2hhNTEy".to_string()),
|
||||
("MD5".to_string(), "bWQ1".to_string()),
|
||||
]);
|
||||
let c = classify_response_checksums(
|
||||
vec![
|
||||
("XXHASH3".to_string(), "eHhoMw==".to_string()),
|
||||
("XXHASH64".to_string(), "eHhoNjQ=".to_string()),
|
||||
("XXHASH128".to_string(), "eHhoMTI4".to_string()),
|
||||
("SHA512".to_string(), "c2hhNTEy".to_string()),
|
||||
("MD5".to_string(), "bWQ1".to_string()),
|
||||
],
|
||||
false,
|
||||
);
|
||||
assert!(c.crc32.is_none() && c.sha256.is_none() && c.crc64nvme.is_none());
|
||||
let names: Vec<&str> = c.extra.iter().map(|(n, _)| *n).collect();
|
||||
for expected in [
|
||||
@@ -8141,12 +8168,15 @@ mod tests {
|
||||
assert_eq!(c.extra.len(), 5);
|
||||
|
||||
// The checksum-type marker is captured as the type, not mistaken for an algorithm.
|
||||
let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())]);
|
||||
let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())], false);
|
||||
assert!(c.checksum_type.is_some());
|
||||
assert!(c.extra.is_empty() && c.crc32.is_none());
|
||||
|
||||
let c = classify_response_checksums(vec![("CRC32".to_string(), "AAAAAA==-2".to_string())], true);
|
||||
assert_eq!(c.checksum_type.as_ref().map(ChecksumType::as_str), Some("COMPOSITE"));
|
||||
|
||||
// Empty input yields an all-default result.
|
||||
let c = classify_response_checksums(Vec::<(String, String)>::new());
|
||||
let c = classify_response_checksums(Vec::<(String, String)>::new(), false);
|
||||
assert!(c.crc32.is_none() && c.extra.is_empty() && c.checksum_type.is_none());
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user