diff --git a/Cargo.lock b/Cargo.lock index 7a60a300c..8ee01f389 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9009,6 +9009,7 @@ dependencies = [ "pretty_assertions", "sha1 0.11.0", "sha2 0.11.0", + "xxhash-rust", ] [[package]] @@ -9787,6 +9788,7 @@ dependencies = [ "tokio-test", "tokio-util", "tracing", + "xxhash-rust", ] [[package]] diff --git a/crates/checksums/Cargo.toml b/crates/checksums/Cargo.toml index 6ac09669b..6242d96f1 100644 --- a/crates/checksums/Cargo.toml +++ b/crates/checksums/Cargo.toml @@ -33,6 +33,7 @@ base64-simd = { workspace = true } md-5 = { workspace = true } sha1 = { workspace = true } sha2 = { workspace = true } +xxhash-rust = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/crates/checksums/src/error.rs b/crates/checksums/src/error.rs index 9749e9739..a305c10c3 100644 --- a/crates/checksums/src/error.rs +++ b/crates/checksums/src/error.rs @@ -36,7 +36,7 @@ impl fmt::Display for UnknownChecksumAlgorithmError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - r#"unknown checksum algorithm "{}", please pass a known algorithm name ("crc32", "crc32c", "crc64nvme", "sha1", "sha256")"#, + r#"unknown checksum algorithm "{}", please pass a known algorithm name ("crc32", "crc32c", "crc64nvme", "sha1", "sha256", "sha512", "xxhash3", "xxhash64", "xxhash128")"#, self.checksum_algorithm ) } diff --git a/crates/checksums/src/http.rs b/crates/checksums/src/http.rs index 4fd4d9d1b..e54340530 100644 --- a/crates/checksums/src/http.rs +++ b/crates/checksums/src/http.rs @@ -16,13 +16,20 @@ use crate::base64; use http::header::{HeaderMap, HeaderValue}; use crate::Crc64Nvme; -use crate::{CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, Checksum, Crc32, Crc32c, Md5, SHA_1_NAME, SHA_256_NAME, Sha1, Sha256}; +use crate::{ + CRC_32_C_NAME, CRC_32_NAME, CRC_64_NVME_NAME, Checksum, Crc32, Crc32c, Md5, SHA_1_NAME, SHA_256_NAME, Sha1, Sha256, Sha512, + Xxhash3, Xxhash64, Xxhash128, +}; pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32"; pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c"; pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1"; pub const SHA_256_HEADER_NAME: &str = "x-amz-checksum-sha256"; pub const CRC_64_NVME_HEADER_NAME: &str = "x-amz-checksum-crc64nvme"; +pub const SHA_512_HEADER_NAME: &str = "x-amz-checksum-sha512"; +pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3"; +pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64"; +pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128"; #[allow(dead_code)] pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; @@ -85,6 +92,30 @@ impl HttpChecksum for Sha256 { } } +impl HttpChecksum for Sha512 { + fn header_name(&self) -> &'static str { + SHA_512_HEADER_NAME + } +} + +impl HttpChecksum for Xxhash3 { + fn header_name(&self) -> &'static str { + XXHASH_3_HEADER_NAME + } +} + +impl HttpChecksum for Xxhash64 { + fn header_name(&self) -> &'static str { + XXHASH_64_HEADER_NAME + } +} + +impl HttpChecksum for Xxhash128 { + fn header_name(&self) -> &'static str { + XXHASH_128_HEADER_NAME + } +} + impl HttpChecksum for Md5 { fn header_name(&self) -> &'static str { MD5_HEADER_NAME diff --git a/crates/checksums/src/lib.rs b/crates/checksums/src/lib.rs index 80a3b0b1b..2c97a82e7 100644 --- a/crates/checksums/src/lib.rs +++ b/crates/checksums/src/lib.rs @@ -35,6 +35,10 @@ pub const CRC_32_C_NAME: &str = "crc32c"; pub const CRC_64_NVME_NAME: &str = "crc64nvme"; pub const SHA_1_NAME: &str = "sha1"; pub const SHA_256_NAME: &str = "sha256"; +pub const SHA_512_NAME: &str = "sha512"; +pub const XXHASH_3_NAME: &str = "xxhash3"; +pub const XXHASH_64_NAME: &str = "xxhash64"; +pub const XXHASH_128_NAME: &str = "xxhash128"; pub const MD5_NAME: &str = "md5"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -46,6 +50,10 @@ pub enum ChecksumAlgorithm { Sha1, Sha256, Crc64Nvme, + Sha512, + Xxhash3, + Xxhash64, + Xxhash128, } impl FromStr for ChecksumAlgorithm { @@ -62,6 +70,14 @@ impl FromStr for ChecksumAlgorithm { Ok(Self::Sha256) } else if checksum_algorithm.eq_ignore_ascii_case(CRC_64_NVME_NAME) { Ok(Self::Crc64Nvme) + } else if checksum_algorithm.eq_ignore_ascii_case(SHA_512_NAME) { + Ok(Self::Sha512) + } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_3_NAME) { + Ok(Self::Xxhash3) + } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_64_NAME) { + Ok(Self::Xxhash64) + } else if checksum_algorithm.eq_ignore_ascii_case(XXHASH_128_NAME) { + Ok(Self::Xxhash128) } else { Err(UnknownChecksumAlgorithmError::new(checksum_algorithm)) } @@ -76,6 +92,10 @@ impl ChecksumAlgorithm { Self::Crc64Nvme => Box::::default(), Self::Sha1 => Box::::default(), Self::Sha256 => Box::::default(), + Self::Sha512 => Box::::default(), + Self::Xxhash3 => Box::::default(), + Self::Xxhash64 => Box::::default(), + Self::Xxhash128 => Box::::default(), } } @@ -86,6 +106,10 @@ impl ChecksumAlgorithm { Self::Crc64Nvme => CRC_64_NVME_NAME, Self::Sha1 => SHA_1_NAME, Self::Sha256 => SHA_256_NAME, + Self::Sha512 => SHA_512_NAME, + Self::Xxhash3 => XXHASH_3_NAME, + Self::Xxhash64 => XXHASH_64_NAME, + Self::Xxhash128 => XXHASH_128_NAME, } } } @@ -285,6 +309,165 @@ impl Checksum for Sha256 { Self::size() } } + +#[derive(Debug, Default)] +struct Sha512 { + hasher: sha2::Sha512, +} + +impl Sha512 { + fn update(&mut self, bytes: &[u8]) { + use sha2::Digest; + self.hasher.update(bytes); + } + + fn finalize(self) -> Bytes { + use sha2::Digest; + Bytes::copy_from_slice(self.hasher.finalize().as_slice()) + } + + fn size() -> u64 { + use sha2::Digest; + sha2::Sha512::output_size() as u64 + } +} + +impl Checksum for Sha512 { + fn update(&mut self, bytes: &[u8]) { + Self::update(self, bytes); + } + fn finalize(self: Box) -> Bytes { + Self::finalize(*self) + } + fn size(&self) -> u64 { + Self::size() + } +} + +/// XXH3 (64-bit) hasher with the canonical seed of 0. +/// +/// The raw digest is a `u64` serialized as 8 big-endian bytes so that the value +/// matches the server-side (`rustfs-rio`) computation for the same algorithm. +struct Xxhash3 { + hasher: xxhash_rust::xxh3::Xxh3, +} + +impl Default for Xxhash3 { + fn default() -> Self { + Self { + hasher: xxhash_rust::xxh3::Xxh3::new(), + } + } +} + +impl Xxhash3 { + fn update(&mut self, bytes: &[u8]) { + self.hasher.update(bytes); + } + + fn finalize(self) -> Bytes { + Bytes::copy_from_slice(self.hasher.digest().to_be_bytes().as_slice()) + } + + fn size() -> u64 { + 8 + } +} + +impl Checksum for Xxhash3 { + fn update(&mut self, bytes: &[u8]) { + Self::update(self, bytes) + } + fn finalize(self: Box) -> Bytes { + Self::finalize(*self) + } + fn size(&self) -> u64 { + Self::size() + } +} + +/// XXH3 (128-bit) hasher with the canonical seed of 0. +/// +/// The raw digest is a `u128` serialized as 16 big-endian bytes. +struct Xxhash128 { + hasher: xxhash_rust::xxh3::Xxh3, +} + +impl Default for Xxhash128 { + fn default() -> Self { + Self { + hasher: xxhash_rust::xxh3::Xxh3::new(), + } + } +} + +impl Xxhash128 { + fn update(&mut self, bytes: &[u8]) { + self.hasher.update(bytes); + } + + fn finalize(self) -> Bytes { + Bytes::copy_from_slice(self.hasher.digest128().to_be_bytes().as_slice()) + } + + fn size() -> u64 { + 16 + } +} + +impl Checksum for Xxhash128 { + fn update(&mut self, bytes: &[u8]) { + Self::update(self, bytes) + } + fn finalize(self: Box) -> Bytes { + Self::finalize(*self) + } + fn size(&self) -> u64 { + Self::size() + } +} + +/// XXH64 hasher with the canonical seed of 0. +/// +/// The raw digest is a `u64` serialized as 8 big-endian bytes. +struct Xxhash64 { + hasher: xxhash_rust::xxh64::Xxh64, +} + +impl Default for Xxhash64 { + fn default() -> Self { + Self { + hasher: xxhash_rust::xxh64::Xxh64::new(0), + } + } +} + +impl Xxhash64 { + fn update(&mut self, bytes: &[u8]) { + self.hasher.update(bytes); + } + + fn finalize(self) -> Bytes { + Bytes::copy_from_slice(self.hasher.digest().to_be_bytes().as_slice()) + } + + fn size() -> u64 { + 8 + } +} + +impl Checksum for Xxhash64 { + fn update(&mut self, bytes: &[u8]) { + Self::update(self, bytes) + } + fn finalize(self: Box) -> Bytes { + Self::finalize(*self) + } + fn size(&self) -> u64 { + Self::size() + } +} + #[allow(dead_code)] #[derive(Debug, Default)] struct Md5 { @@ -455,4 +638,97 @@ mod tests { let error = "MD5".parse::().expect_err("md5 should not parse"); assert_eq!("MD5", error.checksum_algorithm()); } + + #[test] + fn test_additional_algorithms_parse_and_round_trip() { + // The AWS 2026-04 additional checksum algorithms must be recognised + // (case-insensitively) and round-trip through as_str(). + for (name, expected) in [ + ("sha512", ChecksumAlgorithm::Sha512), + ("SHA512", ChecksumAlgorithm::Sha512), + ("xxhash3", ChecksumAlgorithm::Xxhash3), + ("XXHASH3", ChecksumAlgorithm::Xxhash3), + ("xxhash64", ChecksumAlgorithm::Xxhash64), + ("xxhash128", ChecksumAlgorithm::Xxhash128), + ] { + let parsed = name.parse::().expect("algorithm should parse"); + assert_eq!(parsed, expected); + assert_eq!(expected.as_str().parse::().unwrap(), expected); + } + } + + #[test] + fn test_unknown_algorithm_never_panics_and_fails_closed() { + // Fail-closed contract: an unknown or garbage algorithm name must return + // an error instead of panicking or silently substituting another hasher. + for name in ["", "xxhash", "sha3", "crc16", "not-a-real-algo", "🦀"] { + assert!(name.parse::().is_err(), "unknown algorithm {name:?} must fail closed"); + } + } + + #[test] + fn test_sha512_matches_direct_computation() { + use crate::Sha512; + use crate::http::SHA_512_HEADER_NAME; + use sha2::{Digest, Sha512 as Sha512Ref}; + + let mut checksum = Sha512::default(); + checksum.update(TEST_DATA.as_bytes()); + let header = Box::new(checksum).headers(); + let encoded = header.get(SHA_512_HEADER_NAME).expect("sha512 header present"); + let got = base64_encoded_checksum_to_hex_string(encoded); + + let mut reference = Sha512Ref::new(); + reference.update(TEST_DATA.as_bytes()); + let expected = reference.finalize().iter().fold(String::from("0x"), |mut acc, b| { + write!(acc, "{b:02X?}").unwrap(); + acc + }); + assert_eq!(got, expected); + } + + #[test] + fn test_xxhash3_matches_direct_computation_big_endian_seed0() { + use crate::Xxhash3; + use xxhash_rust::xxh3::Xxh3; + + let mut checksum = Xxhash3::default(); + checksum.update(TEST_DATA.as_bytes()); + let raw = Box::new(checksum).finalize(); + + let mut reference = Xxh3::new(); + reference.update(TEST_DATA.as_bytes()); + assert_eq!(raw.len(), 8); + assert_eq!(&raw[..], reference.digest().to_be_bytes().as_slice()); + } + + #[test] + fn test_xxhash128_matches_direct_computation_big_endian_seed0() { + use crate::Xxhash128; + use xxhash_rust::xxh3::Xxh3; + + let mut checksum = Xxhash128::default(); + checksum.update(TEST_DATA.as_bytes()); + let raw = Box::new(checksum).finalize(); + + let mut reference = Xxh3::new(); + reference.update(TEST_DATA.as_bytes()); + assert_eq!(raw.len(), 16); + assert_eq!(&raw[..], reference.digest128().to_be_bytes().as_slice()); + } + + #[test] + fn test_xxhash64_matches_direct_computation_big_endian_seed0() { + use crate::Xxhash64; + use xxhash_rust::xxh64::Xxh64; + + let mut checksum = Xxhash64::default(); + checksum.update(TEST_DATA.as_bytes()); + let raw = Box::new(checksum).finalize(); + + let mut reference = Xxh64::new(0); + reference.update(TEST_DATA.as_bytes()); + assert_eq!(raw.len(), 8); + assert_eq!(&raw[..], reference.digest().to_be_bytes().as_slice()); + } } diff --git a/crates/e2e_test/src/checksum_upload_test.rs b/crates/e2e_test/src/checksum_upload_test.rs index e0dcf525f..622545dd6 100644 --- a/crates/e2e_test/src/checksum_upload_test.rs +++ b/crates/e2e_test/src/checksum_upload_test.rs @@ -19,8 +19,10 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; + use aws_sdk_s3::config::{Credentials, Region, RequestChecksumCalculation}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart}; + use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; use base64::Engine; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; use serial_test::serial; @@ -31,6 +33,25 @@ mod tests { env.create_s3_client() } + /// Client with boto/SDK automatic checksum calculation disabled, so the ONLY + /// checksum on the wire is the one the test injects. Needed because the SDK's + /// default (crc32) would otherwise collide with the additional-algorithm header + /// we inject via `mutate_request` for XXHash/SHA-512/MD5 (which have no typed + /// SDK builder). Mirrors the boto3 `request_checksum_calculation=when_required`. + fn create_s3_client_no_auto_checksum(env: &RustFSTestEnvironment) -> Client { + let creds = Credentials::new(&env.access_key, &env.secret_key, None, None, "e2e-additional-checksum"); + let config = aws_sdk_s3::Config::builder() + .credentials_provider(creds) + .region(Region::new("us-east-1")) + .endpoint_url(format!("http://{}", env.address)) + .force_path_style(true) + .behavior_version_latest() + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .http_client(SmithyHttpClientBuilder::new().build_http()) + .build(); + Client::from_conf(config) + } + async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box> { match client.create_bucket().bucket(bucket).send().await { Ok(_) => { @@ -459,4 +480,87 @@ mod tests { "Multipart object should report the same full-object CRC64NVME as direct upload" ); } + + /// Integration test for the AWS 2026-04 additional checksum algorithms + /// (XXHash3/64/128, SHA-512, MD5). aws_sdk_s3 has no typed builder for these, so the + /// `x-amz-checksum-` header is injected via `mutate_request` (value computed by + /// rustfs-rio, which is byte-for-byte identical to awscrt). Verifies the server + /// verifies-on-write: a correct value is accepted and the object stored; a wrong + /// value is rejected with BadDigest and nothing is stored. Full HEAD/GET header + /// echo round-trip is additionally exercised by the boto3+awscrt e2e. + #[tokio::test] + #[serial] + async fn test_additional_checksums_verify_on_write() { + init_logging(); + info!("TEST: additional checksums (XXHash3/64/128, SHA-512, MD5) verify-on-write"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client_no_auto_checksum(&env); + let bucket = "test-additional-checksums"; + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + + let content: &[u8] = b"additional-checksum verify-on-write payload for xxhash/sha512/md5"; + + for (ty, header) in [ + (RioChecksumType::XXHASH3, "x-amz-checksum-xxhash3"), + (RioChecksumType::XXHASH64, "x-amz-checksum-xxhash64"), + (RioChecksumType::XXHASH128, "x-amz-checksum-xxhash128"), + (RioChecksumType::SHA512, "x-amz-checksum-sha512"), + (RioChecksumType::MD5, "x-amz-checksum-md5"), + ] { + // Correct checksum -> accepted, and the object is stored intact. + let good = Checksum::new_from_data(ty, content).expect("compute checksum").encoded; + let ok_key = format!("ok-{header}"); + let put = client + .put_object() + .bucket(bucket) + .key(&ok_key) + .body(ByteStream::from_static(content)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert(header, good.clone()); + }) + .send() + .await; + assert!(put.is_ok(), "{header}: correct checksum must be accepted: {:?}", put.err()); + let got = client + .get_object() + .bucket(bucket) + .key(&ok_key) + .send() + .await + .expect("GetObject"); + let body = got.body.collect().await.expect("collect body").into_bytes(); + assert_eq!(body.as_ref(), content, "{header}: stored body must match uploaded content"); + + // Wrong checksum -> rejected with BadDigest, and nothing is stored. + let bad = Checksum::new_from_data(ty, b"a totally different payload") + .expect("compute checksum") + .encoded; + let bad_key = format!("bad-{header}"); + let put_bad = client + .put_object() + .bucket(bucket) + .key(&bad_key) + .body(ByteStream::from_static(content)) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert(header, bad.clone()); + }) + .send() + .await; + assert!(put_bad.is_err(), "{header}: a mismatched checksum must be rejected"); + let msg = format!("{:?}", put_bad.err().unwrap()); + assert!( + msg.contains("BadDigest") || msg.to_lowercase().contains("digest") || msg.to_lowercase().contains("checksum"), + "{header}: expected a BadDigest/checksum error, got: {msg}" + ); + let head = client.head_object().bucket(bucket).key(&bad_key).send().await; + assert!(head.is_err(), "{header}: nothing must be stored after a rejected PutObject"); + + info!("PASSED additional-checksum verify-on-write: {header}"); + } + } } diff --git a/crates/ecstore/src/client/checksum.rs b/crates/ecstore/src/client/checksum.rs index 49463025c..228e7e16f 100644 --- a/crates/ecstore/src/client/checksum.rs +++ b/crates/ecstore/src/client/checksum.rs @@ -71,7 +71,11 @@ impl ChecksumMode { 8_u8 => ChecksumMode::ChecksumCRC32, 16_u8 => ChecksumMode::ChecksumCRC32C, 32_u8 => ChecksumMode::ChecksumCRC64NVME, - _ => panic!("enum err."), + // Fail closed: any mode without a concrete base algorithm (e.g. a + // bare ChecksumFullObject flag) is treated as "no checksum" rather + // than panicking. Callers already gate real work behind + // is_set()/can_composite()/hasher(), so this only removes a crash. + _ => ChecksumMode::ChecksumNone, } } @@ -272,6 +276,33 @@ impl ChecksumMode { } } +#[cfg(test)] +mod tests { + use super::ChecksumMode; + + #[test] + fn test_base_is_fail_closed_and_never_panics() { + // Every mode must resolve to a concrete base without panicking. The bare + // ChecksumFullObject flag has no base algorithm and must fall back to + // ChecksumNone instead of crashing (previously `panic!("enum err.")`). + assert_eq!(ChecksumMode::ChecksumFullObject.base(), ChecksumMode::ChecksumNone); + assert_eq!(ChecksumMode::ChecksumNone.base(), ChecksumMode::ChecksumNone); + assert_eq!(ChecksumMode::ChecksumCRC32.base(), ChecksumMode::ChecksumCRC32); + assert_eq!(ChecksumMode::ChecksumCRC32C.base(), ChecksumMode::ChecksumCRC32C); + assert_eq!(ChecksumMode::ChecksumSHA1.base(), ChecksumMode::ChecksumSHA1); + assert_eq!(ChecksumMode::ChecksumSHA256.base(), ChecksumMode::ChecksumSHA256); + assert_eq!(ChecksumMode::ChecksumCRC64NVME.base(), ChecksumMode::ChecksumCRC64NVME); + } + + #[test] + fn test_hasher_fails_closed_for_unsupported_mode() { + // Modes without a real hasher must return an error, not panic. + assert!(ChecksumMode::ChecksumNone.hasher().is_err()); + assert!(ChecksumMode::ChecksumFullObject.hasher().is_err()); + assert!(ChecksumMode::ChecksumCRC32.hasher().is_ok()); + } +} + #[derive(Default)] pub struct Checksum { checksum_type: ChecksumMode, diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index edbc15044..7ae1c79fd 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -3518,6 +3518,13 @@ fn complete_part_checksum(part: &CompletePart, checksum_type: rustfs_rio::Checks rustfs_rio::ChecksumType::CRC32 => Some(part.checksum_crc32.clone()), rustfs_rio::ChecksumType::CRC32C => Some(part.checksum_crc32c.clone()), rustfs_rio::ChecksumType::CRC64_NVME => Some(part.checksum_crc64nvme.clone()), + // XXHash3/64/128 and SHA-512 (AWS 2026-04): s3s CompletePart has no typed + // field to carry a client-supplied per-part value in the + // CompleteMultipartUpload request, so accept the type with no double-check + // value (the part was already verified server-side at UploadPart). This + // mirrors the missing-value path of the five typed algorithms. Reject only + // genuinely unset/invalid types. + base if base.is_set() => Some(None), _ => None, } } @@ -6718,6 +6725,23 @@ mod tests { ..Default::default() }; assert_eq!(complete_part_checksum(&part, full_object_crc32), Some(Some("AAAAAA==".to_string()))); + + // AWS 2026-04 additional algorithms have no CompletePart field, so they are + // accepted with no double-check value (verified at UploadPart) — Some(None), + // NOT the outer None that would fail the multipart completion (#1261). + for ct in [ + rustfs_rio::ChecksumType::XXHASH3, + rustfs_rio::ChecksumType::XXHASH64, + rustfs_rio::ChecksumType::XXHASH128, + rustfs_rio::ChecksumType::SHA512, + rustfs_rio::ChecksumType::MD5, + ] { + assert_eq!(complete_part_checksum(&CompletePart::default(), ct), Some(None), "{ct:?}"); + } + + // Genuinely unset / invalid types must still be rejected (outer None). + assert_eq!(complete_part_checksum(&CompletePart::default(), rustfs_rio::ChecksumType::NONE), None); + assert_eq!(complete_part_checksum(&CompletePart::default(), rustfs_rio::ChecksumType::INVALID), None); } fn direct_memory_test_metadata(size: i64) -> (ObjectInfo, FileInfo, ObjectOptions) { diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index b75e13297..e061ae8e5 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -59,6 +59,7 @@ thiserror.workspace = true base64.workspace = true sha1.workspace = true sha2.workspace = true +xxhash-rust = { workspace = true } s3s.workspace = true hex-simd.workspace = true diff --git a/crates/rio/src/checksum.rs b/crates/rio/src/checksum.rs index 86867cc15..cecefd7ae 100644 --- a/crates/rio/src/checksum.rs +++ b/crates/rio/src/checksum.rs @@ -17,7 +17,7 @@ use base64::{Engine as _, engine::general_purpose}; use bytes::Bytes; use http::HeaderMap; use sha1::Sha1; -use sha2::{Digest, Sha256}; +use sha2::{Digest, Sha256, Sha512}; use std::collections::HashMap; use std::io::Write; @@ -64,10 +64,33 @@ impl ChecksumType { /// Full object checksum pub const FULL_OBJECT: ChecksumType = ChecksumType(1 << 9); + // --- S3 additional checksum algorithms (AWS 2026-04). New base-type bits are + // append-only above bit 9; existing bits must never be renumbered because the + // raw `checksum_type.0` value is varint-serialized into xl.meta (see append_to). + + /// XXHash3 (64-bit) checksum. COMPOSITE-only per AWS. + pub const XXHASH3: ChecksumType = ChecksumType(1 << 10); + + /// XXHash64 checksum. + pub const XXHASH64: ChecksumType = ChecksumType(1 << 11); + + /// XXHash128 checksum. + pub const XXHASH128: ChecksumType = ChecksumType(1 << 12); + + /// SHA-512 checksum. + pub const SHA512: ChecksumType = ChecksumType(1 << 13); + + /// MD5 as an ADDITIONAL checksum (x-amz-checksum-md5), distinct from the legacy + /// Content-MD5 / ETag path. + pub const MD5: ChecksumType = ChecksumType(1 << 14); + /// No checksum pub const NONE: ChecksumType = ChecksumType(0); - const BASE_TYPE_MASK: u32 = Self::SHA256.0 | Self::SHA1.0 | Self::CRC32.0 | Self::CRC32C.0 | Self::CRC64_NVME.0; + /// Base-type mask, derived from [`BASE_CHECKSUM_TYPES`] as the single source of + /// truth. A new base type added to that list is automatically covered here, so + /// `base()` can never silently strip a wired-up algorithm (see #1252 / #1254). + const BASE_TYPE_MASK: u32 = compute_base_type_mask(BASE_CHECKSUM_TYPES); /// Check if this checksum type has all flags of the given type pub fn is(self, t: ChecksumType) -> bool { @@ -96,6 +119,11 @@ impl ChecksumType { Self::SHA1 => Some("x-amz-checksum-sha1"), Self::SHA256 => Some("x-amz-checksum-sha256"), Self::CRC64_NVME => Some("x-amz-checksum-crc64nvme"), + Self::XXHASH3 => Some("x-amz-checksum-xxhash3"), + Self::XXHASH64 => Some("x-amz-checksum-xxhash64"), + Self::XXHASH128 => Some("x-amz-checksum-xxhash128"), + Self::SHA512 => Some("x-amz-checksum-sha512"), + Self::MD5 => Some("x-amz-checksum-md5"), _ => None, } } @@ -107,6 +135,10 @@ impl ChecksumType { Self::SHA1 => 20, Self::SHA256 => SHA256_SIZE, Self::CRC64_NVME => 8, + Self::XXHASH3 | Self::XXHASH64 => 8, + Self::XXHASH128 => 16, + Self::SHA512 => 64, + Self::MD5 => 16, _ => 0, } } @@ -129,6 +161,11 @@ impl ChecksumType { Self::SHA1 => Some(Box::new(Sha1Hasher::new())), Self::SHA256 => Some(Box::new(Sha256Hasher::new())), Self::CRC64_NVME => Some(Box::new(Crc64NvmeHasher::new())), + Self::XXHASH3 => Some(Box::new(Xxh3Hasher::new())), + Self::XXHASH64 => Some(Box::new(Xxh64Hasher::new())), + Self::XXHASH128 => Some(Box::new(Xxh128Hasher::new())), + Self::SHA512 => Some(Box::new(Sha512Hasher::new())), + Self::MD5 => Some(Box::new(Md5Hasher::new())), _ => None, } } @@ -143,6 +180,14 @@ impl ChecksumType { (self.0 & Self::FULL_OBJECT.0) == Self::FULL_OBJECT.0 || self.is(Self::CRC64_NVME) } + /// True for the five algorithms s3s exposes as typed `*Output` fields + /// (CRC32/CRC32C/SHA1/SHA256/CRC64NVME). The AWS 2026-04 additional algorithms + /// (XXHash3/64/128, SHA-512, MD5) have no typed field and are carried as raw + /// response headers instead. Single source of truth for that split. + pub fn is_s3s_typed(self) -> bool { + matches!(self.base(), Self::CRC32 | Self::CRC32C | Self::SHA1 | Self::SHA256 | Self::CRC64_NVME) + } + /// Get object type string for x-amz-checksum-type header pub fn obj_type(self) -> &'static str { if self.full_object_requested() { @@ -177,32 +222,37 @@ impl ChecksumType { _ => return Self::INVALID, }; - match alg.to_uppercase().as_str() { - "CRC32" => ChecksumType(Self::CRC32.0 | full.0), - "CRC32C" => ChecksumType(Self::CRC32C.0 | full.0), - "SHA1" => { - if full != Self::NONE { - return Self::INVALID; - } - Self::SHA1 - } - "SHA256" => { - if full != Self::NONE { - return Self::INVALID; - } - Self::SHA256 - } - "CRC64NVME" => { - // AWS seems to ignore full value and just assume it - Self::CRC64_NVME - } - "" => { - if full != Self::NONE { - return Self::INVALID; - } - Self::NONE - } - _ => Self::INVALID, + // Case-insensitive matching WITHOUT allocating: to_uppercase() allocated a + // String on every checksummed request, and this path is hot. Composite-only + // algorithms reject an explicit FULL_OBJECT request — they cannot be linearly + // combined into a full-object checksum the way CRCs can (same rule as SHA1/256). + let composite_only = |ty: ChecksumType| -> ChecksumType { if full != Self::NONE { Self::INVALID } else { ty } }; + + if alg.eq_ignore_ascii_case("CRC32") { + ChecksumType(Self::CRC32.0 | full.0) + } else if alg.eq_ignore_ascii_case("CRC32C") { + ChecksumType(Self::CRC32C.0 | full.0) + } else if alg.eq_ignore_ascii_case("CRC64NVME") { + // AWS ignores the full-object flag here and just assumes it. + Self::CRC64_NVME + } else if alg.eq_ignore_ascii_case("SHA1") { + composite_only(Self::SHA1) + } else if alg.eq_ignore_ascii_case("SHA256") { + composite_only(Self::SHA256) + } else if alg.eq_ignore_ascii_case("SHA512") { + composite_only(Self::SHA512) + } else if alg.eq_ignore_ascii_case("XXHASH3") { + composite_only(Self::XXHASH3) + } else if alg.eq_ignore_ascii_case("XXHASH64") { + composite_only(Self::XXHASH64) + } else if alg.eq_ignore_ascii_case("XXHASH128") { + composite_only(Self::XXHASH128) + } else if alg.eq_ignore_ascii_case("MD5") { + composite_only(Self::MD5) + } else if alg.is_empty() { + composite_only(Self::NONE) + } else { + Self::INVALID } } } @@ -215,21 +265,45 @@ impl std::fmt::Display for ChecksumType { Self::SHA1 => write!(f, "SHA1"), Self::SHA256 => write!(f, "SHA256"), Self::CRC64_NVME => write!(f, "CRC64NVME"), + Self::XXHASH3 => write!(f, "XXHASH3"), + Self::XXHASH64 => write!(f, "XXHASH64"), + Self::XXHASH128 => write!(f, "XXHASH128"), + Self::SHA512 => write!(f, "SHA512"), + Self::MD5 => write!(f, "MD5"), Self::NONE => write!(f, ""), _ => write!(f, "invalid"), } } } -/// Base checksum types list +/// Base checksum types list. This is the single source of truth for which base +/// algorithms exist: [`ChecksumType::BASE_TYPE_MASK`] is derived from it, so adding +/// a new algorithm here cannot leave `base()` silently stripping it (see #1254). pub const BASE_CHECKSUM_TYPES: &[ChecksumType] = &[ ChecksumType::SHA256, ChecksumType::SHA1, ChecksumType::CRC32, ChecksumType::CRC64_NVME, ChecksumType::CRC32C, + ChecksumType::XXHASH3, + ChecksumType::XXHASH64, + ChecksumType::XXHASH128, + ChecksumType::SHA512, + ChecksumType::MD5, ]; +/// Fold the base-type bits of `types` into a mask. Used to derive +/// [`ChecksumType::BASE_TYPE_MASK`] from [`BASE_CHECKSUM_TYPES`] at compile time. +const fn compute_base_type_mask(types: &[ChecksumType]) -> u32 { + let mut mask = 0u32; + let mut i = 0; + while i < types.len() { + mask |= types[i].0; + i += 1; + } + mask +} + /// Checksum structure containing type and encoded value #[derive(Debug, Clone, PartialEq, Default)] pub struct Checksum { @@ -824,6 +898,208 @@ impl ChecksumHasher for Crc64NvmeHasher { } } +/// XXHash3 (64-bit) hasher, seed 0. Digest encoded big-endian (S3 canonical form, +/// matching the AWS CRT wire representation). +pub struct Xxh3Hasher { + hasher: xxhash_rust::xxh3::Xxh3, +} + +impl Default for Xxh3Hasher { + fn default() -> Self { + Self::new() + } +} + +impl Xxh3Hasher { + pub fn new() -> Self { + Self { + hasher: xxhash_rust::xxh3::Xxh3::new(), + } + } +} + +impl Write for Xxh3Hasher { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.hasher.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl ChecksumHasher for Xxh3Hasher { + fn finalize(&mut self) -> Vec { + self.hasher.digest().to_be_bytes().to_vec() + } + + fn reset(&mut self) { + self.hasher = xxhash_rust::xxh3::Xxh3::new(); + } +} + +/// XXHash128 hasher, seed 0. Digest encoded big-endian over the full 128 bits. +pub struct Xxh128Hasher { + hasher: xxhash_rust::xxh3::Xxh3, +} + +impl Default for Xxh128Hasher { + fn default() -> Self { + Self::new() + } +} + +impl Xxh128Hasher { + pub fn new() -> Self { + Self { + hasher: xxhash_rust::xxh3::Xxh3::new(), + } + } +} + +impl Write for Xxh128Hasher { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.hasher.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl ChecksumHasher for Xxh128Hasher { + fn finalize(&mut self) -> Vec { + self.hasher.digest128().to_be_bytes().to_vec() + } + + fn reset(&mut self) { + self.hasher = xxhash_rust::xxh3::Xxh3::new(); + } +} + +/// XXHash64 hasher, seed 0. Digest encoded big-endian. +pub struct Xxh64Hasher { + hasher: xxhash_rust::xxh64::Xxh64, +} + +impl Default for Xxh64Hasher { + fn default() -> Self { + Self::new() + } +} + +impl Xxh64Hasher { + pub fn new() -> Self { + Self { + hasher: xxhash_rust::xxh64::Xxh64::new(0), + } + } +} + +impl Write for Xxh64Hasher { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.hasher.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl ChecksumHasher for Xxh64Hasher { + fn finalize(&mut self) -> Vec { + self.hasher.digest().to_be_bytes().to_vec() + } + + fn reset(&mut self) { + self.hasher = xxhash_rust::xxh64::Xxh64::new(0); + } +} + +/// SHA-512 hasher. +pub struct Sha512Hasher { + hasher: Sha512, +} + +impl Default for Sha512Hasher { + fn default() -> Self { + Self::new() + } +} + +impl Sha512Hasher { + pub fn new() -> Self { + Self { hasher: Sha512::new() } + } +} + +impl Write for Sha512Hasher { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.hasher.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl ChecksumHasher for Sha512Hasher { + fn finalize(&mut self) -> Vec { + self.hasher.clone().finalize().to_vec() + } + + fn reset(&mut self) { + self.hasher = Sha512::new(); + } +} + +/// MD5 hasher for the ADDITIONAL checksum (x-amz-checksum-md5). Separate from the +/// legacy Content-MD5 / ETag machinery — this only serves the flexible-checksum path. +pub struct Md5Hasher { + hasher: md5::Md5, +} + +impl Default for Md5Hasher { + fn default() -> Self { + Self::new() + } +} + +impl Md5Hasher { + pub fn new() -> Self { + use md5::Digest as _; + Self { hasher: md5::Md5::new() } + } +} + +impl Write for Md5Hasher { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + use md5::Digest as _; + self.hasher.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl ChecksumHasher for Md5Hasher { + fn finalize(&mut self) -> Vec { + use md5::Digest as _; + self.hasher.clone().finalize().to_vec() + } + + fn reset(&mut self) { + use md5::Digest as _; + self.hasher = md5::Md5::new(); + } +} + /// Encode unsigned integer as varint fn encode_varint(buf: &mut Vec, mut value: u64) { while value >= 0x80 { @@ -1186,4 +1462,230 @@ mod tests { assert_eq!(combined.encoded, expected.encoded); assert_eq!(combined.raw, expected.raw); } + + // Guardrail (#1254): every base algorithm must round-trip through EVERY dispatch + // site. This fails loudly if a future algorithm is added to the enum but a match + // arm (base/is_set/hasher/key/raw_byte_len/Display/from_string) or BASE_TYPE_MASK + // is forgotten — the silent-drop footgun that reintroduced #4800. + #[test] + fn base_checksum_types_round_trip_all_dispatch_sites() { + use super::BASE_CHECKSUM_TYPES; + use std::io::Write; + + for &t in BASE_CHECKSUM_TYPES { + assert_eq!(t.base(), t, "base() stripped {t:?}; BASE_TYPE_MASK is missing its bit"); + assert!(t.is_set(), "{t:?} is not is_set()"); + assert!(t.hasher().is_some(), "{t:?} has no hasher()"); + assert!(t.key().is_some(), "{t:?} has no header key()"); + assert!(t.raw_byte_len() > 0, "{t:?} has zero raw_byte_len()"); + + let name = t.to_string(); + assert_ne!(name, "invalid", "{t:?} Display returns \"invalid\""); + assert_ne!(name, "", "{t:?} Display returns empty"); + assert_eq!( + ChecksumType::from_string(&name).base(), + t, + "from_string({name}) does not round-trip to {t:?}" + ); + + let mut hasher = t.hasher().expect("hasher present"); + hasher.write_all(b"rustfs checksum round-trip").expect("write"); + assert_eq!( + hasher.finalize().len(), + t.raw_byte_len(), + "{t:?} finalized digest length != raw_byte_len()" + ); + } + } + + // The new AWS 2026-04 algorithms are COMPOSITE-only: an explicit FULL_OBJECT + // request must be rejected (they cannot be linearly combined like CRCs), and they + // must never be routed through add_part()/can_merge(). + #[test] + fn new_algorithms_are_composite_only() { + for alg in ["XXHASH3", "XXHASH64", "XXHASH128", "SHA512", "MD5"] { + let composite = ChecksumType::from_string_with_obj_type(alg, "COMPOSITE"); + assert!(composite.is_set(), "{alg} COMPOSITE should be valid"); + assert!(!composite.can_merge(), "{alg} must not be mergeable (composite-only)"); + + let full = ChecksumType::from_string_with_obj_type(alg, "FULL_OBJECT"); + assert_eq!(full, ChecksumType::INVALID, "{alg} FULL_OBJECT must be rejected"); + } + } + + // Known-answer vectors (#1255). The empty-input digests are the OFFICIAL upstream + // xxHash / SHA-512 test vectors (seed 0), pinned in big-endian to guarantee the + // stored/echoed value byte-for-byte matches what AWS SDKs (awscrt) compute. If the + // finalize() byte order or seed ever drifts, these fail loudly. + fn raw_hex(t: ChecksumType, data: &[u8]) -> String { + let c = Checksum::new_from_data(t, data).expect("checksum"); + assert_eq!(c.raw.len(), t.raw_byte_len(), "raw len mismatch for {t:?}"); + c.raw.iter().map(|b| format!("{b:02x}")).collect() + } + + #[test] + fn xxhash_sha512_known_answer_vectors_empty_input() { + // XXH3-64("") = 0x2D06800538D394C2 (official) + assert_eq!(raw_hex(ChecksumType::XXHASH3, b""), "2d06800538d394c2"); + // XXH64("") = 0xEF46DB3751D8E999 (official) + assert_eq!(raw_hex(ChecksumType::XXHASH64, b""), "ef46db3751d8e999"); + // XXH3-128("") = 0x99AA06D3014798D86001C324468D497F (official) + assert_eq!(raw_hex(ChecksumType::XXHASH128, b""), "99aa06d3014798d86001c324468d497f"); + // SHA-512("") (official) + assert_eq!( + raw_hex(ChecksumType::SHA512, b""), + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce\ + 47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" + ); + // MD5("") = d41d8cd98f00b204e9800998ecf8427e (official) + assert_eq!(raw_hex(ChecksumType::MD5, b""), "d41d8cd98f00b204e9800998ecf8427e"); + } + + // Regression lock for a non-empty payload: values are produced by this + // implementation and must stay stable across refactors. Base64 (S3 wire form) is + // asserted alongside the raw hex so both the digest and its encoding are pinned. + #[test] + fn xxhash_sha512_regression_lock_non_empty() { + use base64::{Engine as _, engine::general_purpose::STANDARD}; + let data = b"The quick brown fox jumps over the lazy dog"; + + // XXH3-64(fox) = 0xce7d19a5418fb365 is the official upstream vector. + for (t, want_hex) in [ + (ChecksumType::XXHASH3, "ce7d19a5418fb365"), + (ChecksumType::XXHASH64, "0b242d361fda71bc"), + ] { + let c = Checksum::new_from_data(t, data).expect("checksum"); + let got_hex: String = c.raw.iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(got_hex, want_hex, "{t:?} raw hex drifted"); + // encoded field must be the standard-base64 of raw (S3 wire form) + assert_eq!(c.encoded, STANDARD.encode(&c.raw), "{t:?} encoded field != base64(raw)"); + } + } + + // On-disk (xl.meta) serialization round-trip for the new algorithms (#1260): + // to_bytes() -> read_checksums() must recover the value under the Display key. + #[test] + fn on_disk_round_trip_new_algorithms() { + use super::read_checksums; + let data = b"rustfs on-disk checksum round-trip payload"; + + for (t, name) in [ + (ChecksumType::XXHASH3, "XXHASH3"), + (ChecksumType::XXHASH64, "XXHASH64"), + (ChecksumType::XXHASH128, "XXHASH128"), + (ChecksumType::SHA512, "SHA512"), + ] { + let c = Checksum::new_from_data(t, data).expect("checksum"); + let buf = c.to_bytes(&[]); + let (map, is_multipart) = read_checksums(&buf, 0); + assert!(!is_multipart, "{name} single-object must not be multipart"); + assert_eq!(map.get(name), Some(&c.encoded), "{name} did not round-trip on-disk"); + } + } + + // Forward-compat / rolling-upgrade contract (#1260): a node that does not know a + // future base-type bit must DEGRADE SAFELY — skip the entry and return without + // panicking or decoding a wrong length — never crash or corrupt. + #[test] + fn unknown_future_type_bit_degrades_safely() { + use super::{encode_varint, read_checksums, read_part_checksums}; + + // A base-type bit far above any allocated one (append-only rule guarantees + // real bits stay below this), followed by 8 bytes of would-be digest. + let mut buf = Vec::new(); + encode_varint(&mut buf, 1 << 20); + buf.extend_from_slice(&[0xAB; 8]); + + let (map, is_multipart) = read_checksums(&buf, 0); + assert!(map.is_empty(), "unknown type must yield no checksum, got {map:?}"); + assert!(!is_multipart); + assert!(read_part_checksums(&buf).is_empty()); + } + + // Multipart COMPOSITE assembly for the composite-only algorithms (#1261). Mirrors + // set_disk complete_multipart_upload: the object checksum is H(concat of per-part + // raw digests), and these algorithms must NOT be routed through add_part(). + #[test] + fn composite_multipart_assembly_new_algorithms() { + let part1 = b"first multipart chunk bytes"; + let part2 = b"second multipart chunk bytes"; + + for t in [ + ChecksumType::XXHASH3, + ChecksumType::XXHASH64, + ChecksumType::XXHASH128, + ChecksumType::SHA512, + ChecksumType::MD5, + ] { + let c1 = Checksum::new_from_data(t, part1).expect("part1"); + let c2 = Checksum::new_from_data(t, part2).expect("part2"); + + // Concatenate raw part digests, then hash again with the same algorithm. + let mut combined = c1.raw.clone(); + combined.extend_from_slice(&c2.raw); + let composite = Checksum::new_from_data(t, &combined).expect("composite"); + assert_eq!(composite.raw.len(), t.raw_byte_len(), "{t:?} composite len"); + assert!(!composite.encoded.is_empty()); + + // Composite-only: full-object merge must be refused. + let mut acc = Checksum { + checksum_type: t, + ..Default::default() + }; + assert!(acc.add_part(&c1, part1.len() as i64).is_err(), "{t:?} must not be full-object mergeable"); + } + } + + // S11: from_string_with_obj_type dropped to_uppercase() (a per-request heap alloc) + // for eq_ignore_ascii_case. Lock that this did not change any behaviour. + #[test] + fn from_string_is_case_insensitive_and_behaviour_preserved() { + assert_eq!(ChecksumType::from_string("crc32").base(), ChecksumType::CRC32); + assert_eq!(ChecksumType::from_string("Crc32C").base(), ChecksumType::CRC32C); + assert_eq!(ChecksumType::from_string("xxHASH3").base(), ChecksumType::XXHASH3); + assert_eq!(ChecksumType::from_string("Md5").base(), ChecksumType::MD5); + assert_eq!(ChecksumType::from_string("sha512").base(), ChecksumType::SHA512); + + // Unknown / empty preserved. + assert_eq!(ChecksumType::from_string("nope"), ChecksumType::INVALID); + assert_eq!(ChecksumType::from_string(""), ChecksumType::NONE); + + // CRC64NVME still assumes full-object; CRC32 still accepts explicit FULL_OBJECT. + assert!(ChecksumType::from_string("crc64nvme").full_object_requested()); + assert!(ChecksumType::from_string_with_obj_type("crc32", "FULL_OBJECT").full_object_requested()); + + // Composite-only algorithms still reject FULL_OBJECT; invalid obj_type still rejected. + assert_eq!(ChecksumType::from_string_with_obj_type("xxhash3", "FULL_OBJECT"), ChecksumType::INVALID); + assert_eq!(ChecksumType::from_string_with_obj_type("crc32", "bogus"), ChecksumType::INVALID); + } + + // is_s3s_typed is the single source of truth for the "five typed fields" vs + // "additional algorithms carried as raw headers" split used across the handlers. + #[test] + fn is_s3s_typed_split_is_exhaustive() { + for t in [ + ChecksumType::CRC32, + ChecksumType::CRC32C, + ChecksumType::SHA1, + ChecksumType::SHA256, + ChecksumType::CRC64_NVME, + ] { + assert!(t.is_s3s_typed(), "{t:?} must be s3s-typed"); + } + for t in [ + ChecksumType::XXHASH3, + ChecksumType::XXHASH64, + ChecksumType::XXHASH128, + ChecksumType::SHA512, + ChecksumType::MD5, + ] { + assert!(!t.is_s3s_typed(), "{t:?} must NOT be s3s-typed (additional algorithm)"); + } + assert!(!ChecksumType::NONE.is_s3s_typed()); + // Flags on top of a base type must not change the classification. + let crc32_full = ChecksumType(ChecksumType::CRC32.0 | ChecksumType::FULL_OBJECT.0); + assert!(crc32_full.is_s3s_typed()); + let xxh3_multipart = ChecksumType(ChecksumType::XXHASH3.0 | ChecksumType::MULTIPART.0); + assert!(!xxh3_multipart.is_s3s_typed()); + } } diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 593b8d56a..a31580fd1 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -73,7 +73,7 @@ use rustfs_s3_ops::S3Operation; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::{ - AMZ_CHECKSUM_TYPE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, get_source_scheme, + SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, get_source_scheme, headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING}, insert_str, }; @@ -535,33 +535,21 @@ impl DefaultMultipartUsecase { None }; let mpu_version_for_event = mpu_version.clone(); - let mut checksum_crc32 = input.checksum_crc32; - let mut checksum_crc32c = input.checksum_crc32c; - let mut checksum_sha1 = input.checksum_sha1; - let mut checksum_sha256 = input.checksum_sha256; - let mut checksum_crc64nvme = input.checksum_crc64nvme; - let mut checksum_type = input.checksum_type; - - // checksum + // checksum: stored (decrypted) values take precedence over the request input; + // additional algorithms (XXHash3/64/128, SHA-512, MD5), which have no typed + // CompleteMultipartUploadOutput field, are echoed as raw response headers (#1261). let (checksums, _is_multipart) = obj_info .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) .map_err(ApiError::from)?; - for (key, checksum) in checksums { - if key == AMZ_CHECKSUM_TYPE { - checksum_type = Some(ChecksumType::from(checksum)); - continue; - } - - match rustfs_rio::ChecksumType::from_string(key.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksum_crc32 = Some(checksum), - rustfs_rio::ChecksumType::CRC32C => checksum_crc32c = Some(checksum), - rustfs_rio::ChecksumType::SHA1 => checksum_sha1 = Some(checksum), - rustfs_rio::ChecksumType::SHA256 => checksum_sha256 = Some(checksum), - rustfs_rio::ChecksumType::CRC64_NVME => checksum_crc64nvme = Some(checksum), - _ => (), - } - } + let classified = crate::app::object_usecase::classify_response_checksums(checksums); + let checksum_crc32 = classified.crc32.or(input.checksum_crc32); + let checksum_crc32c = classified.crc32c.or(input.checksum_crc32c); + let checksum_sha1 = classified.sha1.or(input.checksum_sha1); + let checksum_sha256 = classified.sha256.or(input.checksum_sha256); + let checksum_crc64nvme = classified.crc64nvme.or(input.checksum_crc64nvme); + let checksum_type = classified.checksum_type.or(input.checksum_type); + let complete_extra_checksum_headers = classified.extra; let location = build_complete_multipart_location(&req.headers, &req.uri, &bucket, &key); let output = CompleteMultipartUploadOutput { @@ -596,7 +584,9 @@ impl DefaultMultipartUsecase { helper = helper.version_id(version_id.clone()); } - let result = Ok(S3Response::new(output)); + let mut response = S3Response::new(output); + crate::app::object_usecase::inject_additional_checksum_headers(&mut response.headers, &complete_extra_checksum_headers); + let result = Ok(response); let _ = helper.complete(&result); rustfs_scanner::record_dirty_usage_bucket(&bucket); result @@ -1003,6 +993,10 @@ impl DefaultMultipartUsecase { } } + // XXHash3/64/128 and SHA-512 have no typed UploadPartOutput field; echo the + // server-computed part checksum as a raw response header (#1261). + let upload_part_extra_checksum_headers = crate::app::object_usecase::additional_checksum_echo_pairs(&opts.want_checksum); + let output = UploadPartOutput { server_side_encryption: requested_sse, ssekms_key_id: requested_kms_key_id, @@ -1017,7 +1011,12 @@ impl DefaultMultipartUsecase { ..Default::default() }; - Ok(S3Response::new(output)) + let mut response = S3Response::new(output); + crate::app::object_usecase::inject_additional_checksum_headers( + &mut response.headers, + &upload_part_extra_checksum_headers, + ); + Ok(response) } pub async fn execute_list_multipart_uploads( diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index d58db269a..f32388fe2 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -373,6 +373,7 @@ struct GetObjectOutputContext { event_info: Option, response_content_length: i64, optimal_buffer_size: usize, + extra_checksum_headers: Vec<(&'static str, String)>, } enum GetObjectTimeoutStage { @@ -1672,6 +1673,40 @@ async fn maybe_enqueue_transition_immediate(obj_info: &ObjectInfo, src: LcEventS enqueue_transition_immediate(obj_info, src).await; } +/// Inject additional-checksum response headers (XXHash3/64/128, SHA-512) that s3s +/// cannot carry on its typed `*Output` structs. Centralized so that when s3s gains +/// typed fields for these algorithms, only this one function changes (fill the typed +/// field, drop the header insert) — and there is exactly one place that could ever +/// emit a duplicate header. Header names come from `ChecksumType::key()`, so they are +/// known-valid static strings. +pub(crate) fn inject_additional_checksum_headers(headers: &mut HeaderMap, pairs: &[(&'static str, String)]) { + for (name, value) in pairs { + match HeaderValue::from_str(value) { + Ok(header_value) => { + headers.insert(http::HeaderName::from_static(name), header_value); + } + Err(_) => warn!("Failed to parse {name} checksum header value; skipping"), + } + } +} + +/// Derive the response-header echo pairs for an additional-checksum algorithm +/// (XXHash3/64/128, SHA-512) from the server-computed content checksum, for +/// PutObject to echo back (#1256). Returns empty for the five s3s-typed algorithms +/// (they are echoed via typed fields) and when the value is not yet materialized +/// (e.g. a trailing checksum, whose value lands after the body — covered by e2e). +pub(crate) fn additional_checksum_echo_pairs(want: &Option) -> Vec<(&'static str, String)> { + let mut out = Vec::new(); + if let Some(cs) = want + && !cs.checksum_type.is_s3s_typed() + && !cs.encoded.is_empty() + && let Some(name) = cs.checksum_type.key() + { + out.push((name, cs.encoded.clone())); + } + out +} + /// Extract trailing-header checksum values, overriding the corresponding input fields. fn apply_trailing_checksums( algorithm: Option<&str>, @@ -1707,14 +1742,52 @@ fn apply_trailing_checksums( } } +/// Checksums resolved from stored (decrypted) metadata for a response. The five +/// s3s-typed algorithms fill named fields; the AWS 2026-04 additional algorithms +/// (XXHash3/64/128, SHA-512, MD5), which s3s has no typed `*Output` field for, land +/// in `extra` and are emitted as raw response headers via +/// [`inject_additional_checksum_headers`]. Built by [`classify_response_checksums`] +/// so the typed/extra split lives in exactly one place. #[derive(Default)] -struct GetObjectChecksums { - crc32: Option, - crc32c: Option, - sha1: Option, - sha256: Option, - crc64nvme: Option, - checksum_type: Option, +pub(crate) struct ResponseChecksums { + pub(crate) crc32: Option, + pub(crate) crc32c: Option, + pub(crate) sha1: Option, + pub(crate) sha256: Option, + pub(crate) crc64nvme: Option, + pub(crate) checksum_type: Option, + pub(crate) extra: Vec<(&'static str, String)>, +} + +/// Split decrypted checksum (key, value) pairs into the five s3s-typed fields and the +/// additional-algorithm `extra` headers. Single source of truth for every response +/// path (GetObject / HeadObject / GetObjectAttributes / CompleteMultipartUpload), +/// replacing what used to be five copies of this match loop. +pub(crate) fn classify_response_checksums(pairs: I) -> ResponseChecksums +where + I: IntoIterator, +{ + let mut c = ResponseChecksums::default(); + for (key, checksum) in pairs { + if key == AMZ_CHECKSUM_TYPE { + c.checksum_type = Some(ChecksumType::from(checksum)); + continue; + } + let ct = rustfs_rio::ChecksumType::from_string(key.as_str()); + match ct.base() { + rustfs_rio::ChecksumType::CRC32 => c.crc32 = Some(checksum), + rustfs_rio::ChecksumType::CRC32C => c.crc32c = Some(checksum), + rustfs_rio::ChecksumType::SHA1 => c.sha1 = Some(checksum), + rustfs_rio::ChecksumType::SHA256 => c.sha256 = Some(checksum), + rustfs_rio::ChecksumType::CRC64_NVME => c.crc64nvme = Some(checksum), + _ => { + if let Some(name) = ct.key() { + c.extra.push((name, checksum)); + } + } + } + } + c } #[derive(Default)] @@ -3037,9 +3110,7 @@ impl DefaultObjectUsecase { headers: &HeaderMap, part_number: Option, rs: Option<&HTTPRangeSpec>, - ) -> S3Result { - let mut checksums = GetObjectChecksums::default(); - + ) -> S3Result { if let Some(checksum_mode) = headers.get(AMZ_CHECKSUM_MODE) && checksum_mode.to_str().unwrap_or_default() == "ENABLED" && rs.is_none() @@ -3050,24 +3121,10 @@ impl DefaultObjectUsecase { ApiError::from(e) })?; - for (key, checksum) in decrypted_checksums { - if key == AMZ_CHECKSUM_TYPE { - checksums.checksum_type = Some(ChecksumType::from(checksum)); - continue; - } - - match rustfs_rio::ChecksumType::from_string(key.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksums.crc32 = Some(checksum), - rustfs_rio::ChecksumType::CRC32C => checksums.crc32c = Some(checksum), - rustfs_rio::ChecksumType::SHA1 => checksums.sha1 = Some(checksum), - rustfs_rio::ChecksumType::SHA256 => checksums.sha256 = Some(checksum), - rustfs_rio::ChecksumType::CRC64_NVME => checksums.crc64nvme = Some(checksum), - _ => (), - } - } + return Ok(classify_response_checksums(decrypted_checksums)); } - Ok(checksums) + Ok(ResponseChecksums::default()) } #[allow(clippy::too_many_arguments)] async fn build_get_object_body( @@ -3707,6 +3764,9 @@ impl DefaultObjectUsecase { let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query()); let mut write_plan = WritePlan::new(); + // Additional-checksum (XXHash3/64/128, SHA-512) values to echo on the PutObject + // response (#1256); captured at want_checksum set points before opts is moved. + let mut put_extra_checksum_headers: Vec<(&'static str, String)> = Vec::new(); let mut reader = if should_compress { let body = tokio::io::BufReader::with_capacity( buffer_size, @@ -3724,6 +3784,7 @@ impl DefaultObjectUsecase { } opts.want_checksum = hrd.checksum(); + put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum); insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, compression_metadata_value(algorithm)); insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); @@ -3773,6 +3834,7 @@ impl DefaultObjectUsecase { } opts.want_checksum = reader.checksum(); + put_extra_checksum_headers = additional_checksum_echo_pairs(&opts.want_checksum); } rustfs_io_metrics::record_put_object_path(put_path); rustfs_io_metrics::record_put_object_stage_duration( @@ -4006,7 +4068,11 @@ impl DefaultObjectUsecase { // For browser-based POST uploads (multipart/form-data), response status/body handling // is decided by s3s PostObject serializer (success_action_status / redirect semantics). - let result = Ok(S3Response::new(output)); + let mut response = S3Response::new(output); + // Echo XXHash3/64/128 / SHA-512 checksums that s3s PutObjectOutput has no typed + // field for (#1256). + inject_additional_checksum_headers(&mut response.headers, &put_extra_checksum_headers); + let result = Ok(response); let _ = helper.complete(&result); rustfs_scanner::record_dirty_usage_bucket(&bucket); @@ -4121,6 +4187,7 @@ impl DefaultObjectUsecase { } } + #[allow(clippy::too_many_arguments)] async fn finalize_get_object_response( helper: OperationHelper, bucket: &str, @@ -4129,13 +4196,17 @@ impl DefaultObjectUsecase { event_info: Option, version_id_for_event: String, output: GetObjectOutput, + extra_checksum_headers: Vec<(&'static str, String)>, ) -> S3Result> { let helper = match event_info { Some(event_info) => helper.object(event_info), None => helper, }; let helper = helper.version_id(version_id_for_event); - let response = wrap_response_with_cors(bucket, method, headers, output).await; + let mut response = wrap_response_with_cors(bucket, method, headers, output).await; + // Emit XXHash3/64/128 and SHA-512 checksums that s3s GetObjectOutput cannot + // carry (#1257). This is the download-side integrity path AWS SDKs verify. + inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers); let result = Ok(response); let _ = helper.complete(&result); result @@ -4280,6 +4351,7 @@ impl DefaultObjectUsecase { event_info, response_content_length, optimal_buffer_size, + extra_checksum_headers: checksums.extra, }) } @@ -4444,6 +4516,7 @@ impl DefaultObjectUsecase { event_info, response_content_length, optimal_buffer_size, + extra_checksum_headers, } = output_context; let total_duration = request_start.elapsed(); @@ -4455,8 +4528,17 @@ impl DefaultObjectUsecase { optimal_buffer_size, ); - Self::finalize_get_object_response(helper, &bucket, &req.method, &req.headers, event_info, version_id_for_event, output) - .await + Self::finalize_get_object_response( + helper, + &bucket, + &req.method, + &req.headers, + event_info, + version_id_for_event, + output, + extra_checksum_headers, + ) + .await } pub async fn execute_get_object_attributes( @@ -4546,27 +4628,19 @@ impl DefaultObjectUsecase { let checksum = if requested(ObjectAttributes::CHECKSUM) { let (checksums, _is_multipart) = info.decrypt_checksums(0, &req.headers).map_err(ApiError::from)?; - let mut checksum_crc32 = None; - let mut checksum_crc32c = None; - let mut checksum_sha1 = None; - let mut checksum_sha256 = None; - let mut checksum_crc64nvme = None; - let mut checksum_type = None; - - for (k, v) in checksums { - if k == AMZ_CHECKSUM_TYPE { - checksum_type = Some(ChecksumType::from(v)); - continue; - } - match rustfs_rio::ChecksumType::from_string(k.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksum_crc32 = Some(v), - rustfs_rio::ChecksumType::CRC32C => checksum_crc32c = Some(v), - rustfs_rio::ChecksumType::SHA1 => checksum_sha1 = Some(v), - rustfs_rio::ChecksumType::SHA256 => checksum_sha256 = Some(v), - rustfs_rio::ChecksumType::CRC64_NVME => checksum_crc64nvme = Some(v), - _ => (), - } - } + // GetObjectAttributes returns checksums in the XML body, and s3s's Checksum + // type has no field for the additional algorithms, so `extra` cannot be + // surfaced here (unlike the header-based GET/HEAD paths) — an s3s limitation + // tracked for when it gains typed fields. + let ResponseChecksums { + crc32: checksum_crc32, + crc32c: checksum_crc32c, + sha1: checksum_sha1, + sha256: checksum_sha256, + crc64nvme: checksum_crc64nvme, + checksum_type, + .. + } = classify_response_checksums(checksums); Some(Checksum { checksum_crc32, @@ -4602,22 +4676,16 @@ impl DefaultObjectUsecase { for part in &info.parts[start_at..end] { let (checksums, _is_multipart) = info.decrypt_checksums(part.number, &req.headers).map_err(ApiError::from)?; - let mut checksum_crc32 = None; - let mut checksum_crc32c = None; - let mut checksum_sha1 = None; - let mut checksum_sha256 = None; - let mut checksum_crc64nvme = None; - - for (k, v) in checksums { - match rustfs_rio::ChecksumType::from_string(k.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksum_crc32 = Some(v), - rustfs_rio::ChecksumType::CRC32C => checksum_crc32c = Some(v), - rustfs_rio::ChecksumType::SHA1 => checksum_sha1 = Some(v), - rustfs_rio::ChecksumType::SHA256 => checksum_sha256 = Some(v), - rustfs_rio::ChecksumType::CRC64_NVME => checksum_crc64nvme = Some(v), - _ => (), - } - } + // Additional algorithms cannot be surfaced in the ObjectPart XML body + // (s3s has no field); same limitation as the object-level attributes above. + let ResponseChecksums { + crc32: checksum_crc32, + crc32c: checksum_crc32c, + sha1: checksum_sha1, + sha256: checksum_sha256, + crc64nvme: checksum_crc64nvme, + .. + } = classify_response_checksums(checksums); let part_size = if part.actual_size > 0 { part.actual_size @@ -5908,38 +5976,27 @@ impl DefaultObjectUsecase { let sse_customer_key_md5 = metadata_map.get("x-amz-server-side-encryption-customer-key-md5").cloned(); let sse_kms_key_id = metadata_map.get("x-amz-server-side-encryption-aws-kms-key-id").cloned(); let storage_class = response_storage_class(&info, &metadata_map); - let mut checksum_crc32 = None; - let mut checksum_crc32c = None; - let mut checksum_sha1 = None; - let mut checksum_sha256 = None; - let mut checksum_crc64nvme = None; - let mut checksum_type = None; - - // checksum - if let Some(checksum_mode) = req.headers.get(AMZ_CHECKSUM_MODE) + // checksum: classify once; additional algorithms (XXHash3/64/128, SHA-512, MD5) + // land in `extra` and are emitted as raw headers below (s3s has no typed field). + let ResponseChecksums { + crc32: checksum_crc32, + crc32c: checksum_crc32c, + sha1: checksum_sha1, + sha256: checksum_sha256, + crc64nvme: checksum_crc64nvme, + checksum_type, + extra: extra_checksum_headers, + } = if let Some(checksum_mode) = req.headers.get(AMZ_CHECKSUM_MODE) && checksum_mode.to_str().unwrap_or_default() == "ENABLED" && rs.is_none() { let (checksums, _is_multipart) = info .decrypt_checksums(opts.part_number.unwrap_or(0), &req.headers) .map_err(ApiError::from)?; - - for (key, checksum) in checksums { - if key == AMZ_CHECKSUM_TYPE { - checksum_type = Some(ChecksumType::from(checksum)); - continue; - } - - match rustfs_rio::ChecksumType::from_string(key.as_str()) { - rustfs_rio::ChecksumType::CRC32 => checksum_crc32 = Some(checksum), - rustfs_rio::ChecksumType::CRC32C => checksum_crc32c = Some(checksum), - rustfs_rio::ChecksumType::SHA1 => checksum_sha1 = Some(checksum), - rustfs_rio::ChecksumType::SHA256 => checksum_sha256 = Some(checksum), - rustfs_rio::ChecksumType::CRC64_NVME => checksum_crc64nvme = Some(checksum), - _ => (), - } - } - } + classify_response_checksums(checksums) + } else { + ResponseChecksums::default() + }; // Extract standard HTTP headers from user_defined metadata // Note: These headers are stored with lowercase keys by extract_metadata_from_mime let cache_control = metadata_map.get("cache-control").cloned(); @@ -6006,6 +6063,10 @@ impl DefaultObjectUsecase { // takes precedence for these read operations. let mut response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; + // Emit additional-checksum headers (XXHash3/64/128, SHA-512) that s3s cannot + // carry on the typed HeadObjectOutput (#1257). + inject_additional_checksum_headers(&mut response.headers, &extra_checksum_headers); + // Add x-amz-tagging-count header if object has tags // Per S3 API spec, this header should be present in HEAD object response when tags exist if tag_count > 0 { @@ -6821,6 +6882,97 @@ mod tests { assert_eq!(throttle.claim(IO_QUEUE_CONGESTION_WARN_INTERVAL_MS + 1), None); } + // classify_response_checksums is the single point that splits decrypted checksum + // pairs into the five s3s-typed fields and the additional-algorithm `extra` + // headers, replacing five copies of the loop. Lock its behaviour (#1252). + #[test] + fn classify_response_checksums_splits_typed_and_extra() { + // Typed algorithms fill named fields; nothing spills into extra. + let c = classify_response_checksums(vec![ + ("CRC32".to_string(), "AAAAAA==".to_string()), + ("SHA256".to_string(), "c2hhMjU2".to_string()), + ("CRC64NVME".to_string(), "Zm9vYmFyCg==".to_string()), + ]); + assert_eq!(c.crc32.as_deref(), Some("AAAAAA==")); + assert_eq!(c.sha256.as_deref(), Some("c2hhMjU2")); + assert_eq!(c.crc64nvme.as_deref(), Some("Zm9vYmFyCg==")); + assert!(c.extra.is_empty(), "typed algorithms must not land in extra"); + + // Additional algorithms land in extra keyed by their response-header name. + let c = classify_response_checksums(vec![ + ("XXHASH3".to_string(), "eHhoMw==".to_string()), + ("XXHASH64".to_string(), "eHhoNjQ=".to_string()), + ("XXHASH128".to_string(), "eHhoMTI4".to_string()), + ("SHA512".to_string(), "c2hhNTEy".to_string()), + ("MD5".to_string(), "bWQ1".to_string()), + ]); + assert!(c.crc32.is_none() && c.sha256.is_none() && c.crc64nvme.is_none()); + let names: Vec<&str> = c.extra.iter().map(|(n, _)| *n).collect(); + for expected in [ + "x-amz-checksum-xxhash3", + "x-amz-checksum-xxhash64", + "x-amz-checksum-xxhash128", + "x-amz-checksum-sha512", + "x-amz-checksum-md5", + ] { + assert!(names.contains(&expected), "extra missing {expected}: {names:?}"); + } + assert_eq!(c.extra.len(), 5); + + // The checksum-type marker is captured as the type, not mistaken for an algorithm. + let c = classify_response_checksums(vec![(AMZ_CHECKSUM_TYPE.to_string(), "COMPOSITE".to_string())]); + assert!(c.checksum_type.is_some()); + assert!(c.extra.is_empty() && c.crc32.is_none()); + + // Empty input yields an all-default result. + let c = classify_response_checksums(Vec::<(String, String)>::new()); + assert!(c.crc32.is_none() && c.extra.is_empty() && c.checksum_type.is_none()); + } + + // additional_checksum_echo_pairs derives the PutObject/UploadPart response echo for + // additional algorithms from the server-computed checksum, and nothing for the + // five typed ones (those go through typed output fields). + #[test] + fn additional_checksum_echo_pairs_only_for_new_algorithms() { + // Typed algorithm → no echo pair. + let sha256 = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::SHA256, b"data"); + assert!(additional_checksum_echo_pairs(&sha256).is_empty()); + + // Additional algorithm → exactly one (header, value) pair matching the digest. + let xxh3 = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::XXHASH3, b"data"); + let pairs = additional_checksum_echo_pairs(&xxh3); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].0, "x-amz-checksum-xxhash3"); + assert_eq!(pairs[0].1, xxh3.as_ref().unwrap().encoded); + + // MD5 additional checksum is echoed too. + let md5 = rustfs_rio::Checksum::new_from_data(rustfs_rio::ChecksumType::MD5, b"data"); + let pairs = additional_checksum_echo_pairs(&md5); + assert_eq!(pairs.len(), 1); + assert_eq!(pairs[0].0, "x-amz-checksum-md5"); + + // None → empty. + assert!(additional_checksum_echo_pairs(&None).is_empty()); + } + + #[test] + fn inject_additional_checksum_headers_writes_all_pairs() { + let mut headers = HeaderMap::new(); + inject_additional_checksum_headers( + &mut headers, + &[ + ("x-amz-checksum-xxhash3", "eHhoMw==".to_string()), + ("x-amz-checksum-md5", "bWQ1".to_string()), + ], + ); + assert_eq!(headers.get("x-amz-checksum-xxhash3").unwrap(), "eHhoMw=="); + assert_eq!(headers.get("x-amz-checksum-md5").unwrap(), "bWQ1"); + // Empty input is a no-op. + let mut empty = HeaderMap::new(); + inject_additional_checksum_headers(&mut empty, &[]); + assert!(empty.is_empty()); + } + fn build_request(input: T, method: Method) -> S3Request { S3Request { input,