diff --git a/Cargo.lock b/Cargo.lock index b3731fc4c..0b91a2e20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3105,6 +3105,7 @@ dependencies = [ "rustfs-lock", "rustfs-madmin", "rustfs-protos", + "rustfs-rio", "rustfs-signer", "rustls", "s3s", diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index 25c6ca0e9..bc4d27721 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -30,6 +30,7 @@ ftps = [] [dependencies] rustfs-ecstore.workspace = true rustfs-common.workspace = true +rustfs-rio.workspace = true flatbuffers.workspace = true futures.workspace = true rustfs-lock.workspace = true diff --git a/crates/e2e_test/src/checksum_upload_test.rs b/crates/e2e_test/src/checksum_upload_test.rs index 68a59bec6..6be69df85 100644 --- a/crates/e2e_test/src/checksum_upload_test.rs +++ b/crates/e2e_test/src/checksum_upload_test.rs @@ -20,8 +20,9 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; + use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart}; use base64::Engine; + use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; use serial_test::serial; use sha2::{Digest, Sha256}; use tracing::info; @@ -57,6 +58,12 @@ mod tests { base64::engine::general_purpose::STANDARD.encode(digest.as_slice()) } + fn checksum_crc64nvme_base64(body: &[u8]) -> String { + Checksum::new_from_data(RioChecksumType::CRC64_NVME, body) + .expect("crc64nvme checksum") + .encoded + } + /// PutObject with Content-MD5: upload succeeds and GetObject returns same content. #[tokio::test] #[serial] @@ -226,4 +233,135 @@ mod tests { ); info!("PASSED: MultipartUpload with checksum and GetObject content match"); } + + /// Regression test for issue #2282: + /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload. + #[tokio::test] + #[serial] + async fn test_crc64nvme_matches_between_put_object_and_multipart_upload() { + init_logging(); + info!("TEST: CRC64NVME matches between direct PutObject and multipart upload"); + + let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); + env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS"); + + let client = create_s3_client(&env); + let bucket = "test-crc64nvme-multipart-match"; + create_bucket(&client, bucket).await.expect("Failed to create bucket"); + + const PART_SIZE: usize = 6 * 1024 * 1024; + let part1: Vec = (0..PART_SIZE).map(|i| (i % 251) as u8).collect(); + let part2: Vec = (0..PART_SIZE).map(|i| ((i + 17) % 251) as u8).collect(); + let content: Vec = part1.iter().chain(part2.iter()).copied().collect(); + + let direct_key = "crc64nvme-direct.bin"; + let multipart_key = "crc64nvme-multipart.bin"; + let full_checksum = checksum_crc64nvme_base64(&content); + let part1_checksum = checksum_crc64nvme_base64(&part1); + let part2_checksum = checksum_crc64nvme_base64(&part2); + + client + .put_object() + .bucket(bucket) + .key(direct_key) + .body(ByteStream::from(content.clone())) + .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme) + .checksum_crc64_nvme(full_checksum.clone()) + .send() + .await + .expect("Failed to put direct object with CRC64NVME"); + + let create_result = client + .create_multipart_upload() + .bucket(bucket) + .key(multipart_key) + .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme) + .send() + .await + .expect("Failed to create multipart upload"); + + let upload_id = create_result.upload_id().expect("No upload_id").to_string(); + + let upload1 = client + .upload_part() + .bucket(bucket) + .key(multipart_key) + .upload_id(&upload_id) + .part_number(1) + .body(ByteStream::from(part1.clone())) + .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme) + .checksum_crc64_nvme(part1_checksum) + .send() + .await + .expect("Failed to upload multipart part 1"); + + let upload2 = client + .upload_part() + .bucket(bucket) + .key(multipart_key) + .upload_id(&upload_id) + .part_number(2) + .body(ByteStream::from(part2.clone())) + .checksum_algorithm(ChecksumAlgorithm::Crc64Nvme) + .checksum_crc64_nvme(part2_checksum) + .send() + .await + .expect("Failed to upload multipart part 2"); + + let completed_upload = CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(upload1.e_tag().expect("No etag for part 1")) + .checksum_crc64_nvme(upload1.checksum_crc64_nvme().expect("No CRC64NVME for part 1")) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(upload2.e_tag().expect("No etag for part 2")) + .checksum_crc64_nvme(upload2.checksum_crc64_nvme().expect("No CRC64NVME for part 2")) + .build(), + ) + .build(); + + client + .complete_multipart_upload() + .bucket(bucket) + .key(multipart_key) + .upload_id(&upload_id) + .multipart_upload(completed_upload) + .send() + .await + .expect("Failed to complete multipart upload"); + + let direct_head = client + .head_object() + .bucket(bucket) + .key(direct_key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("Failed to head direct object"); + + let multipart_head = client + .head_object() + .bucket(bucket) + .key(multipart_key) + .checksum_mode(ChecksumMode::Enabled) + .send() + .await + .expect("Failed to head multipart object"); + + assert_eq!( + direct_head.checksum_crc64_nvme(), + Some(full_checksum.as_str()), + "Direct object should report the uploaded full-object CRC64NVME" + ); + assert_eq!( + multipart_head.checksum_crc64_nvme(), + Some(full_checksum.as_str()), + "Multipart object should report the same full-object CRC64NVME as direct upload" + ); + } } diff --git a/crates/rio/src/checksum.rs b/crates/rio/src/checksum.rs index bb015f68a..86867cc15 100644 --- a/crates/rio/src/checksum.rs +++ b/crates/rio/src/checksum.rs @@ -981,17 +981,17 @@ const CRC64_NVME_POLYNOMIAL: u64 = 0xad93d23594c93659; /// GF(2) matrix multiplication fn gf2_matrix_times(mat: &[u64], mut vec: u64) -> u64 { let mut sum = 0u64; - let mut mat_iter = mat.iter(); + for &m in mat { + if vec == 0 { + break; + } - while vec != 0 { - if vec & 1 != 0 - && let Some(&m) = mat_iter.next() - { + if vec & 1 != 0 { sum ^= m; } vec >>= 1; - mat_iter.next(); } + sum } @@ -1128,3 +1128,62 @@ fn crc64_combine(poly: u64, crc1: u64, crc2: u64, len2: i64) -> u64 { // Return combined crc crc1n ^ crc2 } + +#[cfg(test)] +mod tests { + use super::{Checksum, ChecksumType}; + + #[test] + fn crc64_nvme_add_part_matches_full_object_checksum() { + let data = (0..200_000).map(|i| (i % 251) as u8).collect::>(); + let split_at = 73_421; + let (first, second) = data.split_at(split_at); + + let expected = Checksum::new_from_data(ChecksumType::CRC64_NVME, &data).expect("full checksum"); + let first_checksum = Checksum::new_from_data(ChecksumType::CRC64_NVME, first).expect("first checksum"); + let second_checksum = Checksum::new_from_data(ChecksumType::CRC64_NVME, second).expect("second checksum"); + + let mut combined = Checksum { + checksum_type: ChecksumType::CRC64_NVME, + ..Default::default() + }; + combined + .add_part(&first_checksum, first.len() as i64) + .expect("add first part"); + combined + .add_part(&second_checksum, second.len() as i64) + .expect("add second part"); + + assert_eq!(combined.encoded, expected.encoded); + assert_eq!(combined.raw, expected.raw); + } + + #[test] + fn crc32c_add_part_matches_full_object_checksum() { + let data = (0..32_768).map(|i| (255 - (i % 251)) as u8).collect::>(); + let (first, rest) = data.split_at(7_777); + let (second, third) = rest.split_at(13_333); + + let expected = Checksum::new_from_data(ChecksumType::CRC32C, &data).expect("full checksum"); + let first_checksum = Checksum::new_from_data(ChecksumType::CRC32C, first).expect("first checksum"); + let second_checksum = Checksum::new_from_data(ChecksumType::CRC32C, second).expect("second checksum"); + let third_checksum = Checksum::new_from_data(ChecksumType::CRC32C, third).expect("third checksum"); + + let mut combined = Checksum { + checksum_type: ChecksumType::CRC32C, + ..Default::default() + }; + combined + .add_part(&first_checksum, first.len() as i64) + .expect("add first part"); + combined + .add_part(&second_checksum, second.len() as i64) + .expect("add second part"); + combined + .add_part(&third_checksum, third.len() as i64) + .expect("add third part"); + + assert_eq!(combined.encoded, expected.encoded); + assert_eq!(combined.raw, expected.raw); + } +}