gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-19 02:56:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): add direct chunk GET fast path (#2351)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
This commit is contained in:
houseme
2026-04-07 08:33:46 +08:00
committed by GitHub
parent 8d27170ce4
commit 32bf8f5bf3
84 changed files with 15932 additions and 3592 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/kms/kms_local_test.rs
+96
View File
@@ -344,6 +344,9 @@ async fn test_local_kms_multipart_upload() {
test_multipart_upload_with_sse_c(&s3_client, TEST_BUCKET)
.await
.expect("SSE-C multipart upload test failed");
test_multipart_download_with_wrong_sse_c_key_fails(&s3_client, TEST_BUCKET)
.await
.expect("SSE-C multipart wrong-key download test failed");
// Test 4: Large multipart upload (test streaming encryption with multiple blocks)
// TODO: Re-enable after fixing streaming encryption issues with large files
@@ -648,6 +651,99 @@ async fn test_multipart_upload_with_sse_c(
Ok(())
}
async fn test_multipart_download_with_wrong_sse_c_key_fails(
s3_client: &aws_sdk_s3::Client,
bucket: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let object_key = "multipart-sse-c-bad-download-test";
let part_size = 5 * 1024 * 1024;
let total_parts = 2;
let total_size = part_size * total_parts;
let encryption_key = "01234567890123456789012345678901";
let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
let key_md5 = sse_customer_key_md5_base64(encryption_key);
let wrong_key = "abcdefghijklmnopqrstuvwxyz012345";
let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 5) % 256) as u8).collect();
let create_multipart_output = s3_client
.create_multipart_upload()
.bucket(bucket)
.key(object_key)
.sse_customer_algorithm("AES256")
.sse_customer_key(&key_b64)
.sse_customer_key_md5(&key_md5)
.send()
.await?;
let upload_id = create_multipart_output.upload_id().unwrap();
let mut completed_parts = Vec::new();
for part_number in 1..=total_parts {
let start = (part_number - 1) * part_size;
let end = std::cmp::min(start + part_size, total_size);
let part_data = &test_data[start..end];
let upload_part_output = s3_client
.upload_part()
.bucket(bucket)
.key(object_key)
.upload_id(upload_id)
.part_number(part_number as i32)
.body(aws_sdk_s3::primitives::ByteStream::from(part_data.to_vec()))
.sse_customer_algorithm("AES256")
.sse_customer_key(&key_b64)
.sse_customer_key_md5(&key_md5)
.send()
.await?;
completed_parts.push(
aws_sdk_s3::types::CompletedPart::builder()
.part_number(part_number as i32)
.e_tag(upload_part_output.e_tag().unwrap())
.build(),
);
}
let completed_multipart_upload = aws_sdk_s3::types::CompletedMultipartUpload::builder()
.set_parts(Some(completed_parts))
.build();
s3_client
.complete_multipart_upload()
.bucket(bucket)
.key(object_key)
.upload_id(upload_id)
.multipart_upload(completed_multipart_upload)
.send()
.await?;
let err = s3_client
.get_object()
.bucket(bucket)
.key(object_key)
.sse_customer_algorithm("AES256")
.sse_customer_key(&wrong_key_b64)
.sse_customer_key_md5(&wrong_key_md5)
.send()
.await
.expect_err("multipart SSE-C download with the wrong key should fail");
let service_err = err.into_service_error();
assert_eq!(
service_err.meta().code(),
Some("InvalidRequest"),
"wrong-key multipart SSE-C download should return InvalidRequest, got {:?}",
service_err.meta().code()
);
Ok(())
}
/// Test large multipart upload to verify streaming encryption works correctly
#[allow(dead_code)]
async fn test_large_multipart_upload(