gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 01:09:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: revert standalone #2351 artifacts (phase 5) (#2535)

Browse Source
This commit is contained in:
安正超
2026-04-14 22:35:05 +08:00
committed by GitHub
parent db8ef63674
commit 68d3dba9fc
20 changed files with 138 additions and 1681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/checksum_upload_test.rs
+1 -275
View File
@@ -19,13 +19,9 @@
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::{ByteStream, SdkBody};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumMode, CompletedMultipartUpload, CompletedPart};
use base64::Engine;
use bytes::Bytes;
use futures::StreamExt;
use http_body::Frame;
use http_body_util::StreamBody;
use rustfs_rio::{Checksum, ChecksumType as RioChecksumType};
use serial_test::serial;
use sha2::{Digest, Sha256};
@@ -68,53 +64,6 @@ mod tests {
.encoded
}
fn streamed_body_70kib_of_a() -> ByteStream {
let bytes = Bytes::from_static(&[b'a'; 1024]);
let stream = futures::stream::repeat_with(move || {
let frame = Frame::data(bytes.clone());
Ok::<_, std::io::Error>(frame)
});
let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
ByteStream::new(SdkBody::from_body_1_x(body))
}
struct WithSizeHint<T> {
inner: T,
size_hint: usize,
}
impl<T> WithSizeHint<T> {
fn new(inner: T, size_hint: usize) -> Self {
Self { inner, size_hint }
}
}
impl<T> http_body::Body for WithSizeHint<T>
where
T: http_body::Body + Unpin,
{
type Data = T::Data;
type Error = T::Error;
fn poll_frame(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
let this = self.get_mut();
std::pin::Pin::new(&mut this.inner).poll_frame(cx)
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> http_body::SizeHint {
let mut hint = self.inner.size_hint();
hint.set_exact(self.size_hint as u64);
hint
}
}
/// PutObject with Content-MD5: upload succeeds and GetObject returns same content.
#[tokio::test]
#[serial]
@@ -187,121 +136,6 @@ mod tests {
info!("PASSED: PutObject with checksum_sha256 and GetObject content match");
}
/// Mirrors `s3s-e2e` behavior: only request `checksum_algorithm`, then expect
/// both PutObject and GetObject(checksum_mode=enabled) to expose the same checksum.
#[tokio::test]
#[serial]
async fn test_put_object_with_checksum_algorithm_only() {
init_logging();
info!("TEST: PutObject with checksum_algorithm only");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client(&env);
let bucket = "test-checksum-algorithm-only";
create_bucket(&client, bucket).await.expect("Failed to create bucket");
let key = "obj-with-checksum-algorithm-only.txt";
let content = vec![b'a'; 70 * 1024];
let put_resp = client
.put_object()
.bucket(bucket)
.key(key)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.body(ByteStream::from(content.clone()))
.send()
.await
.expect("PutObject with checksum_algorithm should succeed");
let put_checksum = put_resp
.checksum_crc32()
.expect("PutObject should return checksum_crc32 when checksum_algorithm is used")
.to_string();
let mut get_resp = client
.get_object()
.bucket(bucket)
.key(key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("GetObject should succeed");
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(aws_sdk_s3::primitives::SdkBody::empty()))
.collect()
.await
.expect("collect body")
.into_bytes();
assert_eq!(body_bytes.as_ref(), content.as_slice(), "GetObject body must match uploaded content");
assert_eq!(
get_resp.checksum_crc32().map(str::to_string),
Some(put_checksum),
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum"
);
}
/// Matches the `s3s-e2e` streaming upload shape more closely than `ByteStream::from(Vec<u8>)`.
#[tokio::test]
#[serial]
async fn test_put_object_with_checksum_algorithm_only_streaming_body() {
init_logging();
info!("TEST: PutObject with checksum_algorithm only using streaming body");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client(&env);
let bucket = "test-checksum-algorithm-streaming";
create_bucket(&client, bucket).await.expect("Failed to create bucket");
let key = "obj-with-checksum-algorithm-streaming.txt";
let expected_content = vec![b'a'; 70 * 1024];
let put_resp = client
.put_object()
.bucket(bucket)
.key(key)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.body(streamed_body_70kib_of_a())
.send()
.await
.expect("PutObject with streaming checksum_algorithm should succeed");
let put_checksum = put_resp
.checksum_crc32()
.expect("PutObject should return checksum_crc32 for streaming checksum_algorithm uploads")
.to_string();
let mut get_resp = client
.get_object()
.bucket(bucket)
.key(key)
.checksum_mode(ChecksumMode::Enabled)
.send()
.await
.expect("GetObject should succeed");
let body_bytes = std::mem::replace(&mut get_resp.body, ByteStream::new(SdkBody::empty()))
.collect()
.await
.expect("collect body")
.into_bytes();
assert_eq!(
body_bytes.as_ref(),
expected_content.as_slice(),
"GetObject body must match uploaded content"
);
assert_eq!(
get_resp.checksum_crc32().map(str::to_string),
Some(put_checksum),
"GetObject(checksum_mode=enabled) should expose the stored CRC32 checksum for streaming uploads"
);
}
/// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content.
/// Uses part size >= 5MB (server minimum) for two parts.
#[tokio::test]
@@ -400,114 +234,6 @@ mod tests {
info!("PASSED: MultipartUpload with checksum and GetObject content match");
}
/// Mirrors `s3s-e2e` multipart behavior: request checksum algorithm at MPU creation,
/// rely on auto checksum handling during UploadPart, and expect CompleteMultipartUpload to succeed.
#[tokio::test]
#[serial]
async fn test_multipart_upload_with_crc32_algorithm_only() {
init_logging();
info!("TEST: MultipartUpload with checksum_algorithm only (CRC32)");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
let client = create_s3_client(&env);
let bucket = "test-multipart-checksum-crc32-auto";
create_bucket(&client, bucket).await.expect("Failed to create bucket");
let key = "multipart-with-crc32-auto.bin";
let part1_content = "a".repeat(5 * 1024 * 1024 + 1);
let part2_content = "b".repeat(1024);
let create_resp = client
.create_multipart_upload()
.bucket(bucket)
.key(key)
.checksum_algorithm(ChecksumAlgorithm::Crc32)
.send()
.await
.expect("CreateMultipartUpload should succeed");
let upload_id = create_resp.upload_id().expect("upload_id should be present");
let part1_resp = client
.upload_part()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(part1_content.clone().into_bytes()))
.send()
.await
.expect("UploadPart 1 should succeed");
let part1_checksum = part1_resp
.checksum_crc32()
.expect("UploadPart 1 should return checksum_crc32")
.to_string();
let part2_resp = client
.upload_part()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.part_number(2)
.body(ByteStream::from(part2_content.clone().into_bytes()))
.send()
.await
.expect("UploadPart 2 should succeed");
let part2_checksum = part2_resp
.checksum_crc32()
.expect("UploadPart 2 should return checksum_crc32")
.to_string();
let completed_upload = CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(part1_resp.e_tag().expect("etag part 1"))
.checksum_crc32(part1_checksum)
.build(),
)
.parts(
CompletedPart::builder()
.part_number(2)
.e_tag(part2_resp.e_tag().expect("etag part 2"))
.checksum_crc32(part2_checksum)
.build(),
)
.build();
client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(completed_upload)
.send()
.await
.expect("CompleteMultipartUpload should succeed");
let body_bytes = client
.get_object()
.bucket(bucket)
.key(key)
.send()
.await
.expect("GetObject should succeed")
.body
.collect()
.await
.expect("collect body")
.into_bytes();
let expected_content = format!("{part1_content}{part2_content}");
assert_eq!(
body_bytes.as_ref(),
expected_content.as_bytes(),
"completed multipart object must match concatenated parts"
);
}
/// Regression test for issue #2282:
/// CRC64NVME full-object checksum should match between direct PutObject and multipart upload.
#[tokio::test]