gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 21:26:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): activate conditional write regressions

Browse Source
This commit is contained in:
overtrue
2026-08-23 11:27:51 +08:00
parent 20d1266496
commit f4da36b364
3 changed files with 126 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/e2e-full-selection.txt
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52 sha256-darwin=03bdfb6a9d6e25d744c385f1461e651f05e4da78e5b0ead2adb3e8b2463e3834
sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6 sha256-linux=78c46adad135231017fb8679fb91877b5ae3ec6ce1f75d3d045d2c1076c12a49
crates/e2e_test/src/reliant/conditional_writes.rs
+123 -162
View File
@@ -1,52 +1,26 @@
#![cfg(test)] #![cfg(test)]
use aws_config::meta::region::RegionProviderChain; use crate::common::{RustFSTestEnvironment, TEST_BUCKET, init_logging};
use aws_sdk_s3::Client; use aws_sdk_s3::Client;
use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use bytes::Bytes; use bytes::Bytes;
use std::error::Error; use std::error::Error;
use std::fmt::Debug;
const ENDPOINT: &str = "http://localhost:9000"; type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
const ACCESS_KEY: &str = "rustfsadmin";
const SECRET_KEY: &str = "rustfsadmin";
const BUCKET: &str = "api-test";
async fn create_aws_s3_client() -> Result<Client, Box<dyn Error>> { fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, expected: &str)
let region_provider = RegionProviderChain::default_provider().or_else(Region::new("us-east-1")); where
let shared_config = aws_config::defaults(aws_config::BehaviorVersion::latest()) T: Debug,
.region(region_provider) E: ProvideErrorMetadata + Debug,
.credentials_provider(Credentials::new(ACCESS_KEY, SECRET_KEY, None, None, "static")) {
.endpoint_url(ENDPOINT) let error = result.expect_err("conditional request must fail");
.load() assert_eq!(
.await; error.as_service_error().and_then(ProvideErrorMetadata::code),
Some(expected),
let client = Client::from_conf( "unexpected conditional request error: {error:?}"
aws_sdk_s3::Config::from(&shared_config)
.to_builder()
.force_path_style(true)
.build(),
); );
Ok(client)
}
/// Setup test bucket, creating it if it doesn't exist
async fn setup_test_bucket(client: &Client) -> Result<(), Box<dyn Error>> {
match client.create_bucket().bucket(BUCKET).send().await {
Ok(_) => {}
Err(SdkError::ServiceError(e)) => {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
if !error_code.eq("BucketAlreadyExists") {
return Err(e.into());
}
}
Err(e) => {
return Err(e.into());
}
}
Ok(())
} }
/// Generate test data of specified size /// Generate test data of specified size
@@ -60,7 +34,12 @@ fn generate_test_data(size: usize) -> Vec<u8> {
} }
/// Upload an object and return its ETag /// Upload an object and return its ETag
async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, data: &[u8]) -> Result<String, Box<dyn Error>> { async fn upload_object_with_metadata(
client: &Client,
bucket: &str,
key: &str,
data: &[u8],
) -> Result<String, Box<dyn Error + Send + Sync>> {
let response = client let response = client
.put_object() .put_object()
.bucket(bucket) .bucket(bucket)
@@ -69,188 +48,164 @@ async fn upload_object_with_metadata(client: &Client, bucket: &str, key: &str, d
.send() .send()
.await?; .await?;
let etag = response.e_tag().unwrap_or("").to_string(); response
Ok(etag) .e_tag()
.map(str::to_owned)
.ok_or_else(|| std::io::Error::other("put object response did not include an ETag").into())
} }
/// Cleanup test objects from bucket async fn object_body(client: &Client, key: &str) -> Result<Bytes, Box<dyn Error + Send + Sync>> {
async fn cleanup_objects(client: &Client, bucket: &str, keys: &[&str]) { let response = client.get_object().bucket(TEST_BUCKET).key(key).send().await?;
for key in keys { Ok(response.body.collect().await?.into_bytes())
let _ = client.delete_object().bucket(bucket).key(*key).send().await;
}
}
/// Generate unique test object key
fn generate_test_key(prefix: &str) -> String {
use std::time::{SystemTime, UNIX_EPOCH};
let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
format!("{prefix}-{timestamp}")
} }
#[tokio::test] #[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_okay() -> TestResult {
async fn test_conditional_put_okay() -> Result<(), Box<dyn std::error::Error>> { init_logging();
let client = create_aws_s3_client().await?; let mut env = RustFSTestEnvironment::new().await?;
setup_test_bucket(&client).await?; env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
let test_key = generate_test_key("conditional-put-ok"); let test_key = "conditional-put-ok";
let initial_data = generate_test_data(1024); // 1KB test data let initial_data = generate_test_data(1024); // 1KB test data
let updated_data = generate_test_data(2048); // 2KB updated data let matching_data = generate_test_data(2048); // 2KB updated data
let non_matching_data = generate_test_data(3072); // 3KB updated data
// Upload initial object and get its ETag // Upload initial object and get its ETag
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?; let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
// Test 1: PUT with matching If-Match condition (should succeed) // Test 1: PUT with matching If-Match condition (should succeed)
let response1 = client client
.put_object() .put_object()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.body(Bytes::from(updated_data.clone()).into()) .body(Bytes::from(matching_data.clone()).into())
.if_match(&initial_etag) .if_match(&initial_etag)
.send() .send()
.await; .await?;
assert!(response1.is_ok(), "PUT with matching If-Match should succeed"); assert_eq!(object_body(&client, test_key).await?.as_ref(), matching_data);
// Test 2: PUT with non-matching If-None-Match condition (should succeed) // Test 2: PUT with non-matching If-None-Match condition (should succeed)
let fake_etag = "\"fake-etag-12345\""; let fake_etag = "\"fake-etag-12345\"";
let response2 = client client
.put_object() .put_object()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.body(Bytes::from(updated_data.clone()).into()) .body(Bytes::from(non_matching_data.clone()).into())
.if_none_match(fake_etag) .if_none_match(fake_etag)
.send() .send()
.await; .await?;
assert!(response2.is_ok(), "PUT with non-matching If-None-Match should succeed"); assert_eq!(object_body(&client, test_key).await?.as_ref(), non_matching_data);
// Cleanup
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(()) Ok(())
} }
#[tokio::test] #[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_failed() -> TestResult {
async fn test_conditional_put_failed() -> Result<(), Box<dyn std::error::Error>> { init_logging();
let client = create_aws_s3_client().await?; let mut env = RustFSTestEnvironment::new().await?;
setup_test_bucket(&client).await?; env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
let test_key = generate_test_key("conditional-put-failed"); let test_key = "conditional-put-failed";
let initial_data = generate_test_data(1024); let initial_data = generate_test_data(1024);
let updated_data = generate_test_data(2048); let updated_data = generate_test_data(2048);
// Upload initial object and get its ETag // Upload initial object and get its ETag
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &initial_data).await?; let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &initial_data).await?;
// Test 1: PUT with non-matching If-Match condition (should fail with 412) // Test 1: PUT with non-matching If-Match condition (should fail with 412)
let fake_etag = "\"fake-etag-should-not-match\""; let fake_etag = "\"fake-etag-should-not-match\"";
let response1 = client let response1 = client
.put_object() .put_object()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.body(Bytes::from(updated_data.clone()).into()) .body(Bytes::from(updated_data.clone()).into())
.if_match(fake_etag) .if_match(fake_etag)
.send() .send()
.await; .await;
assert!(response1.is_err(), "PUT with non-matching If-Match should fail"); assert_s3_error_code(response1, "PreconditionFailed");
if let Err(e) = response1 { assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("PreconditionFailed", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// Test 2: PUT with matching If-None-Match condition (should fail with 412) // Test 2: PUT with matching If-None-Match condition (should fail with 412)
let response2 = client let response2 = client
.put_object() .put_object()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.body(Bytes::from(updated_data.clone()).into()) .body(Bytes::from(updated_data.clone()).into())
.if_none_match(&initial_etag) .if_none_match(&initial_etag)
.send() .send()
.await; .await;
assert!(response2.is_err(), "PUT with matching If-None-Match should fail"); assert_s3_error_code(response2, "PreconditionFailed");
if let Err(e) = response2 { assert_eq!(object_body(&client, test_key).await?.as_ref(), initial_data);
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("PreconditionFailed", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// Cleanup - only need to clean up the initial object since failed PUTs shouldn't create objects
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(()) Ok(())
} }
#[tokio::test] #[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_when_object_does_not_exist() -> TestResult {
async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box<dyn std::error::Error>> { init_logging();
let client = create_aws_s3_client().await?; let mut env = RustFSTestEnvironment::new().await?;
setup_test_bucket(&client).await?; env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
let key = "some_key"; let key = "conditional-put-missing";
cleanup_objects(&client, BUCKET, &[key]).await;
// When the object does not exist, the If-Match condition should always fail // When the object does not exist, the If-Match condition should always fail
let response1 = client let response1 = client
.put_object() .put_object()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(key) .key(key)
.body(Bytes::from(generate_test_data(1024)).into()) .body(Bytes::from(generate_test_data(1024)).into())
.if_match("*") .if_match("*")
.send() .send()
.await; .await;
assert!(response1.is_err()); assert_s3_error_code(response1, "NoSuchKey");
if let Err(e) = response1 {
if let SdkError::ServiceError(e) = e {
let e = e.into_err();
let error_code = e.meta().code().unwrap_or("");
assert_eq!("NoSuchKey", error_code);
} else {
panic!("Unexpected error: {e:?}");
}
}
// When the object does not exist, the If-None-Match condition should be able to succeed // When the object does not exist, the If-None-Match condition should be able to succeed
let response2 = client let created_data = generate_test_data(1024);
client
.put_object() .put_object()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(key) .key(key)
.body(Bytes::from(generate_test_data(1024)).into()) .body(Bytes::from(created_data.clone()).into())
.if_none_match("*") .if_none_match("*")
.send() .send()
.await; .await?;
assert!(response2.is_ok()); assert_eq!(object_body(&client, key).await?.as_ref(), created_data);
cleanup_objects(&client, BUCKET, &[key]).await;
Ok(()) Ok(())
} }
#[tokio::test] #[tokio::test]
#[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_multi_part_upload() -> TestResult {
async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::Error>> { init_logging();
let client = create_aws_s3_client().await?; let mut env = RustFSTestEnvironment::new().await?;
setup_test_bucket(&client).await?; env.start_rustfs_server(vec![]).await?;
env.create_test_bucket(TEST_BUCKET).await?;
let client = env.create_s3_client();
let test_key = generate_test_key("multipart-upload-ok"); let test_key = "conditional-multipart-upload";
let test_data = generate_test_data(1024); let test_data = generate_test_data(1024);
let initial_etag = upload_object_with_metadata(&client, BUCKET, &test_key, &test_data).await?; let initial_etag = upload_object_with_metadata(&client, TEST_BUCKET, test_key, &test_data).await?;
let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart) let part_size = 5 * 1024 * 1024; // 5MB per part (minimum for multipart)
let num_parts = 3; let num_parts = 3;
let mut parts = Vec::new(); let mut parts = Vec::new();
let mut expected_data = Vec::with_capacity(part_size * usize::try_from(num_parts)?);
// Initiate multipart upload // Initiate multipart upload
let initiate_response = client.create_multipart_upload().bucket(BUCKET).key(&test_key).send().await?; let initiate_response = client
.create_multipart_upload()
.bucket(TEST_BUCKET)
.key(test_key)
.send()
.await?;
let upload_id = initiate_response let upload_id = initiate_response
.upload_id() .upload_id()
@@ -258,12 +213,13 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
// Upload parts // Upload parts
for part_number in 1..=num_parts { for part_number in 1..=num_parts {
let part_data = generate_test_data(part_size); let part_data = vec![u8::try_from(part_number)?; part_size];
expected_data.extend_from_slice(&part_data);
let upload_part_response = client let upload_part_response = client
.upload_part() .upload_part()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.upload_id(upload_id) .upload_id(upload_id)
.part_number(part_number) .part_number(part_number)
.body(Bytes::from(part_data).into()) .body(Bytes::from(part_data).into())
@@ -286,57 +242,62 @@ async fn test_conditional_multi_part_upload() -> Result<(), Box<dyn std::error::
// Test 1: Multipart upload with wildcard If-None-Match, should fail // Test 1: Multipart upload with wildcard If-None-Match, should fail
let complete_response = client let complete_response = client
.complete_multipart_upload() .complete_multipart_upload()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.upload_id(upload_id) .upload_id(upload_id)
.multipart_upload(completed_upload.clone()) .multipart_upload(completed_upload.clone())
.if_none_match("*") .if_none_match("*")
.send() .send()
.await; .await;
assert!(complete_response.is_err()); assert_s3_error_code(complete_response, "PreconditionFailed");
// Test 2: Multipart upload with matching If-None-Match, should fail // Test 2: Multipart upload with matching If-None-Match, should fail
let complete_response = client let complete_response = client
.complete_multipart_upload() .complete_multipart_upload()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.upload_id(upload_id) .upload_id(upload_id)
.multipart_upload(completed_upload.clone()) .multipart_upload(completed_upload.clone())
.if_none_match(initial_etag.clone()) .if_none_match(initial_etag.clone())
.send() .send()
.await; .await;
assert!(complete_response.is_err()); assert_s3_error_code(complete_response, "PreconditionFailed");
// Test 3: Multipart upload with unmatching If-Match, should fail // Test 3: Multipart upload with unmatching If-Match, should fail
let complete_response = client let complete_response = client
.complete_multipart_upload() .complete_multipart_upload()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.upload_id(upload_id) .upload_id(upload_id)
.multipart_upload(completed_upload.clone()) .multipart_upload(completed_upload.clone())
.if_match("\"abcdef\"") .if_match("\"abcdef\"")
.send() .send()
.await; .await;
assert!(complete_response.is_err()); assert_s3_error_code(complete_response, "PreconditionFailed");
let staged_parts = client
.list_parts()
.bucket(TEST_BUCKET)
.key(test_key)
.upload_id(upload_id)
.send()
.await?;
assert_eq!(staged_parts.parts().len(), usize::try_from(num_parts)?);
// Test 4: Multipart upload with matching If-Match, should succeed // Test 4: Multipart upload with matching If-Match, should succeed
let complete_response = client client
.complete_multipart_upload() .complete_multipart_upload()
.bucket(BUCKET) .bucket(TEST_BUCKET)
.key(&test_key) .key(test_key)
.upload_id(upload_id) .upload_id(upload_id)
.multipart_upload(completed_upload.clone()) .multipart_upload(completed_upload)
.if_match(initial_etag) .if_match(initial_etag)
.send() .send()
.await; .await?;
assert_eq!(object_body(&client, test_key).await?.as_ref(), expected_data);
assert!(complete_response.is_ok());
// Cleanup
cleanup_objects(&client, BUCKET, &[&test_key]).await;
Ok(()) Ok(())
} }
docs/testing/e2e-suite-inventory.md
+1 -1
View File
@@ -84,7 +84,7 @@
| protocols | 16 | 🌙 | | protocols | 16 | 🌙 |
| quota_test | 14 | | | quota_test | 14 | |
| reliability_disk_fault_test | 4 | | | reliability_disk_fault_test | 4 | |
| reliant | 25 | 19 ✅ | | reliant | 29 | 19 ✅ |
| replication_extension_test | 75 | 20 ✅ +55 🌙 | | replication_extension_test | 75 | 20 ✅ +55 🌙 |
| security_boundary_test | 4 | | | security_boundary_test | 4 | |
| server_startup_failfast_test | 1 | | | server_startup_failfast_test | 1 | |