gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 05:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): stabilize tier and storage class checks (#5228)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-25 22:21:52 +08:00
committed by GitHub
parent 2cf5fd6bfc
commit 7ab0955f8b
5 changed files with 103 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/inline_fast_path_cluster_test.rs
+34 -26
View File
@@ -56,6 +56,7 @@ use tokio::net::TcpListener;
use tokio::sync::Mutex;
use tokio::task::JoinHandle;
use tokio::time::{Instant, sleep};
use uuid::Uuid;
use walkdir::WalkDir;
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
@@ -81,7 +82,6 @@ const FALLBACK_REQUEST_DIRECTION: &str = "request";
const FALLBACK_RESPONSE_DIRECTION: &str = "response";
const MPU_PART_1_SIZE: usize = 5 * 1024 * 1024;
const MPU_PART_2_SIZE: usize = 16 * KIB;
const TIER_NAME: &str = "COLDTIER";
const TIER_BUCKET: &str = "inline-fallback-cold-tier";
const TIER_PREFIX: &str = "tiered";
const MSGPACK_FALLBACK_CONTROL_SERIES: [(&str, &str); 4] = [
@@ -961,11 +961,15 @@ async fn signed_admin_request(
Ok((status, text))
}
async fn add_rustfs_tier(hot: &RustFSTestClusterEnvironment, cold: &RustFSTestEnvironment) -> TestResult {
fn unique_tier_name() -> String {
format!("COLDTIER{}", Uuid::new_v4().simple()).to_ascii_uppercase()
}
async fn add_rustfs_tier(hot: &RustFSTestClusterEnvironment, cold: &RustFSTestEnvironment, tier_name: &str) -> TestResult {
let body = serde_json::json!({
"type": "rustfs",
"rustfs": {
"name": TIER_NAME,
"name": tier_name,
"endpoint": cold.url.as_str(),
"accessKey": cold.access_key.as_str(),
"secretKey": cold.secret_key.as_str(),
@@ -990,7 +994,7 @@ async fn add_rustfs_tier(hot: &RustFSTestClusterEnvironment, cold: &RustFSTestEn
.await?;
let attempt = format!("status={status}, body={}", compact_body(&response));
if status.is_success() {
wait_for_tier_verifiable(hot, &format!("status={status}, body={response}")).await?;
wait_for_tier_verifiable(hot, tier_name, &format!("status={status}, body={response}")).await?;
return Ok(());
}
attempts.push(attempt);
@@ -1005,10 +1009,10 @@ async fn add_rustfs_tier(hot: &RustFSTestClusterEnvironment, cold: &RustFSTestEn
Err(format!("AddTier(RustFS) failed after readiness polling: {final_error}").into())
}
async fn wait_for_tier_verifiable(hot: &RustFSTestClusterEnvironment, add_tier_response: &str) -> TestResult {
async fn wait_for_tier_verifiable(hot: &RustFSTestClusterEnvironment, tier_name: &str, add_tier_response: &str) -> TestResult {
let deadline = Instant::now() + Duration::from_secs(60);
let final_error = loop {
let snapshot = tier_readiness_snapshot(hot).await?;
let snapshot = tier_readiness_snapshot(hot, tier_name).await?;
if snapshot.iter().any(|node| node.verify_status.is_success()) {
return Ok(());
}
@@ -1021,7 +1025,7 @@ async fn wait_for_tier_verifiable(hot: &RustFSTestClusterEnvironment, add_tier_r
sleep(Duration::from_millis(500)).await;
};
Err(format!(
"tier {TIER_NAME} was not verifiable on any hot node within 60s after AddTier({add_tier_response}): {final_error}"
"tier {tier_name} was not verifiable on any hot node within 60s after AddTier({add_tier_response}): {final_error}"
)
.into())
}
@@ -1036,7 +1040,7 @@ struct TierNodeReadiness {
verify_body: String,
}
async fn tier_readiness_snapshot(hot: &RustFSTestClusterEnvironment) -> TestResult<Vec<TierNodeReadiness>> {
async fn tier_readiness_snapshot(hot: &RustFSTestClusterEnvironment, tier_name: &str) -> TestResult<Vec<TierNodeReadiness>> {
let mut snapshot = Vec::with_capacity(hot.nodes.len());
for (node_index, node) in hot.nodes.iter().enumerate() {
let (list_status, list_body) =
@@ -1044,7 +1048,7 @@ async fn tier_readiness_snapshot(hot: &RustFSTestClusterEnvironment) -> TestResu
let (verify_status, verify_body) = signed_admin_request(
&node.url,
Method::GET,
&format!("/rustfs/admin/v3/tier/{TIER_NAME}"),
&format!("/rustfs/admin/v3/tier/{tier_name}"),
None,
&hot.access_key,
&hot.secret_key,
@@ -1054,7 +1058,7 @@ async fn tier_readiness_snapshot(hot: &RustFSTestClusterEnvironment) -> TestResu
node_index,
node_url: node.url.clone(),
list_status,
list_has_tier: tier_list_contains(&list_body),
list_has_tier: tier_list_contains(&list_body, tier_name),
list_body,
verify_status,
verify_body,
@@ -1063,18 +1067,18 @@ async fn tier_readiness_snapshot(hot: &RustFSTestClusterEnvironment) -> TestResu
Ok(snapshot)
}
fn tier_list_contains(response: &str) -> bool {
fn tier_list_contains(response: &str, tier_name: &str) -> bool {
serde_json::from_str::<serde_json::Value>(response)
.ok()
.and_then(|value| value.as_array().cloned())
.is_some_and(|tiers| {
tiers.iter().any(|tier| {
tier.get("name").and_then(serde_json::Value::as_str) == Some(TIER_NAME)
tier.get("name").and_then(serde_json::Value::as_str) == Some(tier_name)
|| tier
.get("rustfs")
.and_then(|rustfs| rustfs.get("name"))
.and_then(serde_json::Value::as_str)
== Some(TIER_NAME)
== Some(tier_name)
})
})
}
@@ -1121,30 +1125,30 @@ fn is_retryable_add_tier_error(response: &str) -> bool {
response.contains("Remote tier configuration is already being replaced")
}
fn transition_rule() -> TestResult<LifecycleRule> {
fn transition_rule(tier_name: &str) -> TestResult<LifecycleRule> {
Ok(LifecycleRule::builder()
.id("inline-fallback-transition")
.filter(LifecycleRuleFilter::builder().prefix("transition/").build())
.transitions(
Transition::builder()
.days(0)
.storage_class(TransitionStorageClass::from(TIER_NAME))
.storage_class(TransitionStorageClass::from(tier_name))
.build(),
)
.status(ExpirationStatus::Enabled)
.build()?)
}
async fn wait_for_transition(client: &Client, bucket: &str, key: &str) -> TestResult {
async fn wait_for_transition(client: &Client, bucket: &str, key: &str, tier_name: &str) -> TestResult {
let deadline = Instant::now() + Duration::from_secs(90);
loop {
let head = client.head_object().bucket(bucket).key(key).send().await?;
if head.storage_class().map(|storage_class| storage_class.as_str()) == Some(TIER_NAME) {
if head.storage_class().map(|storage_class| storage_class.as_str()) == Some(tier_name) {
return Ok(());
}
if Instant::now() >= deadline {
return Err(format!(
"object {bucket}/{key} was not transitioned to {TIER_NAME} within 90s (storage_class={:?})",
"object {bucket}/{key} was not transitioned to {tier_name} within 90s (storage_class={:?})",
head.storage_class()
)
.into());
@@ -1163,10 +1167,12 @@ async fn cold_tier_object_count(cold_client: &Client) -> TestResult<usize> {
.len())
}
async fn put_lifecycle_with_transition_retry(client: &Client, bucket: &str) -> TestResult {
async fn put_lifecycle_with_transition_retry(client: &Client, bucket: &str, tier_name: &str) -> TestResult {
let deadline = Instant::now() + Duration::from_secs(30);
loop {
let lifecycle = BucketLifecycleConfiguration::builder().rules(transition_rule()?).build()?;
let lifecycle = BucketLifecycleConfiguration::builder()
.rules(transition_rule(tier_name)?)
.build()?;
match client
.put_bucket_lifecycle_configuration()
.bucket(bucket)
@@ -1541,14 +1547,15 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
let fallback_before = collector.msgpack_json_fallback_totals().await;
add_rustfs_tier(&hot, &cold).await?;
let tier_name = unique_tier_name();
add_rustfs_tier(&hot, &cold, &tier_name).await?;
let bucket = "inline-transitioned-mixed-msgpack-controls";
hot_client.create_bucket().bucket(bucket).send().await?;
put_lifecycle_with_transition_retry(&hot_client, bucket).await?;
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
let key = "transition/mixed-multipart.bin";
let (body, second_part, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
wait_for_transition(&hot_client, bucket, key).await?;
wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
assert!(
cold_tier_object_count(&cold_client).await? >= 1,
"cold-tier bucket must hold transitioned objects"
@@ -1638,14 +1645,15 @@ async fn four_node_transitioned_inline_fallback() -> TestResult {
hot.start().await?;
let hot_client = hot.create_s3_client(0)?;
add_rustfs_tier(&hot, &cold).await?;
let tier_name = unique_tier_name();
add_rustfs_tier(&hot, &cold, &tier_name).await?;
let bucket = "inline-transitioned-fallback";
hot_client.create_bucket().bucket(bucket).send().await?;
put_lifecycle_with_transition_retry(&hot_client, bucket).await?;
put_lifecycle_with_transition_retry(&hot_client, bucket, &tier_name).await?;
let key = "transition/two-part.bin";
let (body, _, etag) = put_two_part_multipart(&hot_client, bucket, key).await?;
wait_for_transition(&hot_client, bucket, key).await?;
wait_for_transition(&hot_client, bucket, key, &tier_name).await?;
assert!(
cold_tier_object_count(&cold_client).await? >= 1,
"cold-tier bucket must hold the transitioned object"
crates/e2e_test/src/multipart_auth_test.rs
+3 -3
View File
@@ -1267,7 +1267,7 @@ async fn test_anonymous_post_object_accepts_storage_class_exact_policy_match()
let bucket = "anon-post-storage-class";
let object_key = "post-storage-class-object.txt";
let expected_body = b"post-storage-class-body".to_vec();
let storage_class = "STANDARD_IA";
let storage_class = "REDUCED_REDUNDANCY";
let admin_client = env.create_s3_client();
admin_client.create_bucket().bucket(bucket).send().await?;
@@ -5138,7 +5138,7 @@ async fn test_signed_put_object_extract_preserves_storage_class() -> Result<(),
.put_object()
.bucket(bucket)
.key(archive_key)
.storage_class(aws_sdk_s3::types::StorageClass::StandardIa)
.storage_class(aws_sdk_s3::types::StorageClass::ReducedRedundancy)
.body(ByteStream::from(tar_bytes))
.customize()
.mutate_request(move |req| {
@@ -5155,7 +5155,7 @@ async fn test_signed_put_object_extract_preserves_storage_class() -> Result<(),
.send()
.await?;
assert_eq!(head.storage_class().map(|value| value.as_str()), Some("STANDARD_IA"));
assert_eq!(head.storage_class().map(|value| value.as_str()), Some("REDUCED_REDUNDANCY"));
Ok(())
}