gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 05:26:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into overtrue/wait-bucket-metadata-reload

Browse Source
This commit is contained in:
houseme
2026-08-23 12:30:36 +08:00
committed by GitHub
parent c57f22c3a0 5f72209446
commit b3c79cd71a
153 changed files with 14834 additions and 1988 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs
+18 -4
View File
@@ -380,10 +380,24 @@ mod tests {
cluster.start_node(1).await?;
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
assert!(
!status_body.contains("MissingContentLength"),
"background heal status should not fail without an explicit Content-Length: {status_body}"
let mut recovered = serde_json::Value::Null;
for _ in 0..60 {
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
assert!(
!status_body.contains("MissingContentLength"),
"background heal status should not fail without an explicit Content-Length: {status_body}"
);
recovered = serde_json::from_str(&status_body)
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
break;
}
sleep(Duration::from_secs(1)).await;
}
assert_eq!(
recovered["clusterStatusComplete"],
serde_json::Value::Bool(true),
"cluster heal status should recover before root heal starts: {recovered}"
);
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
crates/e2e_test/src/kms/bucket_default_encryption_test.rs
+5 -5
View File
@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/common.rs
+107 -20
View File
@@ -189,34 +189,121 @@ pub async fn wait_for_kms_ready(
access_key: &str,
secret_key: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let total_deadline = Duration::from_secs(5);
wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
}
async fn wait_for_kms_ready_with_timeout(
base_url: &str,
access_key: &str,
secret_key: &str,
total_deadline: Duration,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let start = tokio::time::Instant::now();
let deadline = start + total_deadline;
let mut backoff = Duration::from_millis(200);
let max_backoff = Duration::from_secs(1);
let mut first_attempt = true;
loop {
if !first_attempt {
if start.elapsed() >= total_deadline {
return Err("KMS failed to become ready within 5 seconds".into());
}
sleep(backoff).await;
backoff = (backoff * 2).min(max_backoff);
}
first_attempt = false;
match get_kms_status(base_url, access_key, secret_key).await {
Ok(status) => {
info!("KMS is ready (status: {})", status);
return Ok(());
}
Err(e) => {
if start.elapsed() >= total_deadline {
return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
Ok(Ok(status)) => {
let backend_status = serde_json::from_str::<serde_json::Value>(&status)
.ok()
.and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
if backend_status.as_deref() == Some("healthy") {
info!("KMS is ready (status: {})", status);
return Ok(());
}
warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
warn!(
backend_status = backend_status.as_deref().unwrap_or("missing"),
elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
"KMS not ready yet, retrying…"
);
}
Ok(Err(e)) => {
let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
}
Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
}
let now = tokio::time::Instant::now();
if now >= deadline {
return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
}
sleep((now + backoff).min(deadline) - now).await;
backoff = (backoff * 2).min(max_backoff);
}
}
#[cfg(test)]
mod readiness_tests {
use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
#[tokio::test]
async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
let address = listener.local_addr().expect("read readiness test server address");
let requests = Arc::new(AtomicUsize::new(0));
let server_requests = Arc::clone(&requests);
let server = tokio::spawn(async move {
for backend_status in ["error", "healthy"] {
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
let mut request = Vec::new();
let mut chunk = [0_u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let read = socket.read(&mut chunk).await.expect("read readiness request");
if read == 0 {
break;
}
request.extend_from_slice(&chunk[..read]);
}
server_requests.fetch_add(1, Ordering::SeqCst);
let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
socket.write_all(response.as_bytes()).await.expect("write readiness response");
}
});
wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
.await
.expect("KMS should become ready after the healthy response");
let observed_requests = requests.load(Ordering::SeqCst);
server.abort();
assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
}
#[tokio::test]
async fn kms_readiness_deadline_covers_a_stalled_status_request() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
let address = listener.local_addr().expect("read readiness test server address");
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
let mut request = [0_u8; 1024];
let _ = socket.read(&mut request).await.expect("read readiness request");
std::future::pending::<()>().await;
});
let result = tokio::time::timeout(
Duration::from_secs(1),
wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
)
.await
.expect("readiness helper must enforce its own deadline");
server.abort();
assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
}
}
crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs
+3 -3
View File
@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
// Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
// Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
let bucket = "copy-object-self-copy-bucket-default-sse-test";
crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs
+1 -1
View File
@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
)
.await
.expect("failed to start RustFS with local KMS");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
let client = kms_env.base_env.create_s3_client();
let bucket = "copy-object-version-restore-sse-test";
crates/e2e_test/src/kms/encryption_metadata_test.rs
+3 -3
View File
@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_comprehensive_test.rs
+5 -6
View File
@@ -24,7 +24,6 @@ use super::common::{
test_sse_kms_encryption, test_sse_s3_encryption,
};
use crate::common::{TEST_BUCKET, init_logging};
use tokio::time::{Duration, sleep};
use tracing::info;
/// Comprehensive test: Full KMS workflow with all encryption types
@@ -35,7 +34,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -103,7 +102,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -137,7 +136,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -208,7 +207,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -253,7 +252,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_edge_cases_test.rs
+6 -6
View File
@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = Arc::new(kms_env.base_env.create_s3_client());
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_fault_recovery_test.rs
+4 -4
View File
@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/kms_local_test.rs
+4 -4
View File
@@ -46,7 +46,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -127,7 +127,7 @@ async fn test_local_kms_key_isolation() {
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -227,7 +227,7 @@ async fn test_local_kms_large_file() {
.expect("Failed to start RustFS with Local KMS");
// Wait a moment for RustFS to fully start up and initialize KMS
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
@@ -309,7 +309,7 @@ async fn test_local_kms_multipart_upload() {
.expect("Failed to start RustFS with Local KMS");
// Wait for KMS initialization
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await.expect("KMS ready");
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
crates/e2e_test/src/kms/kms_vault_test.rs
+2 -3
View File
@@ -19,7 +19,6 @@
//! multipart upload behaviour.
use crate::common::{TEST_BUCKET, init_logging};
use tokio::time::{Duration, sleep};
use tracing::{error, info};
use super::common::{
@@ -45,8 +44,8 @@ impl VaultKmsTestContext {
start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
// Allow Vault to finish initialising token auth and transit engine.
sleep(Duration::from_secs(2)).await;
// Wait for KMS to finish initialising.
super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
Ok(Self { env })
}
crates/e2e_test/src/kms/mod.rs
-3
View File
@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
#[cfg(test)]
mod kms_fault_recovery_test;
#[cfg(test)]
mod test_runner;
#[cfg(test)]
mod bucket_default_encryption_test;
crates/e2e_test/src/kms/multipart_encryption_test.rs
+5 -5
View File
@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
kms_env.wait_for_kms_ready().await?;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
crates/e2e_test/src/kms/test_runner.rs
-499
View File
@@ -1,499 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
#![allow(dead_code)]
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified KMS test suite runner
//!
//! This module provides a unified interface for running KMS tests with categorization,
//! filtering, and comprehensive reporting capabilities.
use crate::common::init_logging;
use std::time::Instant;
use tokio::time::{Duration, sleep};
use tracing::{debug, error, info, warn};
/// Test category for organization and filtering
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum TestCategory {
CoreFunctionality,
MultipartEncryption,
EdgeCases,
FaultRecovery,
Comprehensive,
Performance,
}
impl TestCategory {
pub fn as_str(&self) -> &'static str {
match self {
TestCategory::CoreFunctionality => "core-functionality",
TestCategory::MultipartEncryption => "multipart-encryption",
TestCategory::EdgeCases => "edge-cases",
TestCategory::FaultRecovery => "fault-recovery",
TestCategory::Comprehensive => "comprehensive",
TestCategory::Performance => "performance",
}
}
}
/// Test definition with metadata
#[derive(Debug, Clone)]
pub struct TestDefinition {
pub name: String,
pub description: String,
pub category: TestCategory,
pub estimated_duration: Duration,
pub is_critical: bool,
}
impl TestDefinition {
pub fn new(
name: impl Into<String>,
description: impl Into<String>,
category: TestCategory,
estimated_duration: Duration,
is_critical: bool,
) -> Self {
Self {
name: name.into(),
description: description.into(),
category,
estimated_duration,
is_critical,
}
}
}
/// Test execution result
#[derive(Debug, Clone)]
pub struct TestResult {
pub test_name: String,
pub category: TestCategory,
pub success: bool,
pub duration: Duration,
pub error_message: Option<String>,
}
impl TestResult {
pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
Self {
test_name,
category,
success: true,
duration,
error_message: None,
}
}
pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
Self {
test_name,
category,
success: false,
duration,
error_message: Some(error),
}
}
}
/// Comprehensive test suite configuration
#[derive(Debug, Clone)]
pub struct TestSuiteConfig {
pub categories: Vec<TestCategory>,
pub include_critical_only: bool,
pub max_duration: Option<Duration>,
pub parallel_execution: bool,
}
impl Default for TestSuiteConfig {
fn default() -> Self {
Self {
categories: vec![
TestCategory::CoreFunctionality,
TestCategory::MultipartEncryption,
TestCategory::EdgeCases,
TestCategory::FaultRecovery,
TestCategory::Comprehensive,
],
include_critical_only: false,
max_duration: None,
parallel_execution: false,
}
}
}
/// Unified KMS test suite runner
pub struct KMSTestSuite {
tests: Vec<TestDefinition>,
config: TestSuiteConfig,
}
impl KMSTestSuite {
/// Create a new test suite with default configuration
pub fn new() -> Self {
let tests = vec![
// Core Functionality Tests
TestDefinition::new(
"test_local_kms_end_to_end",
"End-to-end KMS test with all encryption types",
TestCategory::CoreFunctionality,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_local_kms_key_isolation",
"Test KMS key isolation and security",
TestCategory::CoreFunctionality,
Duration::from_secs(45),
true,
),
// Multipart Encryption Tests
TestDefinition::new(
"test_local_kms_multipart_upload",
"Test large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
TestDefinition::new(
"test_step1_basic_single_file_encryption",
"Basic single file encryption test",
TestCategory::MultipartEncryption,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_step2_basic_multipart_upload_without_encryption",
"Basic multipart upload without encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_step3_multipart_upload_with_sse_s3",
"Multipart upload with SSE-S3 encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(60),
true,
),
TestDefinition::new(
"test_step4_large_multipart_upload_with_encryption",
"Large file multipart upload with encryption",
TestCategory::MultipartEncryption,
Duration::from_secs(90),
false,
),
TestDefinition::new(
"test_step5_all_encryption_types_multipart",
"All encryption types multipart test",
TestCategory::MultipartEncryption,
Duration::from_secs(120),
true,
),
// Edge Cases Tests
TestDefinition::new(
"test_kms_zero_byte_file_encryption",
"Test encryption of zero-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_single_byte_file_encryption",
"Test encryption of single-byte files",
TestCategory::EdgeCases,
Duration::from_secs(20),
false,
),
TestDefinition::new(
"test_kms_multipart_boundary_conditions",
"Test multipart upload boundary conditions",
TestCategory::EdgeCases,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_invalid_key_scenarios",
"Test invalid key scenarios",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_concurrent_encryption",
"Test concurrent encryption operations",
TestCategory::EdgeCases,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_key_validation_security",
"Test key validation security",
TestCategory::EdgeCases,
Duration::from_secs(30),
false,
),
// Fault Recovery Tests
TestDefinition::new(
"test_kms_key_directory_unavailable",
"Test KMS when key directory is unavailable",
TestCategory::FaultRecovery,
Duration::from_secs(45),
false,
),
TestDefinition::new(
"test_kms_corrupted_key_files",
"Test KMS with corrupted key files",
TestCategory::FaultRecovery,
Duration::from_secs(30),
false,
),
TestDefinition::new(
"test_kms_multipart_upload_interruption",
"Test multipart upload interruption recovery",
TestCategory::FaultRecovery,
Duration::from_secs(60),
false,
),
TestDefinition::new(
"test_kms_resource_constraints",
"Test KMS under resource constraints",
TestCategory::FaultRecovery,
Duration::from_secs(90),
false,
),
// Comprehensive Tests
TestDefinition::new(
"test_comprehensive_kms_full_workflow",
"Full KMS workflow comprehensive test",
TestCategory::Comprehensive,
Duration::from_secs(300),
true,
),
TestDefinition::new(
"test_comprehensive_stress_test",
"KMS stress test with large datasets",
TestCategory::Comprehensive,
Duration::from_secs(400),
false,
),
TestDefinition::new(
"test_comprehensive_key_isolation",
"Comprehensive key isolation test",
TestCategory::Comprehensive,
Duration::from_secs(180),
false,
),
TestDefinition::new(
"test_comprehensive_concurrent_operations",
"Comprehensive concurrent operations test",
TestCategory::Comprehensive,
Duration::from_secs(240),
false,
),
TestDefinition::new(
"test_comprehensive_performance_benchmark",
"KMS performance benchmark test",
TestCategory::Comprehensive,
Duration::from_secs(360),
false,
),
];
Self {
tests,
config: TestSuiteConfig::default(),
}
}
/// Configure the test suite
pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
self.config = config;
self
}
/// Filter tests based on category
pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| &test.category == category).collect()
}
/// Filter tests based on criticality
pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
self.tests.iter().filter(|test| test.is_critical).collect()
}
/// Get test summary by category
pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
let mut summary = std::collections::HashMap::new();
for test in &self.tests {
summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
}
summary
}
/// Run the complete test suite
pub async fn run_test_suite(&self) -> Vec<TestResult> {
init_logging();
info!("🚀 Starting unified KMS test suite");
let start_time = Instant::now();
let mut results = Vec::new();
// Filter tests based on configuration
let tests_to_run: Vec<&TestDefinition> = self
.tests
.iter()
.filter(|test| self.config.categories.contains(&test.category))
.filter(|test| !self.config.include_critical_only || test.is_critical)
.collect();
info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
for (i, test) in tests_to_run.iter().enumerate() {
info!(" {}. {} ({})", i + 1, test.name, test.category.as_str());
}
// Execute tests
for (i, test_def) in tests_to_run.iter().enumerate() {
info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
info!(" 📝 Description: {}", test_def.description);
info!(" 🏷️ Category: {}", test_def.category.as_str());
info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration);
let test_start = Instant::now();
let result = self.run_single_test(test_def).await;
let test_duration = test_start.elapsed();
match result {
Ok(_) => {
info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
}
Err(e) => {
error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
results.push(TestResult::failure(
test_def.name.clone(),
test_def.category.clone(),
test_duration,
e.to_string(),
));
}
}
// Add delay between tests to avoid resource conflicts
if i < tests_to_run.len() - 1 {
debug!("⏸️ Waiting two seconds before the next test...");
sleep(Duration::from_secs(2)).await;
}
}
let total_duration = start_time.elapsed();
self.print_test_summary(&results, total_duration);
results
}
/// Run a single test by dispatching to the appropriate test function
async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// This is a placeholder for test dispatch logic
// In a real implementation, this would dispatch to actual test functions
warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name);
Ok(())
}
/// Print comprehensive test summary
fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
info!("📊 KMS test suite summary");
info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64());
info!("📈 Total tests: {}", results.len());
let passed = results.iter().filter(|r| r.success).count();
let failed = results.iter().filter(|r| !r.success).count();
info!("✅ Passed: {}", passed);
info!("❌ Failed: {}", failed);
info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
// Summary by category
let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
for result in results {
let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
*total += 1;
if result.success {
*passed_count += 1;
}
}
info!("📊 Category summary:");
for (category, (total, passed_count)) in category_summary {
info!(
" 🏷️ {}: {}/{} ({:.1}%)",
category.as_str(),
passed_count,
total,
(passed_count as f64 / total as f64) * 100.0
);
}
// List failed tests
if failed > 0 {
warn!("❌ Failing tests:");
for result in results.iter().filter(|r| !r.success) {
warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
}
}
}
}
/// Quick test suite for critical tests only
#[tokio::test]
async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let config = TestSuiteConfig {
categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
include_critical_only: true,
max_duration: Some(Duration::from_secs(600)), // 10 minutes max
parallel_execution: false,
};
let suite = KMSTestSuite::new().with_config(config);
let results = suite.run_test_suite().await;
let failed_count = results.iter().filter(|r| !r.success).count();
if failed_count > 0 {
return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
}
info!("✅ All critical tests passed");
Ok(())
}
/// Full comprehensive test suite
#[tokio::test]
async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let suite = KMSTestSuite::new();
let results = suite.run_test_suite().await;
let total_tests = results.len();
let failed_count = results.iter().filter(|r| !r.success).count();
let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
info!("📊 Full suite success rate: {:.1}%", success_rate);
// Allow up to 10% failure rate for non-critical tests
if success_rate < 90.0 {
return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
}
info!("✅ Full test suite succeeded");
Ok(())
}
crates/e2e_test/src/object_lambda_test.rs
+25 -12
View File
@@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
use aws_sdk_s3::primitives::ByteStream;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
use rustfs_signer::pre_sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use s3s::Body;
@@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let bucket = "object-lambda-e2e-disabled-target";
let key = "input.txt";
@@ -992,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
.send()
.await?;
configure_webhook_target_with_key_values(
&env,
"transformer",
vec![
("endpoint", "http://127.0.0.1:9/transform".to_string()),
("auth_token", "secret-token".to_string()),
("enable", "off".to_string()),
],
let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
tokio::fs::create_dir_all(&queue_dir).await?;
let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
let directive = format!(
"notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
);
let disable_response = signed_request(
http::Method::PUT,
&config_url,
&env.access_key,
&env.secret_key,
Some(directive.into_bytes()),
Some("text/plain"),
)
.await?;
wait_for_target_visibility(&env, "transformer").await?;
let disable_status = disable_response.status();
let disable_body = disable_response.text().await?;
assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
@@ -1021,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let bucket = "object-lambda-e2e-invalid-endpoint";
@@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
.await?;
let response = send_configure_webhook_target_request(
&env,
@@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
cluster.set_env(ENV_NOTIFY_ENABLE, "true");
cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
cluster.start().await?;
let bucket = "listen-notification-cluster";
crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs
+13 -24
View File
@@ -15,7 +15,6 @@
use crate::common::{RustFSTestClusterEnvironment, init_logging};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::CompletedMultipartUpload;
use tokio::time::{Duration, sleep};
use tracing::info;
use uuid::Uuid;
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
}
}
async fn complete_reports_missing_upload(
async fn multipart_listing_reports_missing_upload(
client: &aws_sdk_s3::Client,
bucket: &str,
key: &str,
upload_id: &str,
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
let result = client
.complete_multipart_upload()
.bucket(bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(CompletedMultipartUpload::builder().build())
.send()
.await;
match result {
Ok(_) => Ok(false),
Err(SdkError::ServiceError(err)) => {
let code = err.err().meta().code().unwrap_or("");
if code == "NoSuchUpload" {
Ok(true)
} else {
Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
}
}
Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
}
let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
Ok(!result
.uploads()
.iter()
.any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
}
async fn wait_for_cleanup_on_all_nodes(
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
let mut all_cleaned = true;
for (idx, client) in clients.iter().enumerate() {
let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && complete_missing) {
let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
if !(list_parts_missing && listing_missing) {
info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
all_cleaned = false;
break;
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
1,
"multipart upload should be visible before background cleanup"
);
assert!(
!multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
"multipart upload listing should contain the upload before background cleanup"
);
wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;