diff --git a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs index fecba2b89..c0f0f0181 100644 --- a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs +++ b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs @@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { + let total_deadline = Duration::from_secs(5); + let start = tokio::time::Instant::now(); + let mut backoff = Duration::from_millis(200); + let max_backoff = Duration::from_secs(1); + let mut first_attempt = true; + + loop { + if !first_attempt { + if start.elapsed() >= total_deadline { + return Err("KMS failed to become ready within 5 seconds".into()); + } + sleep(backoff).await; + backoff = (backoff * 2).min(max_backoff); + } + first_attempt = false; + + match get_kms_status(base_url, access_key, secret_key).await { + Ok(status) => { + info!("KMS is ready (status: {})", status); + return Ok(()); + } + Err(e) => { + if start.elapsed() >= total_deadline { + return Err(format!("KMS did not become ready within 5 s: last error: {e}").into()); + } + warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…"); + } + } + } +} + /// Create a default KMS key for testing and return the created key ID pub async fn create_default_key( base_url: &str, @@ -861,6 +904,13 @@ impl LocalKMSTestEnvironment { Ok(default_key_id.to_string()) } + /// Poll the KMS status endpoint until the backend reports ready. + /// + /// Prefer this over a fixed `sleep` after calling `start_rustfs_for_local_kms`. + pub async fn wait_for_kms_ready(&self) -> Result<(), Box> { + wait_for_kms_ready(&self.base_env.url, &self.base_env.access_key, &self.base_env.secret_key).await + } + /// Configure Local KMS backend with a predefined default key pub async fn configure_local_kms(&self) -> Result> { // Use a fixed, predictable default key ID diff --git a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs index 1cf19a565..1e633679d 100644 --- a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs @@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let client = kms_env.base_env.create_s3_client(); // Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service @@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let client = kms_env.base_env.create_s3_client(); // Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below @@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let client = kms_env.base_env.create_s3_client(); let bucket = "copy-object-self-copy-bucket-default-sse-test"; diff --git a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs index 3241a217d..3edaaa665 100644 --- a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs @@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let client = kms_env.base_env.create_s3_client(); let bucket = "copy-object-version-restore-sse-test"; diff --git a/crates/e2e_test/src/kms/encryption_metadata_test.rs b/crates/e2e_test/src/kms/encryption_metadata_test.rs index a316668f6..59501430b 100644 --- a/crates/e2e_test/src/kms/encryption_metadata_test.rs +++ b/crates/e2e_test/src/kms/encryption_metadata_test.rs @@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box Result<(), let mut kms_env = LocalKMSTestEnvironment::new().await?; let default_key_id = kms_env.start_rustfs_for_local_kms().await?; - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let s3_client = kms_env.base_env.create_s3_client(); kms_env.base_env.create_test_bucket(TEST_BUCKET).await?; @@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Bo let mut kms_env = LocalKMSTestEnvironment::new().await?; let _default_key_id = kms_env.start_rustfs_for_local_kms().await?; - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let s3_client = kms_env.base_env.create_s3_client(); kms_env.base_env.create_test_bucket(TEST_BUCKET).await?; @@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box Result<(), Box Result<(), Box