feat(s3): advance parity coverage (#2278)

crates/e2e_test/src/kms/common.rs

@@ -22,11 +22,13 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities
 
-use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging};
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
+};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
-use base64::Engine;
+use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
 use serde_json;
 use std::process::{Child, Command};
 use std::time::Duration;
@@ -51,6 +53,19 @@ pub fn init_logging() {
     // Additional KMS-specific logging configuration can be added here if needed
 }
 
+pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
+    if awscurl_available() {
+        return false;
+    }
+
+    info!("Skipping {} because awscurl is not available in PATH", test_name);
+    true
+}
+
+pub fn sse_customer_key_md5_base64(key: &str) -> String {
+    BASE64.encode(md5::compute(key).0)
+}
+
 // KMS-specific helper functions
 /// Configure KMS backend via admin API
 pub async fn configure_kms(
@@ -133,10 +148,10 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
         "usage": "EncryptDecrypt",
         "status": "Active",
         "metadata": HashMap::<String, String>::new(),
-        "created_at": chrono::Utc::now().to_rfc3339(),
+        "created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
         "rotated_at": serde_json::Value::Null,
         "created_by": "e2e-test",
-        "encrypted_key_material": key_data.to_vec(),
+        "encrypted_key_material": BASE64.encode(key_data),
         "nonce": Vec::<u8>::new()
     });
 
@@ -155,7 +170,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
 
     let test_key = "01234567890123456789012345678901"; // 32-byte key
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let test_data = b"Hello, KMS SSE-C World!";
     let object_key = "test-sse-c-object";
 
@@ -272,6 +287,10 @@ pub async fn test_kms_key_management(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
+        return Ok(());
+    }
+
     info!("Testing KMS key management APIs");
 
     // Test CreateKey
@@ -324,8 +343,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
     let wrong_key = "98765432109876543210987654321098";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
     let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
     let test_data = b"Test data for error scenarios";
     let object_key = "test-error-object";
 
@@ -406,7 +425,7 @@ impl VaultTestEnvironment {
             let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
             if port_check {
                 // Additional check by making a health request
-                if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await
+                if let Ok(response) = local_http_client().get(format!("{VAULT_URL}/v1/sys/health")).send().await
                     && response.status().is_success()
                 {
                     info!("Vault server is ready after {} seconds", i);
@@ -426,7 +445,7 @@ impl VaultTestEnvironment {
 
     /// Setup Vault transit secrets engine
     pub async fn setup_vault_transit(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let client = reqwest::Client::new();
+        let client = local_http_client();
 
         info!("Enabling Vault transit secrets engine");
 
@@ -687,7 +706,7 @@ pub async fn test_multipart_upload_with_config(
 /// Create a standard SSE-C encryption configuration for testing
 pub fn create_sse_c_config() -> EncryptionType {
     let key = "01234567890123456789012345678901"; // 32-byte key
-    let key_md5 = format!("{:x}", md5::compute(key));
+    let key_md5 = sse_customer_key_md5_base64(key);
     EncryptionType::SSEC {
         key: key.to_string(),
         key_md5,

crates/e2e_test/src/kms/encryption_metadata_test.rs

@@ -26,27 +26,20 @@ use serial_test::serial;
 use std::collections::{HashMap, VecDeque};
 use tracing::info;
 
-fn assert_encryption_metadata(metadata: &HashMap<String, String>, expected_size: usize) {
+fn assert_managed_encryption_metadata_hidden(metadata: Option<&HashMap<String, String>>) {
+    let Some(metadata) = metadata else { return };
+
     for key in [
         "x-rustfs-encryption-key",
         "x-rustfs-encryption-iv",
         "x-rustfs-encryption-context",
         "x-rustfs-encryption-original-size",
     ] {
-        assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present");
         assert!(
-            !metadata.get(key).unwrap().is_empty(),
-            "managed encryption metadata '{key}' should not be empty"
+            !metadata.contains_key(key),
+            "managed encryption metadata '{key}' should not be exposed to clients"
         );
     }
-
-    let size_value = metadata
-        .get("x-rustfs-encryption-original-size")
-        .expect("managed encryption metadata should include original size");
-    let parsed_size: usize = size_value
-        .parse()
-        .expect("x-rustfs-encryption-original-size should be numeric");
-    assert_eq!(parsed_size, expected_size, "recorded original size should match uploaded payload length");
 }
 
 fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &str, plaintext: &[u8]) {
@@ -142,10 +135,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
         "head_object should advertise SSE-S3"
     );
 
-    let metadata = head
-        .metadata()
-        .expect("head_object should return managed encryption metadata");
-    assert_encryption_metadata(metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     assert_storage_encrypted(std::path::Path::new(&kms_env.base_env.temp_dir), TEST_BUCKET, key, payload);
 
@@ -210,10 +200,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "source object should maintain the configured KMS key id"
     );
-    let source_metadata = head_source
-        .metadata()
-        .expect("source object should include managed encryption metadata");
-    assert_encryption_metadata(source_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_source.metadata());
 
     let dest_key = "metadata-sse-kms-object-copy";
     let copy_source = format!("{TEST_BUCKET}/{source_key}");
@@ -238,10 +225,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "copied object should keep the default KMS key id"
     );
-    let dest_metadata = head_dest
-        .metadata()
-        .expect("copied object should include managed encryption metadata");
-    assert_encryption_metadata(dest_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_dest.metadata());
 
     let copied_body = s3_client
         .get_object()
@@ -358,10 +342,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
         "multipart object should retain bucket default KMS key"
     );
 
-    assert_encryption_metadata(
-        head.metadata().expect("multipart head_object should expose managed metadata"),
-        combined.len(),
-    );
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     // Data returned to clients should decrypt back to original payload
     let fetched = s3_client

crates/e2e_test/src/kms/kms_comprehensive_test.rs

@@ -19,9 +19,9 @@
 //! complex workflows.
 
 use super::common::{
-    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, test_all_multipart_encryption_types,
-    test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption, test_sse_kms_encryption,
-    test_sse_s3_encryption,
+    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, sse_customer_key_md5_base64,
+    test_all_multipart_encryption_types, test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
@@ -149,8 +149,8 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     // Test different SSE-C keys to ensure isolation
     let key1 = "01234567890123456789012345678901";
     let key2 = "98765432109876543210987654321098";
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let config1 = MultipartTestConfig::new(
         "isolation-test-key1",
@@ -183,7 +183,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     info!("🔒 Verify key isolation");
     let wrong_key = "11111111111111111111111111111111";
     let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
 
     // Try to read file encrypted with key1 using wrong key
     let wrong_read_result = s3_client

crates/e2e_test/src/kms/kms_edge_cases_test.rs

@@ -21,7 +21,7 @@
 //! - Concurrent encryption operations
 //! - Security validation tests
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use aws_sdk_s3::types::ServerSideEncryption;
 use base64::Engine;
@@ -72,7 +72,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
     info!("📤 Testing SSE-C with zero-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "zero-byte-sse-c";
 
     let _put_response_c = s3_client
@@ -166,7 +166,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
     info!("📤 Testing SSE-C with single-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "single-byte-sse-c";
 
     s3_client
@@ -334,7 +334,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
     info!("🔍 Testing access to SSE-C object without key");
 
     // First upload a valid SSE-C object
-    let valid_key_md5 = format!("{:x}", compute(valid_key));
+    let valid_key_md5 = sse_customer_key_md5_base64(valid_key);
     s3_client
         .put_object()
         .bucket(TEST_BUCKET)
@@ -421,7 +421,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
                     // SSE-C
                     let key = format!("testkey{i:026}"); // 32-byte key
                     let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
-                    let key_md5 = format!("{:x}", compute(&key));
+                    let key_md5 = sse_customer_key_md5_base64(&key);
 
                     client
                         .put_object()
@@ -493,8 +493,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
 
     let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
     let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     // Upload same data with different keys
     s3_client

crates/e2e_test/src/kms/kms_local_test.rs

@@ -19,7 +19,10 @@
 //! - S3 object upload/download with SSE-S3, SSE-KMS, SSE-C encryption
 //! - Complete encryption/decryption lifecycle
 
-use super::common::{LocalKMSTestEnvironment, get_kms_status, test_kms_key_management, test_sse_c_encryption};
+use super::common::{
+    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    test_kms_key_management, test_sse_c_encryption,
+};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{error, info};
@@ -28,6 +31,9 @@ use tracing::{error, info};
 #[serial]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Local KMS End-to-End Test");
 
     // Create LocalKMS test environment
@@ -140,8 +146,8 @@ async fn test_local_kms_key_isolation() {
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Data encrypted with key 1";
     let data2 = b"Data encrypted with key 2";
@@ -562,7 +568,7 @@ async fn test_multipart_upload_with_sse_c(
     // SSE-C encryption key
     let encryption_key = "01234567890123456789012345678901";
     let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
-    let key_md5 = format!("{:x}", md5::compute(encryption_key));
+    let key_md5 = sse_customer_key_md5_base64(encryption_key);
 
     // Generate test data
     let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 3) % 256) as u8).collect();

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -19,14 +19,14 @@
 //! multipart upload behaviour.
 
 use crate::common::{TEST_BUCKET, init_logging};
-use md5::compute;
 use serial_test::serial;
 use tokio::time::{Duration, sleep};
 use tracing::{error, info};
 
 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, start_kms, test_all_multipart_encryption_types, test_error_scenarios,
-    test_kms_key_management, test_sse_c_encryption, test_sse_kms_encryption, test_sse_s3_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 
 /// Helper that brings up Vault, configures RustFS, and starts the KMS service.
@@ -65,6 +65,9 @@ impl VaultKmsTestContext {
 #[serial]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
 
     let context = VaultKmsTestContext::new().await?;
@@ -118,6 +121,9 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
+        return Ok(());
+    }
     info!("Starting Vault KMS SSE-C key isolation test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -133,8 +139,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Vault data encrypted with key 1";
     let data2 = b"Vault data encrypted with key 2";
@@ -202,6 +208,9 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[serial]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
+        return Ok(());
+    }
     info!("Starting Vault KMS large file SSE-S3 test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -264,6 +273,9 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
+        return Ok(());
+    }
     info!("Starting Vault KMS multipart upload encryption suite");
 
     let context = VaultKmsTestContext::new().await?;
@@ -292,6 +304,9 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[serial]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
+        return Ok(());
+    }
     info!("Starting Vault KMS key operations test (CRUD)");
 
     let context = VaultKmsTestContext::new().await?;

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -21,7 +21,7 @@
 //! 3. Test the saving and reading of encrypted metadata
 //! 4. Test the complete sharded upload encryption process
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{debug, info};
@@ -504,7 +504,7 @@ async fn test_multipart_encryption_type(
     let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
         let key = "01234567890123456789012345678901";
         let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
-        let key_md5 = format!("{:x}", md5::compute(key));
+        let key_md5 = sse_customer_key_md5_base64(key);
         (Some(key_b64), Some(key_md5))
     } else {
         (None, None)