feat(s3): advance parity coverage (#2278)

crates/e2e_test/src/anonymous_access_test.rs

@@ -16,7 +16,7 @@
 //! Verifies that anonymous access works correctly with bucket policies
 //! when PublicAccessBlock configuration is missing or explicitly set.
 
-use crate::common::{RustFSTestEnvironment, init_logging};
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
 use aws_sdk_s3::types::PublicAccessBlockConfiguration;
 use serial_test::serial;
 use tracing::info;
@@ -67,7 +67,7 @@ async fn anonymous_get_object(
     key: &str,
 ) -> Result<reqwest::Response, reqwest::Error> {
     let url = format!("{}/{}/{}", env.url, bucket_name, key);
-    reqwest::Client::new().get(&url).send().await
+    local_http_client().get(&url).send().await
 }
 
 /// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it

crates/e2e_test/src/bucket_logging_test.rs

@@ -0,0 +1,544 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! End-to-end tests for S3 dummy-compat bucket APIs.
+
+#[cfg(test)]
+mod tests {
+    use crate::common::{RustFSTestEnvironment, init_logging};
+    use aws_sdk_s3::error::ProvideErrorMetadata;
+    use aws_sdk_s3::types::{
+        AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer,
+        RequestPaymentConfiguration, WebsiteConfiguration,
+    };
+    use serial_test::serial;
+    use std::path::PathBuf;
+    use std::process::Command;
+    use tracing::info;
+
+    fn awscurl_binary_path() -> PathBuf {
+        std::env::var_os("AWSCURL_PATH")
+            .map(PathBuf::from)
+            .unwrap_or_else(|| PathBuf::from("awscurl"))
+    }
+
+    fn awscurl_available() -> bool {
+        Command::new(awscurl_binary_path()).arg("--version").output().is_ok()
+    }
+
+    fn execute_s3_awscurl(
+        method: &str,
+        url: &str,
+        access_key: &str,
+        secret_key: &str,
+    ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
+        let output = Command::new(awscurl_binary_path())
+            .args([
+                "--service",
+                "s3",
+                "--region",
+                "us-east-1",
+                "--access_key",
+                access_key,
+                "--secret_key",
+                secret_key,
+                "-i",
+                "-X",
+                method,
+                url,
+            ])
+            .output()?;
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            return Err(format!("awscurl failed: stderr='{stderr}', stdout='{stdout}'").into());
+        }
+        Ok(String::from_utf8_lossy(&output.stdout).to_string())
+    }
+
+    fn parse_status(raw: &str) -> Option<u16> {
+        raw.lines()
+            .filter_map(|line| {
+                if line.starts_with("HTTP/") {
+                    line.split_whitespace().nth(1)?.parse::<u16>().ok()
+                } else {
+                    None
+                }
+            })
+            .next_back()
+    }
+
+    fn parse_body(raw: &str) -> String {
+        if let Some(pos) = raw.rfind("\r\n\r\n") {
+            return raw[pos + 4..].to_string();
+        }
+        if let Some(pos) = raw.rfind("\n\n") {
+            return raw[pos + 2..].to_string();
+        }
+        String::new()
+    }
+
+    fn parse_headers(raw: &str) -> String {
+        let start = raw.rfind("HTTP/").unwrap_or(0);
+        let tail = &raw[start..];
+        if let Some(pos) = tail.find("\r\n\r\n") {
+            return tail[..pos].to_string();
+        }
+        if let Some(pos) = tail.find("\n\n") {
+            return tail[..pos].to_string();
+        }
+        tail.to_string()
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_compatibility_endpoints() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "test-get-bucket-logging";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        let result = client.get_bucket_logging().bucket(bucket).send().await;
+        assert!(
+            result.is_ok(),
+            "GetBucketLogging should return success for existing bucket, got: {:?}",
+            result.err()
+        );
+
+        let output = result.unwrap();
+        assert!(
+            output.logging_enabled().is_none(),
+            "Default GetBucketLogging should return empty logging configuration"
+        );
+
+        let put_logging = client
+            .put_bucket_logging()
+            .bucket(bucket)
+            .bucket_logging_status(
+                BucketLoggingStatus::builder()
+                    .logging_enabled(
+                        LoggingEnabled::builder()
+                            .target_bucket(bucket)
+                            .target_prefix("logs/")
+                            .build()
+                            .expect("failed to build LoggingEnabled"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_logging.is_ok(),
+            "PutBucketLogging should return success for existing bucket, got: {:?}",
+            put_logging.err()
+        );
+
+        let output_after_put = client
+            .get_bucket_logging()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketLogging should succeed after PutBucketLogging");
+        let logging_after_put = output_after_put
+            .logging_enabled()
+            .expect("GetBucketLogging should return persisted logging_enabled");
+        assert_eq!(
+            logging_after_put.target_bucket(),
+            bucket,
+            "GetBucketLogging should preserve target bucket"
+        );
+        assert_eq!(
+            logging_after_put.target_prefix(),
+            "logs/",
+            "GetBucketLogging should preserve target prefix"
+        );
+
+        let accelerate = client
+            .get_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketAccelerateConfiguration should succeed");
+        assert!(
+            accelerate.status().is_none(),
+            "Default GetBucketAccelerateConfiguration should return empty status"
+        );
+
+        let payment = client
+            .get_bucket_request_payment()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketRequestPayment should succeed");
+        assert_eq!(
+            payment.payer().map(|p| p.as_str()),
+            Some("BucketOwner"),
+            "GetBucketRequestPayment should return BucketOwner by default"
+        );
+
+        let put_accelerate = client
+            .put_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .accelerate_configuration(
+                AccelerateConfiguration::builder()
+                    .status(BucketAccelerateStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_accelerate.is_ok(),
+            "PutBucketAccelerateConfiguration should return success for existing bucket, got: {:?}",
+            put_accelerate.err()
+        );
+
+        let put_request_payment = client
+            .put_bucket_request_payment()
+            .bucket(bucket)
+            .request_payment_configuration(
+                RequestPaymentConfiguration::builder()
+                    .payer(Payer::Requester)
+                    .build()
+                    .expect("failed to build RequestPaymentConfiguration"),
+            )
+            .send()
+            .await;
+        assert!(
+            put_request_payment.is_ok(),
+            "PutBucketRequestPayment should return success for existing bucket, got: {:?}",
+            put_request_payment.err()
+        );
+
+        let accelerate_after_put = client
+            .get_bucket_accelerate_configuration()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketAccelerateConfiguration should succeed after put");
+        assert_eq!(
+            accelerate_after_put.status().map(|s| s.as_str()),
+            Some("Suspended"),
+            "GetBucketAccelerateConfiguration should preserve put status"
+        );
+
+        let payment_after_put = client
+            .get_bucket_request_payment()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("GetBucketRequestPayment should succeed after put");
+        assert_eq!(
+            payment_after_put.payer().map(|p| p.as_str()),
+            Some("Requester"),
+            "GetBucketRequestPayment should preserve put payer"
+        );
+
+        let put_website = client
+            .put_bucket_website()
+            .bucket(bucket)
+            .website_configuration(
+                WebsiteConfiguration::builder()
+                    .index_document(
+                        IndexDocument::builder()
+                            .suffix("index.html")
+                            .build()
+                            .expect("failed to build IndexDocument"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(
+            put_website.is_ok(),
+            "PutBucketWebsite should return success for existing bucket, got: {:?}",
+            put_website.err()
+        );
+
+        let website = client.get_bucket_website().bucket(bucket).send().await;
+        assert!(website.is_ok(), "GetBucketWebsite should return persisted website configuration");
+        let website_output = website.unwrap();
+        assert_eq!(
+            website_output.index_document().map(|doc| doc.suffix()),
+            Some("index.html"),
+            "GetBucketWebsite should preserve index document suffix"
+        );
+
+        client
+            .delete_bucket_website()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("DeleteBucketWebsite should return success");
+
+        let website_after_delete = client.get_bucket_website().bucket(bucket).send().await;
+        assert!(
+            website_after_delete.is_err(),
+            "GetBucketWebsite should return NoSuchWebsiteConfiguration after deletion"
+        );
+        let website_err = website_after_delete.err().unwrap();
+        let website_code = website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(website_code, Some("NoSuchWebsiteConfiguration")),
+            "Unexpected GetBucketWebsite error code: {:?}, err: {:?}",
+            website_code,
+            website_err
+        );
+
+        env.stop_server();
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let missing_bucket = "test-dummy-bucket-missing";
+
+        let get_logging = client.get_bucket_logging().bucket(missing_bucket).send().await;
+        assert!(get_logging.is_err(), "GetBucketLogging should fail for missing bucket");
+        let get_logging_err = get_logging.err().unwrap();
+        let get_logging_code = get_logging_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_logging_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketLogging error code: {:?}, err: {:?}",
+            get_logging_code,
+            get_logging_err
+        );
+
+        let put_logging = client
+            .put_bucket_logging()
+            .bucket(missing_bucket)
+            .bucket_logging_status(BucketLoggingStatus::builder().build())
+            .send()
+            .await;
+        assert!(put_logging.is_err(), "PutBucketLogging should fail for missing bucket");
+        let put_logging_err = put_logging.err().unwrap();
+        let put_logging_code = put_logging_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_logging_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketLogging error code: {:?}, err: {:?}",
+            put_logging_code,
+            put_logging_err
+        );
+
+        let get_accelerate = client
+            .get_bucket_accelerate_configuration()
+            .bucket(missing_bucket)
+            .send()
+            .await;
+        assert!(get_accelerate.is_err(), "GetBucketAccelerateConfiguration should fail for missing bucket");
+        let get_accelerate_err = get_accelerate.err().unwrap();
+        let get_accelerate_code = get_accelerate_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_accelerate_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketAccelerateConfiguration error code: {:?}, err: {:?}",
+            get_accelerate_code,
+            get_accelerate_err
+        );
+
+        let get_request_payment = client.get_bucket_request_payment().bucket(missing_bucket).send().await;
+        assert!(get_request_payment.is_err(), "GetBucketRequestPayment should fail for missing bucket");
+        let get_request_payment_err = get_request_payment.err().unwrap();
+        let get_request_payment_code = get_request_payment_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_request_payment_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketRequestPayment error code: {:?}, err: {:?}",
+            get_request_payment_code,
+            get_request_payment_err
+        );
+
+        let put_accelerate = client
+            .put_bucket_accelerate_configuration()
+            .bucket(missing_bucket)
+            .accelerate_configuration(
+                AccelerateConfiguration::builder()
+                    .status(BucketAccelerateStatus::Suspended)
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(put_accelerate.is_err(), "PutBucketAccelerateConfiguration should fail for missing bucket");
+        let put_accelerate_err = put_accelerate.err().unwrap();
+        let put_accelerate_code = put_accelerate_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_accelerate_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketAccelerateConfiguration error code: {:?}, err: {:?}",
+            put_accelerate_code,
+            put_accelerate_err
+        );
+
+        let put_request_payment = client
+            .put_bucket_request_payment()
+            .bucket(missing_bucket)
+            .request_payment_configuration(
+                RequestPaymentConfiguration::builder()
+                    .payer(Payer::BucketOwner)
+                    .build()
+                    .expect("failed to build RequestPaymentConfiguration"),
+            )
+            .send()
+            .await;
+        assert!(put_request_payment.is_err(), "PutBucketRequestPayment should fail for missing bucket");
+        let put_request_payment_err = put_request_payment.err().unwrap();
+        let put_request_payment_code = put_request_payment_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_request_payment_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketRequestPayment error code: {:?}, err: {:?}",
+            put_request_payment_code,
+            put_request_payment_err
+        );
+
+        let put_website = client
+            .put_bucket_website()
+            .bucket(missing_bucket)
+            .website_configuration(
+                WebsiteConfiguration::builder()
+                    .index_document(
+                        IndexDocument::builder()
+                            .suffix("index.html")
+                            .build()
+                            .expect("failed to build IndexDocument"),
+                    )
+                    .build(),
+            )
+            .send()
+            .await;
+        assert!(put_website.is_err(), "PutBucketWebsite should fail for missing bucket");
+        let put_website_err = put_website.err().unwrap();
+        let put_website_code = put_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(put_website_code, Some("NoSuchBucket")),
+            "Unexpected PutBucketWebsite error code: {:?}, err: {:?}",
+            put_website_code,
+            put_website_err
+        );
+
+        let get_website = client.get_bucket_website().bucket(missing_bucket).send().await;
+        assert!(get_website.is_err(), "GetBucketWebsite should fail for missing bucket");
+        let get_website_err = get_website.err().unwrap();
+        let get_website_code = get_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(get_website_code, Some("NoSuchBucket")),
+            "Unexpected GetBucketWebsite error code: {:?}, err: {:?}",
+            get_website_code,
+            get_website_err
+        );
+
+        let delete_website = client.delete_bucket_website().bucket(missing_bucket).send().await;
+        assert!(delete_website.is_err(), "DeleteBucketWebsite should fail for missing bucket");
+        let delete_website_err = delete_website.err().unwrap();
+        let delete_website_code = delete_website_err.as_service_error().and_then(|e| e.code());
+        assert!(
+            matches!(delete_website_code, Some("NoSuchBucket")),
+            "Unexpected DeleteBucketWebsite error code: {:?}, err: {:?}",
+            delete_website_code,
+            delete_website_err
+        );
+
+        env.stop_server();
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn test_dummy_bucket_endpoints_http_contracts() {
+        init_logging();
+        info!("Starting test: dummy-compat bucket API HTTP contracts");
+        if !awscurl_available() {
+            info!("Skipping test_dummy_bucket_endpoints_http_contracts: awscurl binary not found");
+            return;
+        }
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = env.create_s3_client();
+        let bucket = "test-dummy-bucket-http-contracts";
+
+        client
+            .create_bucket()
+            .bucket(bucket)
+            .send()
+            .await
+            .expect("Failed to create bucket");
+
+        let logging_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?logging=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketLogging HTTP request failed");
+        assert_eq!(parse_status(&logging_raw), Some(200), "GetBucketLogging should return 200");
+        let logging_body = parse_body(&logging_raw);
+        assert!(
+            logging_body.contains("<BucketLoggingStatus"),
+            "GetBucketLogging response should contain BucketLoggingStatus XML, got: {logging_body}"
+        );
+
+        let accel_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?accelerate=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketAccelerateConfiguration HTTP request failed");
+        assert_eq!(parse_status(&accel_raw), Some(200), "GetBucketAccelerateConfiguration should return 200");
+        let accel_body = parse_body(&accel_raw);
+        assert!(
+            accel_body.contains("<AccelerateConfiguration"),
+            "GetBucketAccelerateConfiguration response should contain AccelerateConfiguration XML, got: {accel_body}"
+        );
+
+        let payment_raw =
+            execute_s3_awscurl("GET", &format!("{}/{bucket}?requestPayment=", env.url), &env.access_key, &env.secret_key)
+                .expect("GetBucketRequestPayment HTTP request failed");
+        assert_eq!(parse_status(&payment_raw), Some(200), "GetBucketRequestPayment should return 200");
+        let payment_body = parse_body(&payment_raw);
+        assert!(
+            payment_body.contains("<Payer>BucketOwner</Payer>"),
+            "GetBucketRequestPayment should return BucketOwner payer, got: {payment_body}"
+        );
+
+        let website_raw = execute_s3_awscurl("GET", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+            .expect("GetBucketWebsite HTTP request failed");
+        assert_eq!(
+            parse_status(&website_raw),
+            Some(404),
+            "GetBucketWebsite should return 404 when website config is absent"
+        );
+        let website_content_type = parse_headers(&website_raw).to_ascii_lowercase();
+        assert!(
+            website_content_type.contains("content-type:") && website_content_type.contains("xml"),
+            "GetBucketWebsite error response should be XML, got content-type: {website_content_type}"
+        );
+        let website_body = parse_body(&website_raw);
+        assert!(
+            website_body.contains("<Code>NoSuchWebsiteConfiguration</Code>"),
+            "GetBucketWebsite should return NoSuchWebsiteConfiguration code, got: {website_body}"
+        );
+
+        let delete_raw =
+            execute_s3_awscurl("DELETE", &format!("{}/{bucket}?website=", env.url), &env.access_key, &env.secret_key)
+                .expect("DeleteBucketWebsite HTTP request failed");
+        assert_eq!(parse_status(&delete_raw), Some(204), "DeleteBucketWebsite should return 204");
+
+        env.stop_server();
+    }
+}

crates/e2e_test/src/bucket_policy_check_test.rs

@@ -54,6 +54,10 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key:
 #[serial]
 async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if !crate::common::awscurl_available() {
+        info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available");
+        return Ok(());
+    }
     info!("Starting test_bucket_policy_authenticated_user...");
 
     let mut env = RustFSTestEnvironment::new().await?;

crates/e2e_test/src/common.rs

@@ -23,7 +23,11 @@
 
 use aws_sdk_s3::config::{Credentials, Region};
 use aws_sdk_s3::{Client, Config};
-use std::path::PathBuf;
+use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
+use reqwest::Client as HttpClient;
+use std::ffi::OsStr;
+use std::fs as stdfs;
+use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use std::sync::Once;
 use std::time::Duration;
@@ -32,11 +36,29 @@ use tokio::net::TcpStream;
 use tokio::time::sleep;
 use tracing::{error, info, warn};
 use uuid::Uuid;
+use walkdir::WalkDir;
 
 // Common constants for all E2E tests
 pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
 pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
 pub const TEST_BUCKET: &str = "e2e-test-bucket";
+
+fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
+    let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
+    let mut config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(endpoint_url)
+        .force_path_style(true)
+        .behavior_version_latest();
+
+    if endpoint_url.starts_with("http://") {
+        config = config.http_client(SmithyHttpClientBuilder::new().build_http());
+    }
+
+    config.build()
+}
+
 pub fn workspace_root() -> PathBuf {
     let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     path.pop(); // e2e_test
@@ -44,29 +66,125 @@ pub fn workspace_root() -> PathBuf {
     path
 }
 
+pub fn local_http_client() -> HttpClient {
+    HttpClient::builder()
+        .no_proxy()
+        .build()
+        .expect("failed to build local reqwest client")
+}
+
 /// Resolve the RustFS binary relative to the workspace.
-/// Always builds the binary to ensure it's up to date.
 pub fn rustfs_binary_path() -> PathBuf {
+    rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())
+}
+
+/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
+pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
     if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
         return PathBuf::from(path);
     }
 
-    // Always build the binary to ensure it's up to date
-    info!("Building RustFS binary to ensure it's up to date...");
-    build_rustfs_binary();
-
     let mut binary_path = workspace_root();
     binary_path.push("target");
     let profile_dir = if cfg!(debug_assertions) { "debug" } else { "release" };
     binary_path.push(profile_dir);
     binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
 
+    let features_match = binary_features_match(&binary_path, requested_features);
+    let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
+    let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match;
+    if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) {
+        if source_is_newer {
+            warn!(
+                "RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds",
+                binary_path
+            );
+        }
+        info!("Using existing RustFS binary at {:?}", binary_path);
+        return binary_path;
+    }
+
+    info!("Building RustFS binary to ensure it's up to date...");
+    build_rustfs_binary(requested_features);
+
     info!("Using RustFS binary at {:?}", binary_path);
     binary_path
 }
 
+fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool {
+    let Ok(binary_meta) = std::fs::metadata(binary_path) else {
+        return true;
+    };
+    let Ok(binary_modified) = binary_meta.modified() else {
+        return true;
+    };
+
+    let workspace = workspace_root();
+    let watch_roots = [
+        workspace.join("Cargo.toml"),
+        workspace.join("Cargo.lock"),
+        workspace.join("rustfs"),
+        workspace.join("crates"),
+    ];
+
+    watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path))
+}
+
+fn running_inside_e2e_test_binary() -> bool {
+    std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test")
+}
+
+fn requested_rustfs_build_features() -> Option<String> {
+    std::env::var("RUSTFS_BUILD_FEATURES")
+        .ok()
+        .map(|value| value.trim().to_string())
+        .filter(|value| !value.is_empty())
+}
+
+fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf {
+    binary_path.with_extension("features")
+}
+
+fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool {
+    let stamp_path = rustfs_binary_features_stamp_path(binary_path);
+    let recorded = stdfs::read_to_string(stamp_path).ok().map(|value| value.trim().to_string());
+
+    match requested_features {
+        Some(features) => recorded.as_deref() == Some(features),
+        None => recorded.as_deref().is_none_or(str::is_empty),
+    }
+}
+
+fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool {
+    if path.is_file() {
+        return std::fs::metadata(path)
+            .and_then(|meta| meta.modified())
+            .map(|modified| modified > binary_modified)
+            .unwrap_or(false);
+    }
+
+    if !path.is_dir() {
+        return false;
+    }
+
+    WalkDir::new(path)
+        .into_iter()
+        .filter_entry(|entry| {
+            let name = entry.file_name();
+            name != OsStr::new("target") && name != OsStr::new(".git")
+        })
+        .filter_map(Result::ok)
+        .filter(|entry| entry.file_type().is_file())
+        .any(|entry| {
+            std::fs::metadata(entry.path())
+                .and_then(|meta| meta.modified())
+                .map(|modified| modified > binary_modified)
+                .unwrap_or(false)
+        })
+}
+
 /// Build the RustFS binary using cargo
-fn build_rustfs_binary() {
+fn build_rustfs_binary(requested_features: Option<&str>) {
     let workspace = workspace_root();
     info!("Building RustFS binary from workspace: {:?}", workspace);
 
@@ -81,11 +199,8 @@ fn build_rustfs_binary() {
     let mut cmd = Command::new("cargo");
     cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);
 
-    // Read features from environment variable for e2e tests
-    if let Ok(features) = std::env::var("RUSTFS_BUILD_FEATURES")
-        && !features.is_empty()
-    {
-        cmd.arg("--features").arg(&features);
+    if let Some(features) = requested_features {
+        cmd.arg("--features").arg(features);
         info!("Building with features: {}", features);
     }
 
@@ -105,6 +220,15 @@ fn build_rustfs_binary() {
         panic!("Failed to build RustFS binary. Error: {stderr}");
     }
 
+    let mut binary_path = workspace;
+    binary_path.push("target");
+    binary_path.push(if cfg!(debug_assertions) { "debug" } else { "release" });
+    binary_path.push(format!("rustfs{}", std::env::consts::EXE_SUFFIX));
+    let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
+    if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
+        warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
+    }
+
     info!("✅ RustFS binary built successfully");
 }
 
@@ -114,6 +238,17 @@ fn awscurl_binary_path() -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("awscurl"))
 }
 
+pub fn awscurl_available() -> bool {
+    let path = awscurl_binary_path();
+    if path.components().count() > 1 || path.is_absolute() {
+        return path.is_file();
+    }
+
+    std::env::var_os("PATH")
+        .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file()))
+        .unwrap_or(false)
+}
+
 // Global initialization
 static INIT: Once = Once::new();
 
@@ -183,24 +318,22 @@ impl RustFSTestEnvironment {
 
     /// Kill any existing RustFS processes
     pub async fn cleanup_existing_processes(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        info!("Cleaning up any existing RustFS processes");
-        let binary_path = rustfs_binary_path();
-        let binary_name = binary_path.to_string_lossy();
-        let output = Command::new("pkill").args(["-f", &binary_name]).output();
+        info!("Cleaning up any existing RustFS processes for {}", self.address);
 
-        if let Ok(output) = output
-            && output.status.success()
-        {
-            info!("Killed existing RustFS processes: {}", binary_name);
-            sleep(Duration::from_millis(1000)).await;
+        for pattern in [&self.address, &self.temp_dir] {
+            let output = Command::new("pkill").args(["-f", pattern]).output();
+
+            if let Ok(output) = output
+                && output.status.success()
+            {
+                info!("Killed existing RustFS processes matching: {}", pattern);
+                sleep(Duration::from_millis(250)).await;
+            }
         }
         Ok(())
     }
 
-    /// Start RustFS server with basic configuration
-    pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        self.cleanup_existing_processes().await?;
-
+    fn build_start_args<'a>(&'a self, extra_args: Vec<&'a str>) -> Vec<&'a str> {
         let mut args = vec![
             "--address",
             &self.address,
@@ -210,16 +343,29 @@ impl RustFSTestEnvironment {
             &self.secret_key,
         ];
 
-        // Add extra arguments
         args.extend(extra_args);
-
-        // Add temp directory as the last argument
         args.push(&self.temp_dir);
+        args
+    }
+
+    async fn start_rustfs_server_inner(
+        &mut self,
+        extra_args: Vec<&str>,
+        cleanup_existing: bool,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        if cleanup_existing {
+            self.cleanup_existing_processes().await?;
+        }
+
+        let args = self.build_start_args(extra_args);
 
         info!("Starting RustFS server with args: {:?}", args);
 
         let binary_path = rustfs_binary_path();
-        let process = Command::new(&binary_path).args(&args).spawn()?;
+        let process = Command::new(&binary_path)
+            .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
+            .args(&args)
+            .spawn()?;
 
         self.process = Some(process);
 
@@ -229,18 +375,40 @@ impl RustFSTestEnvironment {
         Ok(())
     }
 
-    /// Wait for RustFS server to be ready by checking TCP connectivity
+    /// Start RustFS server with basic configuration
+    pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner(extra_args, true).await
+    }
+
+    /// Start RustFS server without cleaning up other running RustFS processes.
+    ///
+    /// This is useful for tests that need multiple independent RustFS instances
+    /// alive at the same time.
+    pub async fn start_rustfs_server_without_cleanup(
+        &mut self,
+        extra_args: Vec<&str>,
+    ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        self.start_rustfs_server_inner(extra_args, false).await
+    }
+
+    /// Wait for RustFS server to be ready.
+    ///
+    /// A listening TCP port is not sufficient here: the process may accept
+    /// connections before the S3 stack is fully initialized, which causes
+    /// early requests to fail intermittently. Treat readiness as "S3 API
+    /// responds successfully" instead.
     pub async fn wait_for_server_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         info!("Waiting for RustFS server to be ready on {}", self.address);
+        let client = self.create_s3_client();
 
-        for i in 0..30 {
-            if TcpStream::connect(&self.address).await.is_ok() {
+        for i in 0..60 {
+            if TcpStream::connect(&self.address).await.is_ok() && client.list_buckets().send().await.is_ok() {
                 info!("✅ RustFS server is ready after {} attempts", i + 1);
                 return Ok(());
             }
 
-            if i == 29 {
-                return Err("RustFS server failed to become ready within 30 seconds".into());
+            if i == 59 {
+                return Err("RustFS server failed to become ready within 60 seconds".into());
             }
 
             sleep(Duration::from_secs(1)).await;
@@ -251,16 +419,7 @@ impl RustFSTestEnvironment {
 
     /// Create an AWS S3 client configured for this RustFS instance
     pub fn create_s3_client(&self) -> Client {
-        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "e2e-test");
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&self.url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-
-        Client::from_conf(config)
+        Client::from_conf(build_test_s3_config(&self.url, &self.access_key, &self.secret_key, "e2e-test"))
     }
 
     /// Create test bucket
@@ -493,6 +652,7 @@ impl RustFSTestClusterEnvironment {
                 .env("RUSTFS_ACCESS_KEY", &self.access_key)
                 .env("RUSTFS_SECRET_KEY", &self.secret_key)
                 .env("RUSTFS_CONSOLE_ENABLE", "false")
+                .env("RUST_LOG", "rustfs=info,rustfs_notify=debug")
                 .current_dir(&node.data_dir)
                 .spawn()?;
 
@@ -503,7 +663,9 @@ impl RustFSTestClusterEnvironment {
             self.wait_for_node_ready(&node.address, i).await?;
         }
 
-        self.wait_for_service_ready().await?;
+        for node_idx in 0..self.nodes.len() {
+            self.wait_for_node_service_ready(node_idx).await?;
+        }
 
         Ok(())
     }
@@ -523,17 +685,17 @@ impl RustFSTestClusterEnvironment {
         Err(format!("Node {} failed to become ready", idx).into())
     }
 
-    /// Wait for the entire cluster's S3-compatible service to be ready (internal helper method).
+    /// Wait for a specific node's S3-compatible service to be ready (internal helper method).
     ///
-    /// Verifies service availability by calling the S3 `list_buckets` API, retries up to 120 times
-    /// with a 1-second interval between attempts. Fails if the API call remains unsuccessful after all retries.
-    async fn wait_for_service_ready(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let client = self.create_s3_client(0)?;
+    /// Verifies service availability by calling the S3 `list_buckets` API against the requested node,
+    /// retries up to 120 times with a 1-second interval between attempts.
+    async fn wait_for_node_service_ready(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let client = self.create_s3_client(node_idx)?;
 
         for attempt in 0..120 {
             match client.list_buckets().send().await {
                 Ok(_) => {
-                    info!("Cluster service ready after {} attempts", attempt + 1);
+                    info!("Cluster node {} service ready after {} attempts", node_idx, attempt + 1);
                     return Ok(());
                 }
                 Err(_) => {
@@ -541,7 +703,8 @@ impl RustFSTestClusterEnvironment {
                 }
             }
         }
-        Err("Cluster service failed to become ready".into())
+
+        Err(format!("Cluster node {} service failed to become ready", node_idx).into())
     }
 
     /// Create an S3 client configured to communicate with a specific cluster node.
@@ -562,15 +725,12 @@ impl RustFSTestClusterEnvironment {
         if node_idx >= self.nodes.len() {
             return Err("node_idx is invalid".into());
         }
-        let credentials = Credentials::new(&self.access_key, &self.secret_key, None, None, "cluster-test");
-        let config = Config::builder()
-            .credentials_provider(credentials)
-            .region(Region::new("us-east-1"))
-            .endpoint_url(&self.nodes[node_idx].url)
-            .force_path_style(true)
-            .behavior_version_latest()
-            .build();
-        Ok(Client::from_conf(config))
+        Ok(Client::from_conf(build_test_s3_config(
+            &self.nodes[node_idx].url,
+            &self.access_key,
+            &self.secret_key,
+            "cluster-test",
+        )))
     }
 
     /// Create S3 clients for all nodes in the RustFS cluster and collect them into a vector.

crates/e2e_test/src/kms/common.rs

@@ -22,11 +22,13 @@
 //! - KMS backend configuration (Local and Vault)
 //! - SSE encryption testing utilities
 
-use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging};
+use crate::common::{
+    RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client,
+};
 use aws_sdk_s3::Client;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::ServerSideEncryption;
-use base64::Engine;
+use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
 use serde_json;
 use std::process::{Child, Command};
 use std::time::Duration;
@@ -51,6 +53,19 @@ pub fn init_logging() {
     // Additional KMS-specific logging configuration can be added here if needed
 }
 
+pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool {
+    if awscurl_available() {
+        return false;
+    }
+
+    info!("Skipping {} because awscurl is not available in PATH", test_name);
+    true
+}
+
+pub fn sse_customer_key_md5_base64(key: &str) -> String {
+    BASE64.encode(md5::compute(key).0)
+}
+
 // KMS-specific helper functions
 /// Configure KMS backend via admin API
 pub async fn configure_kms(
@@ -133,10 +148,10 @@ pub async fn create_key_with_specific_id(key_dir: &str, key_id: &str) -> Result<
         "usage": "EncryptDecrypt",
         "status": "Active",
         "metadata": HashMap::<String, String>::new(),
-        "created_at": chrono::Utc::now().to_rfc3339(),
+        "created_at": format!("{}[UTC]", chrono::Utc::now().to_rfc3339()),
         "rotated_at": serde_json::Value::Null,
         "created_by": "e2e-test",
-        "encrypted_key_material": key_data.to_vec(),
+        "encrypted_key_material": BASE64.encode(key_data),
         "nonce": Vec::<u8>::new()
     });
 
@@ -155,7 +170,7 @@ pub async fn test_sse_c_encryption(s3_client: &Client, bucket: &str) -> Result<(
 
     let test_key = "01234567890123456789012345678901"; // 32-byte key
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let test_data = b"Hello, KMS SSE-C World!";
     let object_key = "test-sse-c-object";
 
@@ -272,6 +287,10 @@ pub async fn test_kms_key_management(
     access_key: &str,
     secret_key: &str,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    if skip_if_kms_admin_tool_unavailable("test_kms_key_management") {
+        return Ok(());
+    }
+
     info!("Testing KMS key management APIs");
 
     // Test CreateKey
@@ -324,8 +343,8 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<()
     let wrong_key = "98765432109876543210987654321098";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
     let wrong_key_b64 = base64::engine::general_purpose::STANDARD.encode(wrong_key);
-    let test_key_md5 = format!("{:x}", md5::compute(test_key));
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
     let test_data = b"Test data for error scenarios";
     let object_key = "test-error-object";
 
@@ -406,7 +425,7 @@ impl VaultTestEnvironment {
             let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
             if port_check {
                 // Additional check by making a health request
-                if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await
+                if let Ok(response) = local_http_client().get(format!("{VAULT_URL}/v1/sys/health")).send().await
                     && response.status().is_success()
                 {
                     info!("Vault server is ready after {} seconds", i);
@@ -426,7 +445,7 @@ impl VaultTestEnvironment {
 
     /// Setup Vault transit secrets engine
     pub async fn setup_vault_transit(&self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-        let client = reqwest::Client::new();
+        let client = local_http_client();
 
         info!("Enabling Vault transit secrets engine");
 
@@ -687,7 +706,7 @@ pub async fn test_multipart_upload_with_config(
 /// Create a standard SSE-C encryption configuration for testing
 pub fn create_sse_c_config() -> EncryptionType {
     let key = "01234567890123456789012345678901"; // 32-byte key
-    let key_md5 = format!("{:x}", md5::compute(key));
+    let key_md5 = sse_customer_key_md5_base64(key);
     EncryptionType::SSEC {
         key: key.to_string(),
         key_md5,

crates/e2e_test/src/kms/encryption_metadata_test.rs

@@ -26,27 +26,20 @@ use serial_test::serial;
 use std::collections::{HashMap, VecDeque};
 use tracing::info;
 
-fn assert_encryption_metadata(metadata: &HashMap<String, String>, expected_size: usize) {
+fn assert_managed_encryption_metadata_hidden(metadata: Option<&HashMap<String, String>>) {
+    let Some(metadata) = metadata else { return };
+
     for key in [
         "x-rustfs-encryption-key",
         "x-rustfs-encryption-iv",
         "x-rustfs-encryption-context",
         "x-rustfs-encryption-original-size",
     ] {
-        assert!(metadata.contains_key(key), "expected managed encryption metadata '{key}' to be present");
         assert!(
-            !metadata.get(key).unwrap().is_empty(),
-            "managed encryption metadata '{key}' should not be empty"
+            !metadata.contains_key(key),
+            "managed encryption metadata '{key}' should not be exposed to clients"
         );
     }
-
-    let size_value = metadata
-        .get("x-rustfs-encryption-original-size")
-        .expect("managed encryption metadata should include original size");
-    let parsed_size: usize = size_value
-        .parse()
-        .expect("x-rustfs-encryption-original-size should be numeric");
-    assert_eq!(parsed_size, expected_size, "recorded original size should match uploaded payload length");
 }
 
 fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: &str, key: &str, plaintext: &[u8]) {
@@ -142,10 +135,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
         "head_object should advertise SSE-S3"
     );
 
-    let metadata = head
-        .metadata()
-        .expect("head_object should return managed encryption metadata");
-    assert_encryption_metadata(metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     assert_storage_encrypted(std::path::Path::new(&kms_env.base_env.temp_dir), TEST_BUCKET, key, payload);
 
@@ -210,10 +200,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "source object should maintain the configured KMS key id"
     );
-    let source_metadata = head_source
-        .metadata()
-        .expect("source object should include managed encryption metadata");
-    assert_encryption_metadata(source_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_source.metadata());
 
     let dest_key = "metadata-sse-kms-object-copy";
     let copy_source = format!("{TEST_BUCKET}/{source_key}");
@@ -238,10 +225,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
         &default_key_id,
         "copied object should keep the default KMS key id"
     );
-    let dest_metadata = head_dest
-        .metadata()
-        .expect("copied object should include managed encryption metadata");
-    assert_encryption_metadata(dest_metadata, payload.len());
+    assert_managed_encryption_metadata_hidden(head_dest.metadata());
 
     let copied_body = s3_client
         .get_object()
@@ -358,10 +342,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
         "multipart object should retain bucket default KMS key"
     );
 
-    assert_encryption_metadata(
-        head.metadata().expect("multipart head_object should expose managed metadata"),
-        combined.len(),
-    );
+    assert_managed_encryption_metadata_hidden(head.metadata());
 
     // Data returned to clients should decrypt back to original payload
     let fetched = s3_client

crates/e2e_test/src/kms/kms_comprehensive_test.rs

@@ -19,9 +19,9 @@
 //! complex workflows.
 
 use super::common::{
-    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, test_all_multipart_encryption_types,
-    test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption, test_sse_kms_encryption,
-    test_sse_s3_encryption,
+    EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, sse_customer_key_md5_base64,
+    test_all_multipart_encryption_types, test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
@@ -149,8 +149,8 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     // Test different SSE-C keys to ensure isolation
     let key1 = "01234567890123456789012345678901";
     let key2 = "98765432109876543210987654321098";
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let config1 = MultipartTestConfig::new(
         "isolation-test-key1",
@@ -183,7 +183,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
     info!("🔒 Verify key isolation");
     let wrong_key = "11111111111111111111111111111111";
     let wrong_key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, wrong_key);
-    let wrong_key_md5 = format!("{:x}", md5::compute(wrong_key));
+    let wrong_key_md5 = sse_customer_key_md5_base64(wrong_key);
 
     // Try to read file encrypted with key1 using wrong key
     let wrong_read_result = s3_client

crates/e2e_test/src/kms/kms_edge_cases_test.rs

@@ -21,7 +21,7 @@
 //! - Concurrent encryption operations
 //! - Security validation tests
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use aws_sdk_s3::types::ServerSideEncryption;
 use base64::Engine;
@@ -72,7 +72,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
     info!("📤 Testing SSE-C with zero-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "zero-byte-sse-c";
 
     let _put_response_c = s3_client
@@ -166,7 +166,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
     info!("📤 Testing SSE-C with single-byte file");
     let test_key = "01234567890123456789012345678901";
     let test_key_b64 = base64::engine::general_purpose::STANDARD.encode(test_key);
-    let test_key_md5 = format!("{:x}", compute(test_key));
+    let test_key_md5 = sse_customer_key_md5_base64(test_key);
     let object_key_c = "single-byte-sse-c";
 
     s3_client
@@ -334,7 +334,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
     info!("🔍 Testing access to SSE-C object without key");
 
     // First upload a valid SSE-C object
-    let valid_key_md5 = format!("{:x}", compute(valid_key));
+    let valid_key_md5 = sse_customer_key_md5_base64(valid_key);
     s3_client
         .put_object()
         .bucket(TEST_BUCKET)
@@ -421,7 +421,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
                     // SSE-C
                     let key = format!("testkey{i:026}"); // 32-byte key
                     let key_b64 = base64::engine::general_purpose::STANDARD.encode(&key);
-                    let key_md5 = format!("{:x}", compute(&key));
+                    let key_md5 = sse_customer_key_md5_base64(&key);
 
                     client
                         .put_object()
@@ -493,8 +493,8 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
 
     let key1_b64 = base64::engine::general_purpose::STANDARD.encode(key1);
     let key2_b64 = base64::engine::general_purpose::STANDARD.encode(key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     // Upload same data with different keys
     s3_client

crates/e2e_test/src/kms/kms_local_test.rs

@@ -19,7 +19,10 @@
 //! - S3 object upload/download with SSE-S3, SSE-KMS, SSE-C encryption
 //! - Complete encryption/decryption lifecycle
 
-use super::common::{LocalKMSTestEnvironment, get_kms_status, test_kms_key_management, test_sse_c_encryption};
+use super::common::{
+    LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    test_kms_key_management, test_sse_c_encryption,
+};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{error, info};
@@ -28,6 +31,9 @@ use tracing::{error, info};
 #[serial]
 async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Local KMS End-to-End Test");
 
     // Create LocalKMS test environment
@@ -140,8 +146,8 @@ async fn test_local_kms_key_isolation() {
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", md5::compute(key1));
-    let key2_md5 = format!("{:x}", md5::compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Data encrypted with key 1";
     let data2 = b"Data encrypted with key 2";
@@ -562,7 +568,7 @@ async fn test_multipart_upload_with_sse_c(
     // SSE-C encryption key
     let encryption_key = "01234567890123456789012345678901";
     let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, encryption_key);
-    let key_md5 = format!("{:x}", md5::compute(encryption_key));
+    let key_md5 = sse_customer_key_md5_base64(encryption_key);
 
     // Generate test data
     let test_data: Vec<u8> = (0..total_size).map(|i| ((i * 3) % 256) as u8).collect();

crates/e2e_test/src/kms/kms_vault_test.rs

@@ -19,14 +19,14 @@
 //! multipart upload behaviour.
 
 use crate::common::{TEST_BUCKET, init_logging};
-use md5::compute;
 use serial_test::serial;
 use tokio::time::{Duration, sleep};
 use tracing::{error, info};
 
 use super::common::{
-    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, start_kms, test_all_multipart_encryption_types, test_error_scenarios,
-    test_kms_key_management, test_sse_c_encryption, test_sse_kms_encryption, test_sse_s3_encryption,
+    VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64,
+    start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption,
+    test_sse_kms_encryption, test_sse_s3_encryption,
 };
 
 /// Helper that brings up Vault, configures RustFS, and starts the KMS service.
@@ -65,6 +65,9 @@ impl VaultKmsTestContext {
 #[serial]
 async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") {
+        return Ok(());
+    }
     info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME);
 
     let context = VaultKmsTestContext::new().await?;
@@ -118,6 +121,9 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") {
+        return Ok(());
+    }
     info!("Starting Vault KMS SSE-C key isolation test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -133,8 +139,8 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
     let key2 = "98765432109876543210987654321098";
     let key1_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key1);
     let key2_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key2);
-    let key1_md5 = format!("{:x}", compute(key1));
-    let key2_md5 = format!("{:x}", compute(key2));
+    let key1_md5 = sse_customer_key_md5_base64(key1);
+    let key2_md5 = sse_customer_key_md5_base64(key2);
 
     let data1 = b"Vault data encrypted with key 1";
     let data2 = b"Vault data encrypted with key 2";
@@ -202,6 +208,9 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box<dyn std::error::Error
 #[serial]
 async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") {
+        return Ok(());
+    }
     info!("Starting Vault KMS large file SSE-S3 test");
 
     let context = VaultKmsTestContext::new().await?;
@@ -264,6 +273,9 @@ async fn test_vault_kms_large_file() -> Result<(), Box<dyn std::error::Error + S
 #[serial]
 async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") {
+        return Ok(());
+    }
     info!("Starting Vault KMS multipart upload encryption suite");
 
     let context = VaultKmsTestContext::new().await?;
@@ -292,6 +304,9 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box<dyn std::error::Err
 #[serial]
 async fn test_vault_kms_key_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     init_logging();
+    if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") {
+        return Ok(());
+    }
     info!("Starting Vault KMS key operations test (CRUD)");
 
     let context = VaultKmsTestContext::new().await?;

crates/e2e_test/src/kms/multipart_encryption_test.rs

@@ -21,7 +21,7 @@
 //! 3. Test the saving and reading of encrypted metadata
 //! 4. Test the complete sharded upload encryption process
 
-use super::common::LocalKMSTestEnvironment;
+use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64};
 use crate::common::{TEST_BUCKET, init_logging};
 use serial_test::serial;
 use tracing::{debug, info};
@@ -504,7 +504,7 @@ async fn test_multipart_encryption_type(
     let (sse_c_key, sse_c_md5) = if matches!(encryption_type, EncryptionType::SSEC) {
         let key = "01234567890123456789012345678901";
         let key_b64 = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, key);
-        let key_md5 = format!("{:x}", md5::compute(key));
+        let key_md5 = sse_customer_key_md5_base64(key);
         (Some(key_b64), Some(key_md5))
     } else {
         (None, None)

crates/e2e_test/src/lib.rs

@@ -88,3 +88,19 @@ mod checksum_upload_test;
 // Group deletion tests
 #[cfg(test)]
 mod group_delete_test;
+
+// S3 dummy-compat bucket API tests
+#[cfg(test)]
+mod bucket_logging_test;
+
+// Multipart control API auth regression tests
+#[cfg(test)]
+mod multipart_auth_test;
+
+// Object lambda end-to-end regression tests
+#[cfg(test)]
+mod object_lambda_test;
+
+// Replication extension end-to-end regression tests
+#[cfg(test)]
+mod replication_extension_test;

crates/e2e_test/src/object_lambda_test.rs

@@ -0,0 +1,985 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
+use aws_sdk_s3::primitives::ByteStream;
+use http::header::{CONTENT_TYPE, HOST};
+use reqwest::StatusCode;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::{pre_sign_v4, sign_v4};
+use s3s::Body;
+use serial_test::serial;
+use std::collections::HashMap;
+use std::error::Error;
+use time::OffsetDateTime;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::TcpListener;
+use tokio::sync::oneshot;
+use tokio::time::{Duration, timeout};
+
+#[derive(Debug)]
+struct CapturedWebhookRequest {
+    headers: HashMap<String, String>,
+    payload: serde_json::Value,
+}
+
+struct WebhookResponseSpec {
+    status_line: String,
+    body: Vec<u8>,
+    headers: Vec<(String, String)>,
+    include_auth_headers: bool,
+    auth_route_override: Option<String>,
+    auth_token_override: Option<String>,
+}
+
+fn find_header_terminator(buf: &[u8]) -> Option<usize> {
+    buf.windows(4).position(|window| window == b"\r\n\r\n")
+}
+
+async fn read_http_request(
+    stream: &mut tokio::net::TcpStream,
+) -> Result<(HashMap<String, String>, Vec<u8>), Box<dyn Error + Send + Sync>> {
+    let mut buffer = Vec::new();
+    let mut chunk = [0_u8; 4096];
+
+    let header_end = loop {
+        let read = stream.read(&mut chunk).await?;
+        if read == 0 {
+            return Err("webhook request ended before headers were fully received".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+        if let Some(pos) = find_header_terminator(&buffer) {
+            break pos;
+        }
+    };
+
+    let header_bytes = &buffer[..header_end];
+    let header_text = std::str::from_utf8(header_bytes)?;
+    let mut lines = header_text.split("\r\n");
+    let _request_line = lines.next().ok_or("missing request line")?;
+    let mut headers = HashMap::new();
+    for line in lines {
+        if line.is_empty() {
+            continue;
+        }
+        let (name, value) = line.split_once(':').ok_or("invalid header line")?;
+        headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string());
+    }
+
+    let content_length = headers
+        .get("content-length")
+        .ok_or("missing content-length header")?
+        .parse::<usize>()?;
+    let body_offset = header_end + 4;
+    while buffer.len().saturating_sub(body_offset) < content_length {
+        let read = stream.read(&mut chunk).await?;
+        if read == 0 {
+            return Err("webhook request ended before body was fully received".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+    }
+
+    Ok((headers, buffer[body_offset..body_offset + content_length].to_vec()))
+}
+
+async fn spawn_object_lambda_webhook_server() -> Result<
+    (
+        String,
+        oneshot::Receiver<CapturedWebhookRequest>,
+        tokio::task::JoinHandle<Result<(), Box<dyn Error + Send + Sync>>>,
+    ),
+    Box<dyn Error + Send + Sync>,
+> {
+    spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec {
+        status_line: "200 OK".to_string(),
+        body: b"transformed through object lambda".to_vec(),
+        headers: vec![("content-type".to_string(), "text/plain".to_string())],
+        include_auth_headers: true,
+        auth_route_override: None,
+        auth_token_override: None,
+    })
+    .await
+}
+
+async fn spawn_object_lambda_webhook_server_with_response(
+    response_spec: WebhookResponseSpec,
+) -> Result<
+    (
+        String,
+        oneshot::Receiver<CapturedWebhookRequest>,
+        tokio::task::JoinHandle<Result<(), Box<dyn Error + Send + Sync>>>,
+    ),
+    Box<dyn Error + Send + Sync>,
+> {
+    let listener = TcpListener::bind("127.0.0.1:0").await?;
+    let address = listener.local_addr()?;
+    let webhook_url = format!("http://{address}/transform");
+    let (request_tx, request_rx) = oneshot::channel();
+
+    let handle = tokio::spawn(async move {
+        loop {
+            let (mut stream, _) = listener.accept().await?;
+            let Ok(Ok((headers, body))) = timeout(Duration::from_secs(2), read_http_request(&mut stream)).await else {
+                continue;
+            };
+            let payload: serde_json::Value = serde_json::from_slice(&body)?;
+
+            let output_route = payload["getObjectContext"]["outputRoute"]
+                .as_str()
+                .ok_or("missing outputRoute in webhook payload")?
+                .to_string();
+            let output_token = payload["getObjectContext"]["outputToken"]
+                .as_str()
+                .ok_or("missing outputToken in webhook payload")?
+                .to_string();
+
+            let _ = request_tx.send(CapturedWebhookRequest { headers, payload });
+
+            let mut response_head = format!(
+                "HTTP/1.1 {}\r\ncontent-length: {}\r\nconnection: close\r\n",
+                response_spec.status_line,
+                response_spec.body.len()
+            );
+            for (name, value) in &response_spec.headers {
+                response_head.push_str(&format!("{name}: {value}\r\n"));
+            }
+            if response_spec.include_auth_headers {
+                let auth_route = response_spec.auth_route_override.as_deref().unwrap_or(&output_route);
+                let auth_token = response_spec.auth_token_override.as_deref().unwrap_or(&output_token);
+                response_head.push_str(&format!("x-amz-request-route: {auth_route}\r\n"));
+                response_head.push_str(&format!("x-amz-request-token: {auth_token}\r\n"));
+            }
+            response_head.push_str("\r\n");
+            stream.write_all(response_head.as_bytes()).await?;
+            stream.write_all(&response_spec.body).await?;
+            stream.shutdown().await?;
+
+            return Ok(());
+        }
+    });
+
+    Ok((webhook_url, request_rx, handle))
+}
+
+async fn presigned_get_request(
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let signed = pre_sign_v4(
+        http::Request::builder()
+            .method(http::Method::GET)
+            .uri(uri)
+            .header(HOST, authority)
+            .body(Body::empty())?,
+        access_key,
+        secret_key,
+        "",
+        "us-east-1",
+        600,
+        OffsetDateTime::now_utc(),
+    );
+
+    Ok(local_http_client().get(signed.uri().to_string()).send().await?)
+}
+
+async fn signed_request(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+async fn configure_webhook_target(
+    env: &RustFSTestEnvironment,
+    target_name: &str,
+    endpoint: &str,
+    auth_token: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    configure_webhook_target_with_key_values(
+        env,
+        target_name,
+        vec![
+            ("endpoint", endpoint.to_string()),
+            ("auth_token", auth_token.to_string()),
+            ("queue_dir", format!("{}/notify-queue", env.temp_dir)),
+        ],
+    )
+    .await
+}
+
+async fn configure_webhook_target_with_key_values(
+    env: &RustFSTestEnvironment,
+    target_name: &str,
+    key_values: Vec<(&str, String)>,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let queue_dir = format!("{}/notify-queue", env.temp_dir);
+    tokio::fs::create_dir_all(&queue_dir).await?;
+    let mut key_values = key_values
+        .into_iter()
+        .map(|(key, value)| serde_json::json!({ "key": key, "value": value }))
+        .collect::<Vec<_>>();
+    if !key_values.iter().any(|entry| entry["key"].as_str() == Some("queue_dir")) {
+        key_values.push(serde_json::json!({ "key": "queue_dir", "value": queue_dir }));
+    }
+    let response = send_configure_webhook_target_request(env, target_name, key_values).await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("failed to configure object lambda webhook target: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn send_configure_webhook_target_request(
+    env: &RustFSTestEnvironment,
+    target_name: &str,
+    key_values: Vec<serde_json::Value>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let payload = serde_json::json!({ "key_values": key_values });
+    let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{}", env.url, target_name);
+    signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(payload.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await
+}
+
+async fn list_notification_targets(env: &RustFSTestEnvironment) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/target/list", env.url);
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+    let status = response.status();
+    let body = response.bytes().await?;
+    if status != StatusCode::OK {
+        return Err(format!("failed to list notification targets: {status} {}", String::from_utf8_lossy(body.as_ref())).into());
+    }
+
+    Ok(serde_json::from_slice(&body)?)
+}
+
+async fn list_target_arns(env: &RustFSTestEnvironment) -> Result<Vec<String>, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/target/arns", env.url);
+    let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
+    let status = response.status();
+    let body = response.bytes().await?;
+    if status != StatusCode::OK {
+        return Err(format!("failed to list target arns: {status} {}", String::from_utf8_lossy(body.as_ref())).into());
+    }
+
+    Ok(serde_json::from_slice(&body)?)
+}
+
+async fn wait_for_target_visibility(
+    env: &RustFSTestEnvironment,
+    target_name: &str,
+) -> Result<(serde_json::Value, Vec<String>), Box<dyn Error + Send + Sync>> {
+    let mut last_targets = serde_json::Value::Null;
+    let mut last_arns = Vec::new();
+
+    for _ in 0..20 {
+        last_targets = list_notification_targets(env).await?;
+        last_arns = list_target_arns(env).await?;
+
+        let listed = last_targets["notification_endpoints"]
+            .as_array()
+            .into_iter()
+            .flatten()
+            .any(|entry| {
+                entry["account_id"].as_str() == Some(target_name)
+                    && entry["service"]
+                        .as_str()
+                        .is_some_and(|service| service == "webhook" || service.starts_with("webhook-"))
+            });
+
+        if listed {
+            return Ok((last_targets, last_arns));
+        }
+
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("target {target_name} did not become visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into())
+}
+
+async fn read_persisted_server_config(env: &RustFSTestEnvironment) -> String {
+    let path = format!("{}/.rustfs.sys/config/config.json", env.temp_dir);
+    match tokio::fs::read_to_string(&path).await {
+        Ok(content) => content,
+        Err(err) => format!("failed to read persisted config at {path}: {err}"),
+    }
+}
+
+async fn read_listen_notification_event(
+    response: reqwest::Response,
+    expected_key: &str,
+) -> Result<String, Box<dyn Error + Send + Sync>> {
+    let mut response = response;
+    let mut pending = String::new();
+    loop {
+        let chunk = timeout(Duration::from_secs(12), response.chunk()).await??;
+        let Some(chunk) = chunk else {
+            return Err("listen_notification stream ended before payload".into());
+        };
+        if chunk.is_empty() {
+            continue;
+        }
+        pending.push_str(&String::from_utf8(chunk.to_vec())?);
+
+        while let Some(newline) = pending.find('\n') {
+            let line = pending.drain(..=newline).collect::<String>();
+            let payload = line.trim();
+            if payload.is_empty() {
+                continue;
+            }
+
+            let json: serde_json::Value = serde_json::from_str(payload)?;
+            let Some(records) = json["Records"].as_array() else {
+                continue;
+            };
+            if records.is_empty() {
+                continue;
+            }
+
+            let has_expected_key = records.iter().any(|record| {
+                let Some(object_key) = record["s3"]["object"]["key"].as_str() else {
+                    return false;
+                };
+                let decoded = urlencoding::decode(object_key)
+                    .map(|decoded| decoded.into_owned())
+                    .unwrap_or_else(|_| object_key.to_string());
+                decoded == expected_key
+            });
+            if has_expected_key {
+                return Ok(payload.to_string());
+            }
+        }
+    }
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-presigned";
+    let key = "input.txt";
+    let object_body = b"hello presigned object lambda";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(object_body))
+        .send()
+        .await?;
+
+    configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?;
+    wait_for_target_visibility(&env, "transformer").await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = presigned_get_request(&lambda_url, &env.access_key, &env.secret_key).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(response.text().await?, "transformed through object lambda");
+
+    let captured = timeout(Duration::from_secs(10), request_rx).await??;
+    assert_eq!(captured.payload["configuration"]["accessPointArn"].as_str(), Some(lambda_arn));
+
+    webhook_handle.await??;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_accepts_named_webhook_target_arn() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-named-target";
+    let key = "input.txt";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook-preview";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?;
+    wait_for_target_visibility(&env, "transformer").await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(response.text().await?, "transformed through object lambda");
+
+    let captured = timeout(Duration::from_secs(10), request_rx).await??;
+    assert_eq!(captured.payload["configuration"]["accessPointArn"].as_str(), Some(lambda_arn));
+
+    webhook_handle.await??;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e";
+    let key = "input.txt";
+    let object_body = b"hello object lambda";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(object_body))
+        .send()
+        .await?;
+
+    configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?;
+    let (visible_targets, visible_arns) = wait_for_target_visibility(&env, "transformer").await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        let persisted_config = read_persisted_server_config(&env).await;
+        return Err(format!(
+            "object lambda request failed: {status} {body}; visible_targets={visible_targets}; visible_arns={visible_arns:?}; persisted_config={persisted_config}"
+        )
+        .into());
+    }
+    assert_eq!(
+        response.headers().get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
+        Some("text/plain")
+    );
+    assert_eq!(response.text().await?, "transformed through object lambda");
+
+    let captured = timeout(Duration::from_secs(10), request_rx).await??;
+    assert_eq!(captured.headers.get("authorization").map(String::as_str), Some("Bearer secret-token"));
+    assert_eq!(captured.headers.get("x-rustfs-object-lambda-bucket").map(String::as_str), Some(bucket));
+    assert_eq!(captured.headers.get("x-rustfs-object-lambda-key").map(String::as_str), Some(key));
+
+    assert_eq!(captured.payload["configuration"]["accessPointArn"].as_str(), Some(lambda_arn));
+    let expected_request_url = format!("/{bucket}/{key}?lambdaArn={}", urlencoding::encode(lambda_arn));
+    assert_eq!(captured.payload["userRequest"]["url"].as_str(), Some(expected_request_url.as_str()));
+
+    let input_s3_url = captured.payload["getObjectContext"]["inputS3Url"]
+        .as_str()
+        .ok_or("missing inputS3Url in object lambda payload")?;
+    assert!(!input_s3_url.contains("lambdaArn="));
+
+    let source_response = local_http_client().get(input_s3_url).send().await?;
+    assert_eq!(source_response.status(), StatusCode::OK);
+    assert_eq!(source_response.bytes().await?.as_ref(), object_body);
+
+    webhook_handle.await??;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec {
+        status_line: "418 I'm a teapot".to_string(),
+        body: b"lambda upstream rejected".to_vec(),
+        headers: vec![
+            ("content-type".to_string(), "text/plain".to_string()),
+            ("x-rustfs-debug".to_string(), "passthrough".to_string()),
+            ("x-amz-request-route".to_string(), "should-not-leak".to_string()),
+            ("x-amz-request-token".to_string(), "should-not-leak".to_string()),
+        ],
+        include_auth_headers: false,
+        auth_route_override: None,
+        auth_token_override: None,
+    })
+    .await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-failure";
+    let key = "input.txt";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?;
+    wait_for_target_visibility(&env, "transformer").await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    assert_eq!(response.status(), StatusCode::IM_A_TEAPOT);
+    assert_eq!(
+        response.headers().get("content-type").and_then(|value| value.to_str().ok()),
+        Some("text/plain")
+    );
+    assert_eq!(
+        response.headers().get("x-rustfs-debug").and_then(|value| value.to_str().ok()),
+        Some("passthrough")
+    );
+    assert!(response.headers().get("x-amz-request-route").is_none());
+    assert!(response.headers().get("x-amz-request-token").is_none());
+    assert_eq!(response.text().await?, "lambda upstream rejected");
+
+    webhook_handle.await??;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_rejects_success_response_without_auth_headers() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec {
+        status_line: "200 OK".to_string(),
+        body: b"missing auth headers".to_vec(),
+        headers: vec![("content-type".to_string(), "text/plain".to_string())],
+        include_auth_headers: false,
+        auth_route_override: None,
+        auth_token_override: None,
+    })
+    .await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-missing-auth";
+    let key = "input.txt";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?;
+    wait_for_target_visibility(&env, "transformer").await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+    let body = response.text().await?;
+    assert!(body.contains("authorization headers"), "unexpected error body: {body}");
+
+    webhook_handle.await??;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_headers() -> Result<(), Box<dyn Error + Send + Sync>>
+{
+    init_logging();
+
+    let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server_with_response(WebhookResponseSpec {
+        status_line: "200 OK".to_string(),
+        body: b"mismatched auth headers".to_vec(),
+        headers: vec![("content-type".to_string(), "text/plain".to_string())],
+        include_auth_headers: true,
+        auth_route_override: Some("wrong-route".to_string()),
+        auth_token_override: Some("wrong-token".to_string()),
+    })
+    .await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-mismatched-auth";
+    let key = "input.txt";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    configure_webhook_target(&env, "transformer", &webhook_url, "secret-token").await?;
+    wait_for_target_visibility(&env, "transformer").await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+    let body = response.text().await?;
+    assert!(body.contains("authorization headers"), "unexpected error body: {body}");
+
+    webhook_handle.await??;
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_rejects_unsupported_target_type() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-unsupported-target";
+    let key = "input.txt";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:mqtt";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::NOT_IMPLEMENTED);
+    assert!(body.contains("NotImplemented"), "unexpected error body: {body}");
+    assert!(
+        body.to_ascii_lowercase().contains("target type is not supported"),
+        "unexpected error body: {body}"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_rejects_unconfigured_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-missing-target";
+    let key = "input.txt";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected error body: {body}");
+    assert!(
+        body.to_ascii_lowercase().contains("target is not configured"),
+        "unexpected error body: {body}"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-disabled-target";
+    let key = "input.txt";
+    let lambda_arn = "arn:rustfs:s3-object-lambda:us-east-1:transformer:webhook";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    configure_webhook_target_with_key_values(
+        &env,
+        "transformer",
+        vec![
+            ("endpoint", "http://127.0.0.1:9/transform".to_string()),
+            ("auth_token", "secret-token".to_string()),
+            ("enable", "off".to_string()),
+        ],
+    )
+    .await?;
+    wait_for_target_visibility(&env, "transformer").await?;
+
+    let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
+    let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected error body: {body}");
+    assert!(body.to_ascii_lowercase().contains("target is disabled"), "unexpected error body: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "object-lambda-e2e-invalid-endpoint";
+
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    client
+        .put_object()
+        .bucket(bucket)
+        .key("input.txt")
+        .body(ByteStream::from_static(b"hello object lambda"))
+        .send()
+        .await?;
+
+    let response = send_configure_webhook_target_request(
+        &env,
+        "transformer",
+        vec![
+            serde_json::json!({ "key": "endpoint", "value": "://invalid-endpoint" }),
+            serde_json::json!({ "key": "auth_token", "value": "secret-token" }),
+            serde_json::json!({ "key": "queue_dir", "value": format!("{}/notify-queue", env.temp_dir) }),
+        ],
+    )
+    .await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidArgument"), "unexpected error body: {body}");
+    assert!(
+        body.to_ascii_lowercase().contains("invalid endpoint url"),
+        "unexpected error body: {body}"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_configure_object_lambda_notify_webhook_rejects_response_header_timeout_key()
+-> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let response = send_configure_webhook_target_request(
+        &env,
+        "transformer",
+        vec![
+            serde_json::json!({ "key": "endpoint", "value": "http://127.0.0.1:9/transform" }),
+            serde_json::json!({ "key": "auth_token", "value": "secret-token" }),
+            serde_json::json!({ "key": "response_header_timeout", "value": "not-a-duration" }),
+            serde_json::json!({ "key": "queue_dir", "value": format!("{}/notify-queue", env.temp_dir) }),
+        ],
+    )
+    .await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidArgument"), "unexpected error body: {body}");
+    assert!(
+        body.to_ascii_lowercase().contains("response_header_timeout"),
+        "unexpected error body: {body}"
+    );
+    assert!(body.to_ascii_lowercase().contains("not allowed"), "unexpected error body: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_listen_notification_emits_after_put_object() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "listen-notification-e2e";
+    let key = "logs/app.json";
+    let client = env.create_s3_client();
+
+    client.create_bucket().bucket(bucket).send().await?;
+
+    let listen_url = format!(
+        "{}/{bucket}?events={}&prefix={}&suffix={}&ping=1",
+        env.url,
+        urlencoding::encode("s3:ObjectCreated:Put"),
+        urlencoding::encode("logs/"),
+        urlencoding::encode(".json"),
+    );
+    let response = signed_request(http::Method::GET, &listen_url, &env.access_key, &env.secret_key, None, None).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(
+        response.headers().get("content-type").and_then(|value| value.to_str().ok()),
+        Some("text/event-stream")
+    );
+
+    let read_task = tokio::spawn(read_listen_notification_event(response, key));
+
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"listen notification body"))
+        .send()
+        .await?;
+
+    let payload = timeout(Duration::from_secs(12), read_task).await???;
+    assert!(!payload.is_empty(), "listen_notification payload should not be empty");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
+    cluster.start().await?;
+
+    let bucket = "listen-notification-cluster";
+    let key = "logs/cluster.json";
+    let node0_client = cluster.create_s3_client(0)?;
+    let node1_client = cluster.create_s3_client(1)?;
+
+    node0_client.create_bucket().bucket(bucket).send().await?;
+
+    let listen_url = format!(
+        "{}/{bucket}?events={}&prefix={}&suffix={}&ping=1",
+        cluster.nodes[0].url,
+        urlencoding::encode("s3:ObjectCreated:Put"),
+        urlencoding::encode("logs/"),
+        urlencoding::encode(".json"),
+    );
+    let response = signed_request(http::Method::GET, &listen_url, &cluster.access_key, &cluster.secret_key, None, None).await?;
+    assert_eq!(response.status(), StatusCode::OK);
+
+    let read_task = tokio::spawn(read_listen_notification_event(response, key));
+
+    node1_client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"cluster listen notification body"))
+        .send()
+        .await?;
+
+    let payload = timeout(Duration::from_secs(12), read_task).await???;
+    assert!(!payload.is_empty(), "listen_notification cluster payload should not be empty");
+
+    Ok(())
+}

crates/e2e_test/src/protocols/ftps_core.rs

@@ -14,11 +14,13 @@
 
 //! Core FTPS tests
 
-use crate::common::rustfs_binary_path;
+use crate::common::rustfs_binary_path_with_features;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use rcgen::generate_simple_self_signed;
-use rustls::{ClientConfig, RootCertStore, pki_types::CertificateDer, pki_types::pem::PemObject};
+use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
+use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
+use rustls::{ClientConfig, DigitallySignedStruct, Error as RustlsError, SignatureScheme};
 use std::io::Cursor;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -31,6 +33,46 @@ use tracing::info;
 const FTPS_PORT: u16 = 9021;
 const FTPS_ADDRESS: &str = "127.0.0.1:9021";
 
+#[derive(Debug)]
+struct AcceptAnyServerCertVerifier;
+
+impl ServerCertVerifier for AcceptAnyServerCertVerifier {
+    fn verify_server_cert(
+        &self,
+        _end_entity: &CertificateDer<'_>,
+        _intermediates: &[CertificateDer<'_>],
+        _server_name: &ServerName<'_>,
+        _ocsp_response: &[u8],
+        _now: UnixTime,
+    ) -> Result<ServerCertVerified, RustlsError> {
+        Ok(ServerCertVerified::assertion())
+    }
+
+    fn verify_tls12_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn verify_tls13_signature(
+        &self,
+        _message: &[u8],
+        _cert: &CertificateDer<'_>,
+        _dss: &DigitallySignedStruct,
+    ) -> Result<HandshakeSignatureValid, RustlsError> {
+        Ok(HandshakeSignatureValid::assertion())
+    }
+
+    fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+        rustls::crypto::aws_lc_rs::default_provider()
+            .signature_verification_algorithms
+            .supported_schemes()
+    }
+}
+
 /// Test FTPS: put, ls, mkdir, rmdir, delete operations
 pub async fn test_ftps_core_operations() -> Result<()> {
     let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
@@ -58,7 +100,7 @@ pub async fn test_ftps_core_operations() -> Result<()> {
 
     // Start server manually
     info!("Starting FTPS server on {}", FTPS_ADDRESS);
-    let binary_path = rustfs_binary_path();
+    let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav"));
     let mut server_process = Command::new(&binary_path)
         .env("RUSTFS_FTPS_ENABLE", "true")
         .env("RUSTFS_FTPS_ADDRESS", FTPS_ADDRESS)
@@ -78,19 +120,9 @@ pub async fn test_ftps_core_operations() -> Result<()> {
             .install_default()
             .map_err(|e| anyhow::anyhow!("Failed to install crypto provider: {:?}", e))?;
 
-        // Create a simple rustls config that accepts any certificate for testing
-        let mut root_store = RootCertStore::empty();
-        // Add the self-signed certificate to the trust store for e2e
-        // Note: In a real environment, you'd use proper root certificates
-        let cert_pem = default_cert.cert.pem();
-        let cert_der = CertificateDer::pem_reader_iter(&mut Cursor::new(cert_pem))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(|e| anyhow::anyhow!("Failed to parse cert: {}", e))?;
-
-        root_store.add_parsable_certificates(cert_der);
-
         let config = ClientConfig::builder()
-            .with_root_certificates(root_store)
+            .dangerous()
+            .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCertVerifier))
             .with_no_client_auth();
 
         // Wrap in suppaftp's RustlsConnector

crates/e2e_test/src/protocols/webdav_core.rs

@@ -14,7 +14,7 @@
 
 //! Core WebDAV tests
 
-use crate::common::rustfs_binary_path;
+use crate::common::rustfs_binary_path_with_features;
 use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
 use anyhow::Result;
 use base64::Engine;
@@ -47,7 +47,7 @@ pub async fn test_webdav_core_operations() -> Result<()> {
 
     // Start server manually
     info!("Starting WebDAV server on {}", WEBDAV_ADDRESS);
-    let binary_path = rustfs_binary_path();
+    let binary_path = rustfs_binary_path_with_features(Some("ftps,webdav"));
     let mut server_process = Command::new(&binary_path)
         .env("RUSTFS_WEBDAV_ENABLE", "true")
         .env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)

crates/e2e_test/src/quota_test.rs

@@ -17,6 +17,15 @@ use aws_sdk_s3::Client;
 use serial_test::serial;
 use tracing::{debug, info};
 
+fn skip_without_awscurl() -> bool {
+    if crate::common::awscurl_available() {
+        return false;
+    }
+
+    info!("Skipping quota test because awscurl is not available");
+    true
+}
+
 /// Test environment setup for quota tests
 pub struct QuotaTestEnv {
     pub env: RustFSTestEnvironment,
@@ -233,6 +242,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_basic_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         // Create test bucket
@@ -269,6 +281,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_update_and_clear() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -302,6 +317,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_delete_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -336,6 +354,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_usage_tracking() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -372,6 +393,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_statistics() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -403,6 +427,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_check_api() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -441,6 +468,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_multiple_buckets() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         // Create two buckets in the same environment
@@ -479,6 +509,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_error_handling() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -512,6 +545,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_http_endpoints() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -575,6 +611,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_normal_user_permissions() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
         env.create_bucket().await?;
 
@@ -628,6 +667,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_copy_operations() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -671,6 +713,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_batch_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;
@@ -727,6 +772,9 @@ mod integration_tests {
     #[serial]
     async fn test_quota_multipart_upload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         init_logging();
+        if skip_without_awscurl() {
+            return Ok(());
+        }
         let env = QuotaTestEnv::new().await?;
 
         env.create_bucket().await?;

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -551,6 +551,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn get_live_events(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::GetLiveEventsRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::GetLiveEventsResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn start_profiling(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::StartProfilingRequest>,

crates/e2e_test/src/replication_extension_test.rs

@@ -0,0 +1,802 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
+use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+use http::header::{CONTENT_TYPE, HOST};
+use reqwest::StatusCode;
+use rustfs_signer::constants::UNSIGNED_PAYLOAD;
+use rustfs_signer::sign_v4;
+use s3s::Body;
+use serial_test::serial;
+use std::error::Error;
+
+async fn signed_request(
+    method: http::Method,
+    url: &str,
+    access_key: &str,
+    secret_key: &str,
+    body: Option<Vec<u8>>,
+    content_type: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let uri = url.parse::<http::Uri>()?;
+    let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
+    let mut request = http::Request::builder().method(method.clone()).uri(uri);
+    request = request.header(HOST, authority);
+    request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
+    if let Some(content_type) = content_type {
+        request = request.header(CONTENT_TYPE, content_type);
+    }
+
+    let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
+    let signed = sign_v4(request.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
+
+    let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
+    let client = local_http_client();
+    let mut request_builder = client.request(reqwest_method, url);
+    for (name, value) in signed.headers() {
+        request_builder = request_builder.header(name, value);
+    }
+    if let Some(body) = body {
+        request_builder = request_builder.body(body);
+    }
+
+    Ok(request_builder.send().await?)
+}
+
+async fn set_replication_target(
+    source_env: &RustFSTestEnvironment,
+    source_bucket: &str,
+    target_env: &RustFSTestEnvironment,
+    target_bucket: &str,
+) -> Result<String, Box<dyn Error + Send + Sync>> {
+    let body = serde_json::json!({
+        "endpoint": target_env.address,
+        "credentials": {
+            "accessKey": target_env.access_key,
+            "secretKey": target_env.secret_key
+        },
+        "targetbucket": target_bucket,
+        "secure": false,
+        "type": "replication"
+    });
+    let url = format!(
+        "{}/rustfs/admin/v3/set-remote-target?bucket={}",
+        source_env.url,
+        urlencoding::encode(source_bucket)
+    );
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &source_env.access_key,
+        &source_env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("set remote target failed: {status} {body}").into());
+    }
+
+    let body = response.bytes().await?;
+    let arn: String = serde_json::from_slice(&body)?;
+    Ok(arn)
+}
+
+async fn send_set_replication_target_request(
+    source_env: &RustFSTestEnvironment,
+    source_bucket: &str,
+    update: bool,
+    body: serde_json::Value,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let mut url = format!(
+        "{}/rustfs/admin/v3/set-remote-target?bucket={}",
+        source_env.url,
+        urlencoding::encode(source_bucket)
+    );
+    if update {
+        url.push_str("&update=true");
+    }
+    signed_request(
+        http::Method::PUT,
+        &url,
+        &source_env.access_key,
+        &source_env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await
+}
+
+async fn put_bucket_replication(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    target_arn: &str,
+) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let body = format!(
+        r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Role></Role>
+  <Rule>
+    <ID>rule-1</ID>
+    <Priority>1</Priority>
+    <Status>Enabled</Status>
+    <DeleteMarkerReplication>
+      <Status>Enabled</Status>
+    </DeleteMarkerReplication>
+    <ExistingObjectReplication>
+      <Status>Enabled</Status>
+    </ExistingObjectReplication>
+    <Destination>
+      <Bucket>{target_arn}</Bucket>
+    </Destination>
+  </Rule>
+</ReplicationConfiguration>"#
+    );
+    let url = format!("{}/{bucket}?replication", env.url);
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.into_bytes()),
+        Some("application/xml"),
+    )
+    .await?;
+
+    if response.status() != StatusCode::OK {
+        let status = response.status();
+        let body = response.text().await.unwrap_or_default();
+        return Err(format!("put bucket replication failed: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let client = env.create_s3_client();
+    client
+        .put_bucket_versioning()
+        .bucket(bucket)
+        .versioning_configuration(
+            VersioningConfiguration::builder()
+                .status(BucketVersioningStatus::Enabled)
+                .build(),
+        )
+        .send()
+        .await?;
+    Ok(())
+}
+
+async fn run_replication_check(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/{bucket}?replication-check", env.url);
+    signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
+async fn remove_replication_target(
+    env: &RustFSTestEnvironment,
+    bucket: &str,
+    arn: &str,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let url = format!(
+        "{}/rustfs/admin/v3/remove-remote-target?bucket={}&arn={}",
+        env.url,
+        urlencoding::encode(bucket),
+        urlencoding::encode(arn)
+    );
+    signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
+async fn remove_replication_target_request(
+    env: &RustFSTestEnvironment,
+    bucket: Option<&str>,
+    arn: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let mut url = format!("{}/rustfs/admin/v3/remove-remote-target", env.url);
+    let mut separator = '?';
+
+    if let Some(bucket) = bucket {
+        url.push(separator);
+        separator = '&';
+        url.push_str("bucket=");
+        url.push_str(&urlencoding::encode(bucket));
+    }
+
+    if let Some(arn) = arn {
+        url.push(separator);
+        url.push_str("arn=");
+        url.push_str(&urlencoding::encode(arn));
+    }
+
+    signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
+async fn list_replication_targets_request(
+    env: &RustFSTestEnvironment,
+    bucket: Option<&str>,
+) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
+    let mut url = format!("{}/rustfs/admin/v3/list-remote-targets", env.url);
+    if let Some(bucket) = bucket {
+        url.push_str("?bucket=");
+        url.push_str(&urlencoding::encode(bucket));
+    }
+    signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await
+}
+
+async fn build_replication_pair(
+    enable_target_versioning: bool,
+) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment, String), Box<dyn Error + Send + Sync>> {
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-check-src";
+    let target_bucket = "replication-check-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    if enable_target_versioning {
+        enable_bucket_versioning(&target_env, target_bucket).await?;
+    }
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    Ok((source_env, target_env, source_bucket.to_string()))
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (_source_env, _target_env, source_bucket) = build_replication_pair(true).await?;
+    let response = run_replication_check(&_source_env, &source_bucket).await?;
+
+    assert_eq!(response.status(), StatusCode::OK);
+    assert!(response.text().await?.is_empty());
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_check_rejects_target_without_object_lock() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-check-lock-src";
+    let target_bucket = "replication-check-lock-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client
+        .create_bucket()
+        .bucket(source_bucket)
+        .object_lock_enabled_for_bucket(true)
+        .send()
+        .await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
+    put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
+
+    let response = run_replication_check(&source_env, source_bucket).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("object lock"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_set_remote_target_rejects_unversioned_source_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-check-unversioned-src";
+    let target_bucket = "replication-check-unversioned-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let err = set_replication_target(&source_env, source_bucket, &target_env, target_bucket)
+        .await
+        .expect_err("unversioned source bucket should be rejected during remote target setup");
+    let err = err.to_string();
+
+    assert!(err.contains("400 Bad Request"), "unexpected set remote target error: {err}");
+    assert!(err.contains("InvalidRequest"), "unexpected set remote target error: {err}");
+    assert!(
+        err.to_ascii_lowercase().contains("not versioned"),
+        "unexpected set remote target error: {err}"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_check_rejects_unversioned_source_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "replication-check-source-unversioned";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+
+    let response = run_replication_check(&env, bucket).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("versioning"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_check_rejects_missing_replication_config() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "replication-check-missing-config";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&env, bucket).await?;
+
+    let response = run_replication_check(&env, bucket).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::NOT_FOUND);
+    assert!(body.contains("ReplicationConfigurationNotFoundError"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_replication_check_rejects_invalid_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let response = run_replication_check(&env, "replication-check-no-such-bucket").await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::NOT_FOUND);
+    assert!(body.contains("NoSuchBucket"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_set_remote_target_rejects_same_bucket_on_same_deployment() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "replication-check-same-target";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&env, bucket).await?;
+
+    let body = serde_json::json!({
+        "endpoint": env.address,
+        "credentials": {
+            "accessKey": env.access_key,
+            "secretKey": env.secret_key
+        },
+        "targetbucket": bucket,
+        "secure": false,
+        "type": "replication"
+    });
+    let url = format!("{}/rustfs/admin/v3/set-remote-target?bucket={}", env.url, urlencoding::encode(bucket));
+    let response = signed_request(
+        http::Method::PUT,
+        &url,
+        &env.access_key,
+        &env.secret_key,
+        Some(body.to_string().into_bytes()),
+        Some("application/json"),
+    )
+    .await?;
+
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("IncorrectEndpoint"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_set_remote_target_rejects_unversioned_target_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-check-src";
+    let target_bucket = "replication-check-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+
+    let err = set_replication_target(&source_env, source_bucket, &target_env, target_bucket)
+        .await
+        .expect_err("unversioned target bucket should be rejected during remote target setup");
+    assert!(err.to_string().contains("not versioned"), "unexpected set remote target error: {err}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_set_remote_target_update_requires_arn() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-update-needs-arn-src";
+    let target_bucket = "replication-update-needs-arn-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let response = send_set_replication_target_request(
+        &source_env,
+        source_bucket,
+        true,
+        serde_json::json!({
+            "endpoint": target_env.address,
+            "credentials": {
+                "accessKey": target_env.access_key,
+                "secretKey": target_env.secret_key
+            },
+            "targetbucket": target_bucket,
+            "secure": false,
+            "type": "replication"
+        }),
+    )
+    .await?;
+
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("arn is empty"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_set_remote_target_update_rejects_missing_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let source_bucket = "replication-update-missing-target-src";
+    let target_bucket = "replication-update-missing-target-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(source_bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+
+    enable_bucket_versioning(&source_env, source_bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let response = send_set_replication_target_request(
+        &source_env,
+        source_bucket,
+        true,
+        serde_json::json!({
+            "endpoint": target_env.address,
+            "credentials": {
+                "accessKey": target_env.access_key,
+                "secretKey": target_env.secret_key
+            },
+            "targetbucket": target_bucket,
+            "secure": false,
+            "type": "replication",
+            "arn": "arn:aws:s3:us-east-1:123456789012:replication::missing-target"
+        }),
+    )
+    .await?;
+
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("target not found"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_set_remote_target_rejects_invalid_target_url() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "replication-invalid-target-url-src";
+    let source_client = source_env.create_s3_client();
+    source_client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&source_env, bucket).await?;
+
+    let response = send_set_replication_target_request(
+        &source_env,
+        bucket,
+        false,
+        serde_json::json!({
+            "endpoint": "://invalid-target-url",
+            "credentials": {
+                "accessKey": "replication",
+                "secretKey": "replication"
+            },
+            "targetbucket": "target-bucket",
+            "secure": false,
+            "type": "replication"
+        }),
+    )
+    .await?;
+
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("invalid target url"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_list_remote_targets_rejects_empty_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let response = list_replication_targets_request(&env, Some("")).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("bucket is required"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_list_remote_targets_rejects_invalid_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let response = list_replication_targets_request(&env, Some("missing-replication-target-bucket")).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::NOT_FOUND);
+    assert!(body.contains("NoSuchBucket"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_remove_remote_target_rejects_missing_target() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut source_env = RustFSTestEnvironment::new().await?;
+    source_env.start_rustfs_server(vec![]).await?;
+
+    let mut target_env = RustFSTestEnvironment::new().await?;
+    target_env.start_rustfs_server_without_cleanup(vec![]).await?;
+
+    let bucket = "replication-remove-missing-target";
+    let target_bucket = "replication-remove-missing-target-dst";
+
+    let source_client = source_env.create_s3_client();
+    let target_client = target_env.create_s3_client();
+
+    source_client.create_bucket().bucket(bucket).send().await?;
+    target_client.create_bucket().bucket(target_bucket).send().await?;
+
+    enable_bucket_versioning(&source_env, bucket).await?;
+    enable_bucket_versioning(&target_env, target_bucket).await?;
+
+    let arn = set_replication_target(&source_env, bucket, &target_env, target_bucket).await?;
+
+    let first_remove = remove_replication_target(&source_env, bucket, &arn).await?;
+    assert_eq!(first_remove.status(), StatusCode::NO_CONTENT);
+
+    let response = remove_replication_target(&source_env, bucket, &arn).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("not found"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_remove_remote_target_rejects_missing_arn() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let bucket = "replication-remove-missing-arn";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
+    enable_bucket_versioning(&env, bucket).await?;
+
+    let response = remove_replication_target_request(&env, Some(bucket), None).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("arn is required"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_remove_remote_target_rejects_invalid_bucket() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let response = remove_replication_target_request(
+        &env,
+        Some("missing-replication-remove-bucket"),
+        Some("arn:aws:s3:us-east-1:123456789012:replication::missing"),
+    )
+    .await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::NOT_FOUND);
+    assert!(body.contains("NoSuchBucket"), "unexpected response: {body}");
+
+    Ok(())
+}
+
+#[tokio::test]
+#[serial]
+async fn test_remove_remote_target_rejects_target_used_by_replication() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (source_env, _target_env, source_bucket) = build_replication_pair(true).await?;
+    let targets_url = format!(
+        "{}/rustfs/admin/v3/list-remote-targets?bucket={}",
+        source_env.url,
+        urlencoding::encode(&source_bucket)
+    );
+    let targets_response = signed_request(
+        http::Method::GET,
+        &targets_url,
+        &source_env.access_key,
+        &source_env.secret_key,
+        None,
+        None,
+    )
+    .await?;
+    assert_eq!(targets_response.status(), StatusCode::OK);
+    let targets: Vec<serde_json::Value> = targets_response.json().await?;
+    let arn = targets
+        .first()
+        .and_then(|target| target.get("arn"))
+        .and_then(|arn| arn.as_str())
+        .ok_or("replication target arn missing")?
+        .to_string();
+
+    let response = remove_replication_target(&source_env, &source_bucket, &arn).await?;
+    let status = response.status();
+    let body = response.text().await?;
+
+    assert_eq!(status, StatusCode::BAD_REQUEST);
+    assert!(body.contains("InvalidRequest"), "unexpected response: {body}");
+    assert!(body.to_ascii_lowercase().contains("removal disallowed"), "unexpected response: {body}");
+
+    Ok(())
+}

crates/e2e_test/src/version_id_regression_test.rs

@@ -454,12 +454,12 @@ mod tests {
         Ok(())
     }
 
-    /// Test 7: PutObject should return "null" version_id when versioning is Suspended
+    /// Test 7: PutObject should omit version_id when versioning is Suspended
     #[tokio::test]
     #[serial]
-    async fn test_put_object_returns_null_version_id_with_suspended_versioning() {
+    async fn test_put_object_omits_version_id_with_suspended_versioning() {
         init_logging();
-        info!("🧪 TEST: PutObject returns null version_id with versioning suspended");
+        info!("🧪 TEST: PutObject omits version_id with versioning suspended");
 
         let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
         env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
@@ -489,13 +489,130 @@ mod tests {
 
         info!("📥 PutObject response - version_id: {:?}", output.version_id);
 
-        // When suspended, version_id must be "null"
+        // When suspended, version_id must be omitted
         assert_eq!(
-            output.version_id.as_deref(),
-            Some("null"),
-            "❌ FAILED: version_id should be 'null' when versioning is suspended"
+            output.version_id, None,
+            "❌ FAILED: version_id should be omitted when versioning is suspended"
         );
 
-        info!("✅ PASSED: PutObject correctly returns 'null' version_id");
+        info!("✅ PASSED: PutObject correctly omits version_id");
+    }
+
+    /// Test 8: CopyObject should omit version_id when versioning is Suspended
+    #[tokio::test]
+    #[serial]
+    async fn test_copy_object_omits_version_id_with_suspended_versioning() {
+        init_logging();
+        info!("🧪 TEST: CopyObject omits version_id with versioning suspended");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-copy-suspended-version-id";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+        suspend_versioning(&client, bucket)
+            .await
+            .expect("Failed to suspend versioning");
+
+        let source_key = "source-file.txt";
+        let dest_key = "dest-file.txt";
+        let content = b"Content to copy into suspended bucket";
+
+        client
+            .put_object()
+            .bucket(bucket)
+            .key(source_key)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("Failed to create source object");
+
+        let result = client
+            .copy_object()
+            .bucket(bucket)
+            .key(dest_key)
+            .copy_source(format!("{}/{}", bucket, source_key))
+            .send()
+            .await;
+
+        assert!(result.is_ok(), "CopyObject failed: {:?}", result.err());
+        let output = result.unwrap();
+
+        info!("📥 CopyObject response - version_id: {:?}", output.version_id);
+        assert_eq!(
+            output.version_id, None,
+            "❌ FAILED: version_id should be omitted when versioning is suspended"
+        );
+
+        info!("✅ PASSED: CopyObject correctly omits version_id");
+    }
+
+    /// Test 9: CompleteMultipartUpload should omit version_id when versioning is Suspended
+    #[tokio::test]
+    #[serial]
+    async fn test_multipart_upload_omits_version_id_with_suspended_versioning() {
+        init_logging();
+        info!("🧪 TEST: CompleteMultipartUpload omits version_id with versioning suspended");
+
+        let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
+        env.start_rustfs_server(vec![]).await.expect("Failed to start RustFS");
+
+        let client = create_s3_client(&env);
+        let bucket = "test-multipart-suspended-version-id";
+
+        create_bucket(&client, bucket).await.expect("Failed to create bucket");
+        suspend_versioning(&client, bucket)
+            .await
+            .expect("Failed to suspend versioning");
+
+        let key = "multipart-file.txt";
+        let content = b"Part 1 content for suspended multipart upload test";
+
+        let create_result = client
+            .create_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .send()
+            .await
+            .expect("Failed to create multipart upload");
+
+        let upload_id = create_result.upload_id().expect("No upload_id returned");
+
+        let upload_part_result = client
+            .upload_part()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .part_number(1)
+            .body(ByteStream::from_static(content))
+            .send()
+            .await
+            .expect("Failed to upload part");
+
+        let etag = upload_part_result.e_tag().expect("No etag returned").to_string();
+        let completed_part = CompletedPart::builder().part_number(1).e_tag(etag).build();
+        let completed_upload = CompletedMultipartUpload::builder().parts(completed_part).build();
+
+        let result = client
+            .complete_multipart_upload()
+            .bucket(bucket)
+            .key(key)
+            .upload_id(upload_id)
+            .multipart_upload(completed_upload)
+            .send()
+            .await;
+
+        assert!(result.is_ok(), "CompleteMultipartUpload failed: {:?}", result.err());
+        let output = result.unwrap();
+
+        info!("📥 CompleteMultipartUpload response - version_id: {:?}", output.version_id);
+        assert_eq!(
+            output.version_id, None,
+            "❌ FAILED: version_id should be omitted when versioning is suspended"
+        );
+
+        info!("✅ PASSED: CompleteMultipartUpload correctly omits version_id");
     }
 }