fix(replication): handle TLS CA trust and force-delete replication edge cases (#1983)

Cargo.lock

@@ -7345,6 +7345,7 @@ dependencies = [
  "aws-config",
  "aws-credential-types",
  "aws-sdk-s3",
+ "aws-smithy-http-client",
  "aws-smithy-types",
  "base64 0.22.1",
  "base64-simd",

Cargo.toml

@@ -182,6 +182,7 @@ atomic_enum = "0.3.0"
 aws-config = { version = "1.8.14" }
 aws-credential-types = { version = "1.2.13" }
 aws-sdk-s3 = { version = "1.124.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
+aws-smithy-http-client = { version = "1.1.11", default-features = false, features = ["default-client", "rustls-aws-lc"] }
 aws-smithy-types = { version = "1.4.5" }
 backtrace = "0.3.76"
 base64 = "0.22.1"

crates/ecstore/Cargo.toml

@@ -112,6 +112,7 @@ google-cloud-auth = { workspace = true }
 aws-config = { workspace = true }
 faster-hex = { workspace = true }
 ratelimit = { workspace = true }
+aws-smithy-http-client.workspace = true
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -36,8 +36,10 @@ use aws_sdk_s3::types::{
 };
 use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
 use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
+use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls};
 use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
 use reqwest::Client as HttpClient;
+use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT};
 use rustfs_filemeta::{ReplicationStatusType, ReplicationType};
 use rustfs_utils::http::{
     AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
@@ -50,6 +52,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::error::Error;
 use std::fmt;
+use std::path::Path;
 use std::str::FromStr as _;
 use std::sync::Arc;
 use std::sync::OnceLock;
@@ -640,12 +643,23 @@ impl BucketTargetSys {
             format!("http://{}", target.endpoint)
         };
 
-        let config = S3Config::builder()
+        let mut config_builder = S3Config::builder()
             .endpoint_url(endpoint.clone())
             .credentials_provider(SharedCredentialsProvider::new(creds))
             .region(SdkRegion::new(target.region.clone()))
-            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
-            .build();
+            .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
+
+        if should_force_path_style(target) {
+            config_builder = config_builder.force_path_style(true);
+        }
+
+        if target.secure
+            && let Some(http_client) = build_aws_s3_http_client_from_tls_path().await
+        {
+            config_builder = config_builder.http_client(http_client);
+        }
+
+        let config = config_builder.build();
 
         Ok(TargetClient {
             endpoint,
@@ -789,6 +803,72 @@ impl BucketTargetSys {
     }
 }
 
+async fn build_aws_s3_http_client_from_tls_path() -> Option<aws_sdk_s3::config::SharedHttpClient> {
+    let tls_path = std::env::var("RUSTFS_TLS_PATH").ok()?;
+    if tls_path.is_empty() {
+        return None;
+    }
+
+    let tls_dir = Path::new(&tls_path);
+    let mut trust_store = smithy_tls::TrustStore::default();
+    let mut has_custom_certs = false;
+
+    let ca_path = tls_dir.join(RUSTFS_CA_CERT);
+    match tokio::fs::read(&ca_path).await {
+        Ok(pem) => {
+            trust_store.add_pem_certificate(pem);
+            has_custom_certs = true;
+        }
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+        Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e),
+    }
+
+    if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
+        let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT);
+        match tokio::fs::read(&leaf_cert_path).await {
+            Ok(pem) => {
+                trust_store.add_pem_certificate(pem);
+                has_custom_certs = true;
+            }
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+            Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e),
+        }
+    }
+
+    if !has_custom_certs {
+        return None;
+    }
+
+    let tls_context = match smithy_tls::TlsContext::builder().with_trust_store(trust_store).build() {
+        Ok(ctx) => ctx,
+        Err(e) => {
+            warn!("failed to build AWS SDK TLS context for replication client: {}", e);
+            return None;
+        }
+    };
+
+    Some(
+        SmithyHttpClientBuilder::new()
+            .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc))
+            .tls_context(tls_context)
+            .build_https(),
+    )
+}
+
+fn should_force_path_style(target: &BucketTarget) -> bool {
+    match target.path.trim().to_ascii_lowercase().as_str() {
+        // Explicit DNS/virtual-hosted-style requested by user.
+        "dns" | "off" | "false" => false,
+        // Explicit path-style or legacy boolean-like values.
+        "path" | "on" | "true" => true,
+        // `auto` and empty are defaulted to path-style for custom S3-compatible endpoints.
+        // RustFS/MinIO-style deployments typically do not configure virtual-hosted-style routing.
+        "auto" | "" => true,
+        // Unknown values: prefer compatibility with S3-compatible services.
+        _ => true,
+    }
+}
+
 // generate ARN that is unique to this target type
 fn generate_arn(t: &BucketTarget, depl_id: &str) -> String {
     let uuid = if depl_id.is_empty() {

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -2051,6 +2051,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
                         warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);
                         return rinfo;
                     }
+                } else if e.raw_response().is_some_and(|resp| resp.status().as_u16() == 404) {
+                    // Some HEAD Object 404 responses are surfaced by the AWS SDK as `response error`
+                    // instead of `service error (NotFound)`. Treat raw HTTP 404 as object-not-found
+                    // so replication can proceed with PUT.
                 } else {
                     rinfo.error = Some(e.to_string());
                     warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e);

rustfs/src/app/object_usecase.rs

@@ -2518,6 +2518,7 @@ impl DefaultObjectUsecase {
         let mut opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, metadata)
             .await
             .map_err(ApiError::from)?;
+        let force_delete = opts.delete_prefix;
 
         let lock_cfg = BucketObjectLockSys::get(&bucket).await;
         if lock_cfg.is_some() && opts.delete_prefix {
@@ -2541,6 +2542,17 @@ impl DefaultObjectUsecase {
             return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
         };
 
+        let replicate_force_delete = force_delete
+            && !replica
+            && has_replication_rules(
+                &bucket,
+                &[ObjectToDelete {
+                    object_name: key.clone(),
+                    ..Default::default()
+                }],
+            )
+            .await;
+
         // Check Object Lock retention before deletion
         // TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
         // 1. Integrate OptimizedFileCache (file_cache.rs) into the read_version() path
@@ -2602,6 +2614,19 @@ impl DefaultObjectUsecase {
         });
 
         if obj_info.name.is_empty() {
+            if replicate_force_delete {
+                schedule_replication_delete(DeletedObjectReplicationInfo {
+                    delete_object: rustfs_ecstore::store_api::DeletedObject {
+                        object_name: key.clone(),
+                        force_delete: true,
+                        ..Default::default()
+                    },
+                    bucket: bucket.clone(),
+                    event_type: REPLICATE_INCOMING_DELETE.to_string(),
+                    ..Default::default()
+                })
+                .await;
+            }
             return Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT));
         }