diff --git a/Cargo.lock b/Cargo.lock index b42e5a9eb..750df5563 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7345,6 +7345,7 @@ dependencies = [ "aws-config", "aws-credential-types", "aws-sdk-s3", + "aws-smithy-http-client", "aws-smithy-types", "base64 0.22.1", "base64-simd", diff --git a/Cargo.toml b/Cargo.toml index 58d64c362..237235c64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -182,6 +182,7 @@ atomic_enum = "0.3.0" aws-config = { version = "1.8.14" } aws-credential-types = { version = "1.2.13" } aws-sdk-s3 = { version = "1.124.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } +aws-smithy-http-client = { version = "1.1.11", default-features = false, features = ["default-client", "rustls-aws-lc"] } aws-smithy-types = { version = "1.4.5" } backtrace = "0.3.76" base64 = "0.22.1" diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 8d2d147ec..ddfb6bcd9 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -112,6 +112,7 @@ google-cloud-auth = { workspace = true } aws-config = { workspace = true } faster-hex = { workspace = true } ratelimit = { workspace = true } +aws-smithy-http-client.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 4f286e35e..e8efafbef 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -36,8 +36,10 @@ use aws_sdk_s3::types::{ }; use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput}; use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus}; +use aws_smithy_http_client::{Builder as SmithyHttpClientBuilder, tls as smithy_tls}; use http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; use reqwest::Client as HttpClient; +use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RUSTFS_CA_CERT, RUSTFS_TLS_CERT}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, @@ -50,6 +52,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::error::Error; use std::fmt; +use std::path::Path; use std::str::FromStr as _; use std::sync::Arc; use std::sync::OnceLock; @@ -640,12 +643,23 @@ impl BucketTargetSys { format!("http://{}", target.endpoint) }; - let config = S3Config::builder() + let mut config_builder = S3Config::builder() .endpoint_url(endpoint.clone()) .credentials_provider(SharedCredentialsProvider::new(creds)) .region(SdkRegion::new(target.region.clone())) - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) - .build(); + .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()); + + if should_force_path_style(target) { + config_builder = config_builder.force_path_style(true); + } + + if target.secure + && let Some(http_client) = build_aws_s3_http_client_from_tls_path().await + { + config_builder = config_builder.http_client(http_client); + } + + let config = config_builder.build(); Ok(TargetClient { endpoint, @@ -789,6 +803,72 @@ impl BucketTargetSys { } } +async fn build_aws_s3_http_client_from_tls_path() -> Option { + let tls_path = std::env::var("RUSTFS_TLS_PATH").ok()?; + if tls_path.is_empty() { + return None; + } + + let tls_dir = Path::new(&tls_path); + let mut trust_store = smithy_tls::TrustStore::default(); + let mut has_custom_certs = false; + + let ca_path = tls_dir.join(RUSTFS_CA_CERT); + match tokio::fs::read(&ca_path).await { + Ok(pem) => { + trust_store.add_pem_certificate(pem); + has_custom_certs = true; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!("failed to read custom CA bundle {:?} for replication client: {}", ca_path, e), + } + + if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) { + let leaf_cert_path = tls_dir.join(RUSTFS_TLS_CERT); + match tokio::fs::read(&leaf_cert_path).await { + Ok(pem) => { + trust_store.add_pem_certificate(pem); + has_custom_certs = true; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => warn!("failed to read leaf cert {:?} for replication client trust store: {}", leaf_cert_path, e), + } + } + + if !has_custom_certs { + return None; + } + + let tls_context = match smithy_tls::TlsContext::builder().with_trust_store(trust_store).build() { + Ok(ctx) => ctx, + Err(e) => { + warn!("failed to build AWS SDK TLS context for replication client: {}", e); + return None; + } + }; + + Some( + SmithyHttpClientBuilder::new() + .tls_provider(smithy_tls::Provider::rustls(smithy_tls::rustls_provider::CryptoMode::AwsLc)) + .tls_context(tls_context) + .build_https(), + ) +} + +fn should_force_path_style(target: &BucketTarget) -> bool { + match target.path.trim().to_ascii_lowercase().as_str() { + // Explicit DNS/virtual-hosted-style requested by user. + "dns" | "off" | "false" => false, + // Explicit path-style or legacy boolean-like values. + "path" | "on" | "true" => true, + // `auto` and empty are defaulted to path-style for custom S3-compatible endpoints. + // RustFS/MinIO-style deployments typically do not configure virtual-hosted-style routing. + "auto" | "" => true, + // Unknown values: prefer compatibility with S3-compatible services. + _ => true, + } +} + // generate ARN that is unique to this target type fn generate_arn(t: &BucketTarget, depl_id: &str) -> String { let uuid = if depl_id.is_empty() { diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index 6f3c83ae6..023e504a5 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -2051,6 +2051,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e); return rinfo; } + } else if e.raw_response().is_some_and(|resp| resp.status().as_u16() == 404) { + // Some HEAD Object 404 responses are surfaced by the AWS SDK as `response error` + // instead of `service error (NotFound)`. Treat raw HTTP 404 as object-not-found + // so replication can proceed with PUT. } else { rinfo.error = Some(e.to_string()); warn!("replication head_object failed bucket:{} arn:{} error:{}", bucket, tgt_client.arn, e); diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 81f2bc9fa..e299ef6de 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -2518,6 +2518,7 @@ impl DefaultObjectUsecase { let mut opts: ObjectOptions = del_opts(&bucket, &key, version_id, &req.headers, metadata) .await .map_err(ApiError::from)?; + let force_delete = opts.delete_prefix; let lock_cfg = BucketObjectLockSys::get(&bucket).await; if lock_cfg.is_some() && opts.delete_prefix { @@ -2541,6 +2542,17 @@ impl DefaultObjectUsecase { return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); }; + let replicate_force_delete = force_delete + && !replica + && has_replication_rules( + &bucket, + &[ObjectToDelete { + object_name: key.clone(), + ..Default::default() + }], + ) + .await; + // Check Object Lock retention before deletion // TODO: Future optimization (separate PR) - If performance becomes critical under high delete load: // 1. Integrate OptimizedFileCache (file_cache.rs) into the read_version() path @@ -2602,6 +2614,19 @@ impl DefaultObjectUsecase { }); if obj_info.name.is_empty() { + if replicate_force_delete { + schedule_replication_delete(DeletedObjectReplicationInfo { + delete_object: rustfs_ecstore::store_api::DeletedObject { + object_name: key.clone(), + force_delete: true, + ..Default::default() + }, + bucket: bucket.clone(), + event_type: REPLICATE_INCOMING_DELETE.to_string(), + ..Default::default() + }) + .await; + } return Ok(S3Response::with_status(DeleteObjectOutput::default(), StatusCode::NO_CONTENT)); }