gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-04 12:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(tls): centralize runtime foundation (#3065)

Browse Source 
* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* feat(tls-runtime): add TLS debug state and admin handler

* refactor(tls-runtime): unify TLS debug consumer status view

* fix(tls): address PR3065 review feedback

* refactor(tls): align debug status payload types

* refactor(targets): harden TLS hot reload paths

* fix(targets): resolve review-4348251652 findings

* fix(targets): finalize tls runtime review follow-ups

* fix(targets): harden tls reload and review follow-ups

* fix(targets): align tls reload handling across targets

* fix(targets): finalize tls reload state and metrics updates

* chore(deps): trim unused TLS deps

* style(targets): normalize TLS reload formatting

* refactor(targets): introduce tls runtime adapter path

* chore: update workspace manifests for tls refactor

* fix(tls): stabilize material reload and audit workflow

* fix(targets): refresh tls fingerprint flow across sinks

* fix(tls): align runtime coordinator and http reader updates

* fix(sftp): simplify protocol error mapping

* fix(tls): harmonize material loading behavior

* fix(server): finalize tls material wiring in startup flow

* fix(protos): tighten tls generation cache and deps
This commit is contained in:
houseme
2026-05-24 14:41:15 +08:00
committed by GitHub
parent 8be787387c
commit d74e6eb042
67 changed files with 4978 additions and 1725 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/Cargo.toml
+3 -3
View File
@@ -85,6 +85,7 @@ rustfs-data-usage = { workspace = true }
rustfs-s3select-api = { workspace = true }
rustfs-s3select-query = { workspace = true }
rustfs-targets = { workspace = true }
rustfs-tls-runtime = { workspace = true }
rustfs-trusted-proxies = { workspace = true }
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-zip = { workspace = true }
@@ -127,6 +128,7 @@ serde_urlencoded = { workspace = true }
# Cryptography and Security
rustls = { workspace = true }
rustls-pki-types = { workspace = true }
subtle = { workspace = true }
jiff = { workspace = true }
time = { workspace = true, features = ["parsing", "formatting", "serde"] }
@@ -173,9 +175,7 @@ libsystemd.workspace = true
[target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
mimalloc = { workspace = true }
libmimalloc-sys = { version = "0.1.48", features = ["extended"] }
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
# Only enable pprof-based profiling on linux + gnu + x86_64.
[target.'cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))'.dependencies]
rustfs/src/admin/handlers/mod.rs
+2
View File
@@ -45,6 +45,7 @@ pub mod sts;
pub mod system;
mod target_descriptor;
pub mod tier;
pub mod tls_debug;
pub mod trace;
pub mod user;
pub mod user_iam;
@@ -75,6 +76,7 @@ mod tests {
let _metrics_handler = metrics::MetricsHandler {};
let _profile_handler = profile_admin::ProfileHandler {};
let _profile_status_handler = profile_admin::ProfileStatusHandler {};
let _tls_status_handler = tls_debug::TlsStatusHandler {};
let _heal_handler = heal::HealHandler {};
let _bg_heal_handler = heal::BackgroundHealStatusHandler {};
let _replication_metrics_handler = replication::GetReplicationMetricsHandler {};
rustfs/src/admin/handlers/site_replication.rs
+30 -53
View File
@@ -29,10 +29,7 @@ use http::header::{CONTENT_TYPE, HOST};
use http::{HeaderMap, HeaderValue, Uri};
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_config::{
DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_RUSTFS_TLS_PATH, ENV_TRUST_LEAF_CERT_AS_CA,
MAX_ADMIN_REQUEST_BODY_SIZE, RUSTFS_CA_CERT, RUSTFS_TLS_CERT,
};
use rustfs_config::{DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_TLS_PATH, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
use rustfs_ecstore::bucket::metadata::{
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
@@ -67,7 +64,9 @@ use rustfs_policy::policy::{
};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use rustfs_tls_runtime::load_global_outbound_tls_state;
use rustfs_utils::http::get_source_scheme;
use rustls_pki_types::pem::PemObject;
use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus,
Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, ReplicationConfiguration, ReplicationRule,
@@ -80,9 +79,9 @@ use serde::de::DeserializeOwned;
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
use time::OffsetDateTime;
use tokio::sync::OnceCell;
use tracing::warn;
use url::{Url, form_urlencoded};
use uuid::Uuid;
@@ -103,7 +102,7 @@ const SITE_REPLICATOR_SERVICE_ACCOUNT: &str = "site-replicator-0";
const SITE_REPLICATION_PEER_JOIN_PATH: &str = "/rustfs/admin/v3/site-replication/peer/join";
const SITE_REPLICATION_PEER_EDIT_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit";
const SITE_REPLICATION_PEER_REMOVE_PATH: &str = "/rustfs/admin/v3/site-replication/peer/remove";
static SITE_REPLICATION_PEER_CLIENT: OnceLock<Result<reqwest::Client, String>> = OnceLock::new();
static SITE_REPLICATION_PEER_CLIENT: OnceCell<Result<reqwest::Client, String>> = OnceCell::const_new();
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
struct SiteReplicationState {
@@ -448,56 +447,32 @@ async fn persist_site_replication_state(state: &SiteReplicationState) -> S3Resul
}
}
fn add_root_certificates_from_file(
mut builder: reqwest::ClientBuilder,
cert_path: &std::path::Path,
description: &str,
) -> S3Result<reqwest::ClientBuilder> {
if !cert_path.exists() {
return Ok(builder);
}
std::fs::read(cert_path).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to read {description} {}: {e}", cert_path.display()),
)
})?;
let certs_der = rustfs_utils::load_cert_bundle_der_bytes(cert_path.to_string_lossy().as_ref()).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to parse {description} {}: {e}", cert_path.display()),
)
})?;
for cert_der in certs_der {
let cert = reqwest::Certificate::from_der(&cert_der).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to load {description} {}: {e}", cert_path.display()),
)
})?;
builder = builder.add_root_certificate(cert);
}
Ok(builder)
}
fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
async fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
let mut builder = reqwest::Client::builder()
.timeout(SITE_REPLICATION_PEER_REQUEST_TIMEOUT)
.connect_timeout(SITE_REPLICATION_PEER_CONNECT_TIMEOUT)
.pool_idle_timeout(Some(Duration::from_secs(60)));
let tls_path = rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH);
if !tls_path.is_empty() {
let tls_dir = std::path::Path::new(&tls_path);
builder = add_root_certificates_from_file(builder, &tls_dir.join(RUSTFS_CA_CERT), "site-replication CA cert")?;
let outbound_tls = load_global_outbound_tls_state().await;
if let Some(root_ca_pem) = outbound_tls.root_ca_pem.as_ref() {
let mut reader = std::io::BufReader::new(root_ca_pem.as_slice());
let certs_der = rustls_pki_types::CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to parse published site-replication CA certs: {e}"),
)
})?;
if rustfs_utils::get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA) {
builder =
add_root_certificates_from_file(builder, &tls_dir.join(RUSTFS_TLS_CERT), "site-replication leaf cert as CA")?;
for cert_der in certs_der {
let cert = reqwest::Certificate::from_der(cert_der.as_ref()).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to load published site-replication CA cert: {e}"),
)
})?;
builder = builder.add_root_certificate(cert);
}
}
@@ -506,8 +481,10 @@ fn build_site_replication_peer_client() -> S3Result<reqwest::Client> {
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("build site replication peer client failed: {e}")))
}
fn site_replication_peer_client() -> S3Result<&'static reqwest::Client> {
let result = SITE_REPLICATION_PEER_CLIENT.get_or_init(|| build_site_replication_peer_client().map_err(|e| e.to_string()));
async fn site_replication_peer_client() -> S3Result<&'static reqwest::Client> {
let result = SITE_REPLICATION_PEER_CLIENT
.get_or_init(|| async { build_site_replication_peer_client().await.map_err(|e| e.to_string()) })
.await;
result.as_ref().map_err(|err| {
S3Error::with_message(
S3ErrorCode::InternalError,
@@ -969,7 +946,7 @@ async fn send_peer_admin_request<T: Serialize>(
.unwrap_or("us-east-1"),
);
let mut req = site_replication_peer_client()?.request(reqwest::Method::PUT, &url);
let mut req = site_replication_peer_client().await?.request(reqwest::Method::PUT, &url);
for (name, value) in signed.headers() {
req = req.header(name, value);
}
rustfs/src/admin/handlers/tls_debug.rs
+154
View File
@@ -0,0 +1,154 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::profile::authorize_profile_request;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::server::ADMIN_PREFIX;
use http::StatusCode;
use http::{HeaderMap, HeaderValue};
use hyper::Method;
use matchit::Params;
use rustfs_tls_runtime::{
OutboundOnlySnapshotArgs, TlsConsumerStatusItem, TlsDebugStatusResponse, TlsRuntimeStatusSnapshot,
summarize_global_outbound_tls_state,
};
use s3s::header::CONTENT_TYPE;
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
pub fn register_tls_debug_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/debug/tls/status").as_str(),
AdminOperation(&TlsStatusHandler {}),
)?;
Ok(())
}
pub struct TlsStatusHandler {}
const PROTOS_GRPC_CHANNEL_CONSUMER: &str = "protos_grpc_channel";
const RIO_HTTP_READER_CONSUMER: &str = "rio_http_reader";
const ECSTORE_TRANSITION_CLIENT_CONSUMER: &str = "ecstore_transition_client";
#[async_trait::async_trait]
impl Operation for TlsStatusHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
authorize_profile_request(&req).await?;
let tls_path = rustfs_utils::get_env_opt_str(rustfs_config::ENV_RUSTFS_TLS_PATH).unwrap_or_default();
let outbound = summarize_global_outbound_tls_state().await;
let status = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: tls_path,
generation: outbound.generation.0,
reload_enabled: rustfs_utils::get_env_bool(
rustfs_config::ENV_TLS_RELOAD_ENABLE,
rustfs_config::DEFAULT_TLS_RELOAD_ENABLE,
),
detect_mode: "poll",
last_attempt_time: None,
last_success_time: None,
last_error: None,
has_roots: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
});
let payload = TlsDebugStatusResponse::builder(status)
.push_consumers([
TlsConsumerStatusItem {
consumer: PROTOS_GRPC_CHANNEL_CONSUMER,
generation: outbound.generation.0,
has_root_ca: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
},
TlsConsumerStatusItem {
consumer: RIO_HTTP_READER_CONSUMER,
generation: outbound.generation.0,
has_root_ca: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
},
TlsConsumerStatusItem {
consumer: ECSTORE_TRANSITION_CLIENT_CONSUMER,
generation: outbound.generation.0,
has_root_ca: outbound.has_root_ca,
has_mtls_identity: outbound.has_mtls_identity,
},
])
.build();
let body = serde_json::to_vec(&payload)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize tls status failed: {e}")))?;
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(body)), headers))
}
}
#[cfg(test)]
mod tests {
use super::TlsStatusHandler;
use crate::admin::router::Operation;
use http::{Extensions, HeaderMap, Uri};
use hyper::Method;
use matchit::Params;
use rustfs_tls_runtime::{OutboundOnlySnapshotArgs, TlsConsumerStatusItem, TlsRuntimeStatusSnapshot};
use s3s::{Body, S3ErrorCode, S3Request};
fn build_tls_status_request() -> S3Request<Body> {
S3Request {
input: Body::empty(),
method: Method::GET,
uri: Uri::from_static("/rustfs/admin/debug/tls/status"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
#[tokio::test]
async fn tls_status_handler_rejects_missing_credentials() {
let result = TlsStatusHandler {}.call(build_tls_status_request(), Params::new()).await;
let err = result.expect_err("tls status handler must reject unauthenticated requests");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
}
#[test]
fn tls_debug_response_schema_is_structured() {
let status = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: "/tmp/tls".to_string(),
generation: 7,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: Some(1),
last_success_time: Some(2),
last_error: Some("x".to_string()),
has_roots: true,
has_mtls_identity: false,
});
let payload = rustfs_tls_runtime::TlsDebugStatusResponse::builder(status)
.push_consumers([TlsConsumerStatusItem {
consumer: "protos_grpc_channel",
generation: 7,
has_root_ca: true,
has_mtls_identity: false,
}])
.build();
let json = serde_json::to_value(payload).expect("json should serialize");
assert!(json.get("foundation").is_some());
assert!(json.get("consumers").is_some());
assert!(json["consumers"].is_array());
}
}
rustfs/src/admin/mod.rs
+2 -1
View File
@@ -28,7 +28,7 @@ mod route_registration_test;
use handlers::{
audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota,
rebalance, replication, site_replication, sts, system, tier, user,
rebalance, replication, site_replication, sts, system, tier, tls_debug, user,
};
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
@@ -65,6 +65,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
replication::register_replication_route(&mut r)?;
site_replication::register_site_replication_route(&mut r)?;
profile_admin::register_profiling_route(&mut r)?;
tls_debug::register_tls_debug_route(&mut r)?;
kms::register_kms_route(&mut r)?;
oidc::register_oidc_route(&mut r)?;
rustfs/src/admin/route_registration_test.rs
+3 -1
View File
@@ -15,7 +15,7 @@
use crate::admin::{
handlers::{
audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin,
quota, rebalance, replication, site_replication, sts, system, tier, user,
quota, rebalance, replication, site_replication, sts, system, tier, tls_debug, user,
},
router::{AdminOperation, S3Router},
};
@@ -59,6 +59,7 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
replication::register_replication_route(router).expect("register replication route");
site_replication::register_site_replication_route(router).expect("register site replication route");
profile_admin::register_profiling_route(router).expect("register profile route");
tls_debug::register_tls_debug_route(router).expect("register tls debug route");
kms::register_kms_route(router).expect("register kms route");
oidc::register_oidc_route(router).expect("register oidc route");
}
@@ -157,6 +158,7 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/resync/op"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/state/edit"));
assert_route(&router, Method::GET, &admin_path("/debug/pprof/profile"));
assert_route(&router, Method::GET, &admin_path("/debug/tls/status"));
assert_route(&router, Method::POST, &admin_path("/v3/kms/create-key"));
assert_route(&router, Method::POST, &admin_path("/v3/kms/key/create"));
rustfs/src/app/object_usecase.rs
+4 -5
View File
@@ -92,7 +92,10 @@ use rustfs_s3select_api::{
query::{Context, Query},
};
use rustfs_s3select_query::get_global_db;
use rustfs_targets::EventName;
use rustfs_targets::{
EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE,
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP,
@@ -108,10 +111,6 @@ use rustfs_utils::http::{
insert_str, remove_str,
};
use rustfs_utils::path::{is_dir_object, path_join_buf};
use rustfs_utils::{
CompressionAlgorithm, extract_params_header, extract_resp_elements, get_request_host, get_request_port,
get_request_user_agent,
};
use rustfs_zip::CompressionFormat;
use s3s::dto::*;
use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
rustfs/src/main.rs
+11 -4
View File
@@ -221,12 +221,16 @@ async fn async_main() -> Result<()> {
debug!("rustls crypto provider already installed, skipping aws-lc-rs default install");
}
// Initialize TLS outbound material (root CAs, mTLS identity) if configured.
// Server-side TLS acceptor is built separately inside start_http_server()
// using the same TlsMaterialSnapshot loading logic.
// Server-side TLS acceptor is built separately inside start_http_server().
// Single load via tls-runtime; outbound is enriched with platform CAs and
// published to the global state before any HTTP listener starts.
if let Some(tls_path) = config.tls_path.as_deref().map(str::trim).filter(|path| !path.is_empty()) {
match rustfs::server::tls_material::TlsMaterialSnapshot::load(tls_path).await {
match rustfs::server::tls_material::load_tls_material(tls_path).await {
Ok(snapshot) => {
snapshot.apply_outbound().await;
use rustfs_tls_runtime::{TlsGeneration, publish_global_outbound_tls_state, record_tls_generation};
let generation = TlsGeneration(rustfs_common::get_global_outbound_tls_generation().saturating_add(1));
publish_global_outbound_tls_state(generation, &snapshot.outbound).await;
record_tls_generation("rustfs_server_startup", generation.0);
info!(target: "rustfs::main", "TLS outbound material initialized from {}", tls_path);
}
Err(e) => {
@@ -234,6 +238,9 @@ async fn async_main() -> Result<()> {
return Err(Error::other(e.to_string()));
}
}
if rustfs_obs::observability_metric_enabled() {
rustfs_tls_runtime::init_tls_metrics();
}
}
// Run parameters
rustfs/src/server/http.rs
+26 -12
View File
@@ -25,7 +25,9 @@ use crate::server::{
BodylessStatusFixLayer, ConditionalCorsLayer, EmptyBodyContentLengthCompatLayer, HeadRequestBodyFixLayer,
ObjectAttributesEtagFixLayer, PublicHealthEndpointLayer, RedirectLayer, RequestContextLayer, S3ErrorMessageCompatLayer,
},
tls_material::{TlsAcceptorHolder, TlsHandshakeFailureKind, TlsMaterialSnapshot, spawn_reload_loop},
tls_material::{
TlsAcceptorHolder, TlsHandshakeFailureKind, build_acceptor_from_loaded, load_tls_material, spawn_reload_loop,
},
};
use crate::storage;
use crate::storage::rpc::InternodeRpcService;
@@ -217,22 +219,34 @@ pub async fn start_http_server(
let tls_path = config.tls_path.as_deref().map(str::trim).unwrap_or_default();
let tls_path_configured = !tls_path.is_empty();
// Load TLS materials and build server acceptor.
// Note: outbound material (root CAs, mTLS identity) is already applied in main.rs.
let tls_snapshot = TlsMaterialSnapshot::load(tls_path)
.await
.map_err(|e| Error::other(e.to_string()))?;
let tls_acceptor = tls_snapshot.build_tls_acceptor(tls_path).await.map_err(|e| {
if tls_path_configured {
// Load TLS materials and build server acceptor in a single pass.
// Outbound material (root CAs, mTLS identity) was already published in main.rs;
// this load is needed for the server-side TLS acceptor and reload loop.
let tls_acceptor = if tls_path_configured {
let snapshot = load_tls_material(tls_path).await.map_err(|e| {
Error::other(format!(
"TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but TLS acceptor initialization failed: {}",
tls_path, e
))
} else {
Error::other(e.to_string())
})?;
let acceptor = build_acceptor_from_loaded(snapshot.server, std::path::Path::new(tls_path))
.await
.map_err(|e| Error::other(e.to_string()))?;
// Fail closed: if TLS was explicitly configured but no server certificates
// were found, refuse to start rather than silently falling back to plain HTTP.
match acceptor {
None => {
return Err(Error::other(format!(
"TLS is explicitly configured via RUSTFS_TLS_PATH/tls_path='{}' but no server certificates were found",
tls_path
)));
}
Some(a) => Some(a),
}
})?;
} else {
None
};
let tls_enabled = tls_acceptor.is_some();
let protocol = if tls_enabled { "https" } else { "http" };
rustfs/src/server/tls_material.rs
+222 -372
View File
@@ -12,32 +12,32 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified TLS Material Snapshot
//! TLS material loading, enrichment, and acceptor construction.
//!
//! Provides a single loading point for all TLS materials, eliminating duplicate
//! directory scanning and PEM parsing between outbound and inbound paths.
//!
//! Usage:
//! 1. Call `TlsMaterialSnapshot::load(tls_path)` once at startup.
//! 2. Call `snapshot.apply_outbound()` to set global root CAs and mTLS identity.
//! 3. TLS acceptor construction is handled internally during server startup.
//! Single-load architecture: `rustfs_tls_runtime::TlsMaterialSnapshot` loads all
//! TLS materials (server certs + outbound CAs + mTLS identity) from disk in one
//! pass. This module enriches the outbound material with platform-specific CAs
//! (system roots, leaf-as-CA, mTLS env-var path overrides) and builds the server
//! TLS acceptor directly from the pre-loaded server material — no double reads.
use rustfs_common::{MtlsIdentityPem, set_global_mtls_identity, set_global_root_cert};
use rustfs_common::{MtlsIdentityPem, get_global_outbound_tls_generation};
use rustfs_config::{
DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL,
DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_SERVER_MTLS_ENABLE,
ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_SYSTEM_CA,
RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_PUBLIC_CERT,
RUSTFS_TLS_CERT, RUSTFS_TLS_KEY,
RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_TLS_CERT,
};
use rustfs_tls_runtime::{
ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions,
build_webpki_client_verifier, create_multi_cert_resolver, publish_global_outbound_tls_state, record_tls_generation,
record_tls_reload_result, record_tls_reload_skipped,
};
use rustfs_utils::{get_env_bool, get_env_opt_str};
use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::RwLock;
use std::time::Duration;
use std::{fs, io};
use tokio_rustls::TlsAcceptor;
use tracing::{debug, info, warn};
@@ -54,310 +54,86 @@ const SYSTEM_CA_PATHS: &[&str] = &[
"/usr/share/pki/ca-trust-legacy/ca-bundle.legacy.crt", // RHEL legacy
];
/// Outbound TLS material for client connections (inter-node RPC).
#[derive(Debug, Clone)]
pub struct OutboundTlsMaterial {
/// Concatenated PEM-encoded root CA certificates.
pub root_ca_pem: Vec<u8>,
/// Optional mTLS client identity.
pub mtls_identity: Option<MtlsIdentityPem>,
}
// ── Public API ──
/// Complete TLS material snapshot loaded once at startup.
#[derive(Debug)]
pub struct TlsMaterialSnapshot {
/// Material for outbound client connections.
pub outbound: OutboundTlsMaterial,
}
impl TlsMaterialSnapshot {
/// Load all TLS materials from the given directory.
///
/// This is the single entry point that replaces both the old
/// `cert.rs::init_cert()` and `http.rs::setup_tls_acceptor()` loading logic.
pub async fn load(tls_path: &str) -> Result<Self, TlsMaterialError> {
if tls_path.is_empty() {
info!("No TLS path configured; skipping TLS material loading");
return Ok(Self::empty());
}
let tls_dir = PathBuf::from(tls_path);
// Load outbound material (root CAs + mTLS identity)
let outbound = load_outbound_material(&tls_dir).await?;
Ok(Self { outbound })
/// Load all TLS materials from the given directory in a single pass.
///
/// Uses `rustfs_tls_runtime::TlsMaterialSnapshot` as the single disk-read point,
/// then enriches the outbound material with platform-specific CAs that the runtime
/// crate does not handle (system roots, leaf-as-CA, mTLS env-var path overrides).
///
/// Returns the fully enriched snapshot ready for both outbound publishing and
/// acceptor construction.
pub async fn load_tls_material(tls_path: &str) -> Result<rustfs_tls_runtime::TlsMaterialSnapshot, TlsMaterialError> {
if tls_path.is_empty() {
return Err(TlsMaterialError::Io("TLS path is empty".into()));
}
/// Apply outbound material to global state (root CAs, mTLS identity).
pub async fn apply_outbound(&self) {
if !self.outbound.root_ca_pem.is_empty() {
set_global_root_cert(self.outbound.root_ca_pem.clone()).await;
info!("Configured custom root certificates for inter-node communication");
}
set_global_mtls_identity(self.outbound.mtls_identity.clone()).await;
}
let tls_source = TlsSource::from_directory(tls_path);
let mut snapshot = rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source)
.await
.map_err(map_runtime_tls_error)?;
/// Build a `TlsAcceptorHolder` from the loaded snapshot.
///
/// This is the single place that constructs the server `ServerConfig`,
/// handling both multi-cert (SNI resolver) and single-cert fallback.
/// Returns `None` if no TLS certificates are available.
pub(crate) async fn build_tls_acceptor(&self, tls_path: &str) -> Result<Option<Arc<TlsAcceptorHolder>>, TlsMaterialError> {
if tls_path.is_empty() {
return Ok(None);
}
enrich_outbound(&mut snapshot.outbound, &PathBuf::from(tls_path)).await?;
let tls_dir = validate_server_tls_directory(tls_path)?;
let mtls_verifier = rustfs_utils::build_webpki_client_verifier(
rustfs_utils::WebPkiClientVerifierOptions::builder(&tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT)
.enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE))
.build(),
)
.map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?;
match load_server_certificate_material(&tls_dir)? {
ServerCertificateMaterial::SingleCert { certs, key } => {
let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier)?;
info!("Created TLS acceptor with root single certificate");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
ServerCertificateMaterial::MultiCert { cert_key_pairs } => {
let resolver = rustfs_utils::create_multi_cert_resolver(cert_key_pairs)
.map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?;
let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier)?;
info!("Created TLS acceptor with SNI resolver");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
}
}
fn empty() -> Self {
Self {
outbound: OutboundTlsMaterial {
root_ca_pem: Vec::new(),
mtls_identity: None,
},
}
}
Ok(snapshot)
}
fn validate_server_tls_directory(tls_path: &str) -> Result<PathBuf, TlsMaterialError> {
let tls_dir = PathBuf::from(tls_path);
let metadata = fs::metadata(&tls_dir).map_err(|e| {
let kind = if e.kind() == io::ErrorKind::NotFound {
"TLS directory does not exist"
} else {
"Failed to inspect TLS directory"
};
TlsMaterialError::Io(format!("{kind}: {} ({e})", tls_dir.display()))
})?;
if !metadata.is_dir() {
return Err(TlsMaterialError::Io(format!("TLS path is not a directory: {}", tls_dir.display())));
}
Ok(tls_dir)
}
fn load_server_certificate_material(tls_dir: &Path) -> Result<ServerCertificateMaterial, TlsMaterialError> {
let root_cert_pair = inspect_root_server_cert_pair(tls_dir);
if has_discoverable_domain_cert_pair(tls_dir)? {
return load_multi_cert_material(tls_dir);
}
match root_cert_pair {
RootCertPairState::Complete { cert_path, key_path } => load_root_single_cert_material(&cert_path, &key_path),
RootCertPairState::MissingCert { cert_path } => Err(TlsMaterialError::Io(format!(
"TLS root certificate file missing: {}",
cert_path.display()
))),
RootCertPairState::MissingKey { key_path } => {
Err(TlsMaterialError::Io(format!("TLS root private key file missing: {}", key_path.display())))
}
RootCertPairState::Absent => Err(TlsMaterialError::Io(format!(
"No usable TLS certificate material found under '{}'",
tls_dir.display()
))),
}
}
fn inspect_root_server_cert_pair(tls_dir: &Path) -> RootCertPairState {
let cert_path = tls_dir.join(RUSTFS_TLS_CERT);
let key_path = tls_dir.join(RUSTFS_TLS_KEY);
let cert_exists = cert_path.exists();
let key_exists = key_path.exists();
match (cert_exists, key_exists) {
(true, true) => RootCertPairState::Complete { cert_path, key_path },
(false, false) => RootCertPairState::Absent,
(false, true) => RootCertPairState::MissingCert { cert_path },
(true, false) => RootCertPairState::MissingKey { key_path },
}
}
fn has_discoverable_domain_cert_pair(tls_dir: &Path) -> Result<bool, TlsMaterialError> {
for entry in
fs::read_dir(tls_dir).map_err(|e| TlsMaterialError::Io(format!("read TLS directory {}: {e}", tls_dir.display())))?
{
let entry = entry.map_err(|e| TlsMaterialError::Io(format!("read TLS directory entry in {}: {e}", tls_dir.display())))?;
let path = entry.path();
if !path.is_dir() {
continue;
}
let Some(domain_name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
if !is_discoverable_cert_domain_dir(domain_name) {
continue;
}
if path.join(RUSTFS_TLS_CERT).exists() && path.join(RUSTFS_TLS_KEY).exists() {
return Ok(true);
}
}
Ok(false)
}
fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool {
!domain_name.starts_with('.')
}
fn load_root_single_cert_material(cert_path: &Path, key_path: &Path) -> Result<ServerCertificateMaterial, TlsMaterialError> {
let cert_path_str = cert_path
.to_str()
.ok_or_else(|| TlsMaterialError::Io(format!("invalid UTF-8 in TLS root certificate path: {cert_path:?}")))?;
let key_path_str = key_path
.to_str()
.ok_or_else(|| TlsMaterialError::Io(format!("invalid UTF-8 in TLS root private key path: {key_path:?}")))?;
let certs = rustfs_utils::load_certs(cert_path_str)
.map_err(|e| TlsMaterialError::Io(format!("load root TLS certificate {}: {e}", cert_path.display())))?;
let key = rustfs_utils::load_private_key(key_path_str)
.map_err(|e| TlsMaterialError::Io(format!("load root TLS private key {}: {e}", key_path.display())))?;
Ok(ServerCertificateMaterial::SingleCert { certs, key })
}
fn load_multi_cert_material(tls_dir: &Path) -> Result<ServerCertificateMaterial, TlsMaterialError> {
let cert_key_pairs = rustfs_utils::load_all_certs_from_directory(
rustfs_utils::CertDirectoryLoadOptions::builder(tls_dir, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY).build(),
/// Build a TLS acceptor from pre-loaded server material.
///
/// Takes the server material already loaded by `load_tls_material` and constructs
/// the `rustls::ServerConfig` without any additional disk reads (except the mTLS
/// client CA verifier, which reads the client CA cert from disk).
pub(crate) async fn build_acceptor_from_loaded(
server: Option<RuntimeServerTlsMaterial>,
tls_dir: &Path,
) -> Result<Option<Arc<TlsAcceptorHolder>>, TlsMaterialError> {
let mtls_verifier = build_webpki_client_verifier(
WebPkiClientVerifierOptions::builder(tls_dir, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CA_CERT)
.enabled(get_env_bool(ENV_SERVER_MTLS_ENABLE, DEFAULT_SERVER_MTLS_ENABLE))
.build(),
)
.map_err(|e| TlsMaterialError::Io(format!("discover multi-cert TLS certificates under '{}': {e}", tls_dir.display())))?;
.map_err(|e| TlsMaterialError::Io(format!("build mTLS verifier: {e}")))?;
Ok(ServerCertificateMaterial::MultiCert { cert_key_pairs })
}
enum RootCertPairState {
Complete { cert_path: PathBuf, key_path: PathBuf },
MissingCert { cert_path: PathBuf },
MissingKey { key_path: PathBuf },
Absent,
}
enum ServerCertificateMaterial {
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
MultiCert {
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
},
}
// ── Server Config Construction ──
/// Certificate source for building a `ServerConfig`.
enum ServerCertSource {
/// Pre-built SNI resolver from multi-cert directory.
Resolver(Arc<dyn rustls::server::ResolvesServerCert + Send + Sync>),
/// Single certificate/key pair.
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
}
/// Build a `ServerConfig` with standardized ALPN, session cache, and key log settings.
///
/// This is the single place for `ServerConfig` construction, used by both
/// initial startup and hot-reload.
fn build_server_config(
cert_source: ServerCertSource,
mtls_verifier: Option<Arc<dyn rustls::server::danger::ClientCertVerifier>>,
) -> Result<rustls::ServerConfig, TlsMaterialError> {
let mut config = match cert_source {
ServerCertSource::Resolver(resolver) => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_cert_resolver(resolver)
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_cert_resolver(resolver)
}
match server {
Some(RuntimeServerTlsMaterial::SingleCert { certs, key }) => {
let config = build_server_config(ServerCertSource::SingleCert { certs, key }, mtls_verifier)?;
info!("Created TLS acceptor with root single certificate");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
ServerCertSource::SingleCert { certs, key } => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))?
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))?
}
Some(RuntimeServerTlsMaterial::MultiCert { cert_key_pairs }) => {
let resolver = create_multi_cert_resolver(cert_key_pairs)
.map_err(|e| TlsMaterialError::Parse(format!("build multi-cert resolver: {e}")))?;
let config = build_server_config(ServerCertSource::Resolver(Arc::new(resolver)), mtls_verifier)?;
info!("Created TLS acceptor with SNI resolver");
let acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
Ok(Some(Arc::new(TlsAcceptorHolder::new(acceptor))))
}
};
config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000);
if tls_key_log() {
config.key_log = Arc::new(rustls::KeyLogFile::new());
None => Ok(None),
}
Ok(config)
}
/// Checks if TLS key logging is enabled.
///
/// # Returns
/// * A boolean indicating whether TLS key logging is enabled based on the `RUSTFS_TLS_KEYLOG` environment variable.
///
fn tls_key_log() -> bool {
get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG)
}
// ── Outbound Material Loading ──
/// Load root CA certificates and mTLS identity for outbound connections.
async fn load_outbound_material(tls_dir: &Path) -> Result<OutboundTlsMaterial, TlsMaterialError> {
let mut root_ca_pem = Vec::new();
// ── Outbound Enrichment ──
/// Enrich the outbound TLS material with platform-specific additions that the
/// tls-runtime crate does not handle:
/// 1. Optional: server leaf cert as root CA (`RUSTFS_TRUST_LEAF_CERT_AS_CA`)
/// 2. Optional: system root CAs from platform paths (`RUSTFS_TRUST_SYSTEM_CA`)
/// 3. Optional: mTLS identity from env-var-overridden paths
async fn enrich_outbound(outbound: &mut rustfs_tls_runtime::OutboundTlsMaterial, tls_dir: &Path) -> Result<(), TlsMaterialError> {
// 1. Optional: load leaf certs as root CAs
if get_env_bool(ENV_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_LEAF_CERT_AS_CA)
&& load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut root_ca_pem).await
&& load_cert_file_by_name(tls_dir, RUSTFS_TLS_CERT, &mut outbound.root_ca_pem).await
{
info!("Loaded leaf certificate(s) as root CA as per RUSTFS_TRUST_LEAF_CERT_AS_CA");
}
// 2. Load public.crt and ca.crt
load_cert_file(&tls_dir.join(RUSTFS_PUBLIC_CERT), &mut root_ca_pem, "CA certificate").await;
load_cert_file(&tls_dir.join(RUSTFS_CA_CERT), &mut root_ca_pem, "CA certificate").await;
// 3. Optional: load system root CAs
// 2. Optional: load system root CAs
if get_env_bool(ENV_TRUST_SYSTEM_CA, DEFAULT_TRUST_SYSTEM_CA) {
let mut system_loaded = false;
for path in SYSTEM_CA_PATHS {
if load_cert_file(Path::new(path), &mut root_ca_pem, "system root certificates").await {
if load_cert_file(Path::new(path), &mut outbound.root_ca_pem, "system root certificates").await {
system_loaded = true;
info!("Loaded system root certificates from {}", path);
break;
@@ -370,23 +146,31 @@ async fn load_outbound_material(tls_dir: &Path) -> Result<OutboundTlsMaterial, T
info!("Loading system root certificates disabled via RUSTFS_TRUST_SYSTEM_CA");
}
// 4. Load optional mTLS identity
let mtls_identity = load_mtls_identity(tls_dir).await?;
// 3. Optional: override mTLS identity from env-var paths
let env_cert = get_env_opt_str(ENV_MTLS_CLIENT_CERT);
let env_key = get_env_opt_str(ENV_MTLS_CLIENT_KEY);
if (env_cert.is_some() || env_key.is_some())
&& let Some(identity) = load_mtls_identity_from_overridden_paths(env_cert.as_deref(), env_key.as_deref(), tls_dir).await?
{
outbound.mtls_identity = Some(identity);
}
Ok(OutboundTlsMaterial {
root_ca_pem,
mtls_identity,
})
Ok(())
}
/// Load mTLS client identity from the TLS directory.
async fn load_mtls_identity(tls_dir: &Path) -> Result<Option<MtlsIdentityPem>, TlsMaterialError> {
let client_cert_path = match get_env_opt_str(ENV_MTLS_CLIENT_CERT) {
/// Load mTLS client identity when env-var path overrides are set.
/// If neither env var is set, returns Ok(None) (the tls-runtime already loaded
/// identity from the default path).
async fn load_mtls_identity_from_overridden_paths(
env_cert: Option<&str>,
env_key: Option<&str>,
tls_dir: &Path,
) -> Result<Option<MtlsIdentityPem>, TlsMaterialError> {
let client_cert_path = match env_cert {
Some(p) => PathBuf::from(p),
None => tls_dir.join(RUSTFS_CLIENT_CERT_FILENAME),
};
let client_key_path = match get_env_opt_str(ENV_MTLS_CLIENT_KEY) {
let client_key_path = match env_key {
Some(p) => PathBuf::from(p),
None => tls_dir.join(RUSTFS_CLIENT_KEY_FILENAME),
};
@@ -406,7 +190,6 @@ async fn load_mtls_identity(tls_dir: &Path) -> Result<Option<MtlsIdentityPem>, T
.await
.map_err(|e| TlsMaterialError::Io(format!("read client key {client_key_path:?}: {e}")))?;
// Validate parse-ability
let mut reader = std::io::Cursor::new(&cert_pem);
if CertificateDer::pem_reader_iter(&mut reader).next().is_none() {
return Err(TlsMaterialError::Parse("no valid certificate in client cert PEM".into()));
@@ -418,6 +201,17 @@ async fn load_mtls_identity(tls_dir: &Path) -> Result<Option<MtlsIdentityPem>, T
Ok(Some(MtlsIdentityPem { cert_pem, key_pem }))
}
// ── Helpers ──
fn map_runtime_tls_error(err: rustfs_tls_runtime::TlsRuntimeError) -> TlsMaterialError {
match &err {
rustfs_tls_runtime::TlsRuntimeError::Material(msg) | rustfs_tls_runtime::TlsRuntimeError::Publication(msg) => {
TlsMaterialError::Parse(msg.clone())
}
_ => TlsMaterialError::Io(err.to_string()),
}
}
/// Load a single certificate file and append PEM data.
/// Returns true if the file was successfully loaded.
async fn load_cert_file(path: &Path, pem_data: &mut Vec<u8>, desc: &str) -> bool {
@@ -457,17 +251,16 @@ async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec<
if fname == cert_name && load_cert_file(&entry.path(), pem_data, "certificate").await {
loaded = true;
}
} else if ft.is_dir() {
// Only check direct subdirectories (one level deep)
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await
&& sub_ft.is_file()
{
let fname = sub_entry.file_name().to_string_lossy().to_string();
if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await {
loaded = true;
}
} else if ft.is_dir()
&& let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await
{
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await
&& sub_ft.is_file()
{
let fname = sub_entry.file_name().to_string_lossy().to_string();
if fname == cert_name && load_cert_file(&sub_entry.path(), pem_data, "certificate").await {
loaded = true;
}
}
}
@@ -476,12 +269,66 @@ async fn load_cert_file_by_name(dir: &Path, cert_name: &str, pem_data: &mut Vec<
loaded
}
/// Errors that can occur during TLS material loading.
// ── Server Config Construction ──
enum ServerCertSource {
Resolver(Arc<dyn rustls::server::ResolvesServerCert + Send + Sync>),
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
}
fn build_server_config(
cert_source: ServerCertSource,
mtls_verifier: Option<Arc<dyn rustls::server::danger::ClientCertVerifier>>,
) -> Result<rustls::ServerConfig, TlsMaterialError> {
let mut config = match cert_source {
ServerCertSource::Resolver(resolver) => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_cert_resolver(resolver)
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_cert_resolver(resolver)
}
}
ServerCertSource::SingleCert { certs, key } => {
if let Some(verifier) = mtls_verifier {
rustls::ServerConfig::builder()
.with_client_cert_verifier(verifier)
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert with mTLS: {e}")))?
} else {
rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.map_err(|e| TlsMaterialError::Io(format!("configure single cert: {e}")))?
}
}
};
config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec(), b"http/1.0".to_vec()];
config.session_storage = rustls::server::ServerSessionMemoryCache::new(10000);
if tls_key_log() {
config.key_log = Arc::new(rustls::KeyLogFile::new());
}
Ok(config)
}
fn tls_key_log() -> bool {
get_env_bool(ENV_TLS_KEYLOG, DEFAULT_TLS_KEYLOG)
}
// ── Errors ──
#[derive(Debug)]
pub enum TlsMaterialError {
/// I/O error (file read, directory access).
Io(String),
/// PEM parsing error.
Parse(String),
}
@@ -498,7 +345,6 @@ impl std::error::Error for TlsMaterialError {}
// ── TLS Handshake Error Classification ──
/// Structured classification of TLS handshake failures.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TlsHandshakeFailureKind {
UnexpectedEof,
@@ -509,7 +355,6 @@ pub(crate) enum TlsHandshakeFailureKind {
}
impl TlsHandshakeFailureKind {
/// Classify a TLS accept error into a structured failure kind.
pub(crate) fn classify(err_msg: &str) -> Self {
if err_msg.contains("unexpected EOF") || err_msg.contains("handshake eof") {
Self::UnexpectedEof
@@ -524,7 +369,6 @@ impl TlsHandshakeFailureKind {
}
}
/// Metric label string for Prometheus.
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::UnexpectedEof => "UNEXPECTED_EOF",
@@ -538,10 +382,6 @@ impl TlsHandshakeFailureKind {
// ── TLS Acceptor Holder (for hot reload) ──
/// Holds the current TLS acceptor and supports atomic swap for certificate rotation.
///
/// Uses `RwLock` so that multiple readers (per-connection `get()` calls)
/// do not block each other. The write lock is held only briefly during swap.
pub(crate) struct TlsAcceptorHolder {
current: RwLock<Arc<TlsAcceptor>>,
}
@@ -553,7 +393,6 @@ impl TlsAcceptorHolder {
}
}
/// Get the current TLS acceptor for handling a new connection.
#[inline]
pub(crate) fn get(&self) -> Arc<TlsAcceptor> {
match self.current.read() {
@@ -562,7 +401,6 @@ impl TlsAcceptorHolder {
}
}
/// Atomically replace the TLS acceptor with a new one.
fn swap(&self, new_holder: &TlsAcceptorHolder) {
let new_acceptor = new_holder.get();
match self.current.write() {
@@ -575,7 +413,11 @@ impl TlsAcceptorHolder {
}
}
// ── Reload Loop ──
/// Spawn a background task that periodically checks for TLS certificate changes.
/// Single load per tick: loads once via tls-runtime, enriches, publishes outbound,
/// and builds acceptor — no double reads.
pub(crate) fn spawn_reload_loop(tls_path: String, holder: Arc<TlsAcceptorHolder>) {
let enabled = get_env_bool(ENV_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_ENABLE);
if !enabled {
@@ -584,31 +426,49 @@ pub(crate) fn spawn_reload_loop(tls_path: String, holder: Arc<TlsAcceptorHolder>
}
let interval_secs = rustfs_utils::get_env_u64(ENV_TLS_RELOAD_INTERVAL, DEFAULT_TLS_RELOAD_INTERVAL).max(5);
let tls_source = TlsSource::from_directory(&tls_path);
info!("TLS certificate hot reload enabled, checking every {}s", interval_secs);
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(interval_secs));
let tls_dir = PathBuf::from(&tls_path);
loop {
interval.tick().await;
match TlsMaterialSnapshot::load(&tls_path).await {
Ok(snapshot) => {
// Always refresh outbound material (root CAs, mTLS identity) on reload.
snapshot.apply_outbound().await;
match rustfs_tls_runtime::TlsMaterialSnapshot::load(&tls_source).await {
Ok(mut snapshot) => {
if let Err(e) = enrich_outbound(&mut snapshot.outbound, &tls_dir).await {
record_tls_reload_result("rustfs_server_reload_loop", "enrich_err", None, None);
warn!("TLS outbound enrichment failed (will retry): {}", e);
continue;
}
match snapshot.build_tls_acceptor(&tls_path).await {
let generation = get_global_outbound_tls_generation().saturating_add(1);
publish_global_outbound_tls_state(TlsGeneration(generation), &snapshot.outbound).await;
record_tls_generation("rustfs_server_reload_loop", generation);
if !snapshot.outbound.root_ca_pem.is_empty() {
info!("Configured custom root certificates for inter-node communication");
}
match build_acceptor_from_loaded(snapshot.server, &tls_dir).await {
Ok(Some(new_holder)) => {
info!("TLS certificates reloaded successfully");
holder.swap(&new_holder);
record_tls_reload_result("rustfs_server_reload_loop", "ok", None, Some(generation));
}
Ok(None) => {
record_tls_reload_skipped("rustfs_server_reload_loop", "no_acceptor");
warn!("TLS reload returned no acceptor despite configured TLS path; keeping previous acceptor")
}
Err(e) => warn!("TLS certificate reload failed (will retry): {}", e),
Err(e) => {
record_tls_reload_result("rustfs_server_reload_loop", "acceptor_err", None, Some(generation));
warn!("TLS certificate reload failed (will retry): {}", e)
}
}
}
Err(e) => {
record_tls_reload_result("rustfs_server_reload_loop", "load_err", None, None);
warn!("TLS material reload failed (will retry): {}", e);
}
}
@@ -636,20 +496,19 @@ mod tests {
fn write_test_cert_pair(dir: &Path, subject: &str) {
let CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![subject.to_string()]).unwrap();
fs::write(dir.join(RUSTFS_TLS_CERT), cert.pem()).unwrap();
fs::write(dir.join(RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap();
fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap();
}
#[tokio::test]
async fn build_tls_acceptor_accepts_root_single_cert_with_trailing_slash() {
async fn build_acceptor_accepts_root_single_cert_with_trailing_slash() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
write_test_cert_pair(temp_dir.path(), "localhost");
let snapshot = TlsMaterialSnapshot::load(&format!("{}/", temp_dir.path().display()))
let snapshot = load_tls_material(&format!("{}/", temp_dir.path().display()))
.await
.expect("TLS material load should succeed");
let acceptor = snapshot
.build_tls_acceptor(&format!("{}/", temp_dir.path().display()))
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("root single-cert TLS acceptor should build");
@@ -657,7 +516,7 @@ mod tests {
}
#[tokio::test]
async fn build_tls_acceptor_accepts_symlinked_root_single_cert_directory() {
async fn build_acceptor_accepts_symlinked_root_single_cert_directory() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
let real_dir = temp_dir.path().join("tls-real");
@@ -670,11 +529,10 @@ mod tests {
#[cfg(windows)]
std::os::windows::fs::symlink_dir(&real_dir, &symlink_dir).unwrap();
let snapshot = TlsMaterialSnapshot::load(symlink_dir.to_str().unwrap())
let snapshot = load_tls_material(symlink_dir.to_str().unwrap())
.await
.expect("TLS material load through symlink should succeed");
let acceptor = snapshot
.build_tls_acceptor(symlink_dir.to_str().unwrap())
let acceptor = build_acceptor_from_loaded(snapshot.server, &symlink_dir)
.await
.expect("TLS acceptor should build through symlink");
@@ -682,62 +540,55 @@ mod tests {
}
#[tokio::test]
async fn build_tls_acceptor_rejects_missing_tls_directory() {
async fn build_acceptor_rejects_missing_tls_directory() {
let temp_dir = TempDir::new().unwrap();
let missing_dir = temp_dir.path().join("missing");
let snapshot = TlsMaterialSnapshot::empty();
let err = match snapshot.build_tls_acceptor(missing_dir.to_str().unwrap()).await {
Ok(_) => panic!("missing TLS directory should fail"),
Err(err) => err,
};
let err = load_tls_material(missing_dir.to_str().unwrap())
.await
.expect_err("missing TLS directory should fail");
assert!(err.to_string().contains("TLS directory does not exist"));
}
#[tokio::test]
async fn build_tls_acceptor_rejects_tls_path_that_is_not_directory() {
async fn build_acceptor_rejects_tls_path_that_is_not_directory() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("tls-file");
fs::write(&file_path, "not-a-directory").unwrap();
let snapshot = TlsMaterialSnapshot::empty();
let err = match snapshot.build_tls_acceptor(file_path.to_str().unwrap()).await {
Ok(_) => panic!("regular file TLS path should fail"),
Err(err) => err,
};
let err = load_tls_material(file_path.to_str().unwrap())
.await
.expect_err("regular file TLS path should fail");
assert!(err.to_string().contains("TLS path is not a directory"));
}
#[tokio::test]
async fn build_tls_acceptor_rejects_empty_tls_directory() {
async fn build_acceptor_returns_none_for_empty_tls_directory() {
let temp_dir = TempDir::new().unwrap();
let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap())
let snapshot = load_tls_material(temp_dir.path().to_str().unwrap())
.await
.expect("outbound TLS load for empty dir should succeed");
.expect("TLS material load for empty dir should succeed");
let err = match snapshot.build_tls_acceptor(temp_dir.path().to_str().unwrap()).await {
Ok(_) => panic!("empty TLS directory should fail"),
Err(err) => err,
};
assert!(err.to_string().contains("No usable TLS certificate material found"));
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("empty TLS directory should be treated as no acceptor");
assert!(acceptor.is_none());
}
#[tokio::test]
async fn build_tls_acceptor_still_supports_multi_cert_directories() {
async fn build_acceptor_still_supports_multi_cert_directories() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
let domain_dir = temp_dir.path().join("example.com");
fs::create_dir(&domain_dir).unwrap();
write_test_cert_pair(&domain_dir, "example.com");
let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap())
let snapshot = load_tls_material(temp_dir.path().to_str().unwrap())
.await
.expect("TLS material load for multi-cert dir should succeed");
let acceptor = snapshot
.build_tls_acceptor(temp_dir.path().to_str().unwrap())
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("multi-cert TLS acceptor should build");
@@ -745,7 +596,7 @@ mod tests {
}
#[tokio::test]
async fn build_tls_acceptor_prefers_multi_cert_when_root_and_domain_pairs_both_exist() {
async fn build_acceptor_prefers_multi_cert_when_root_and_domain_pairs_both_exist() {
ensure_rustls_crypto_provider();
let temp_dir = TempDir::new().unwrap();
write_test_cert_pair(temp_dir.path(), "default.local");
@@ -754,11 +605,10 @@ mod tests {
fs::create_dir(&domain_dir).unwrap();
write_test_cert_pair(&domain_dir, "example.com");
let snapshot = TlsMaterialSnapshot::load(temp_dir.path().to_str().unwrap())
let snapshot = load_tls_material(temp_dir.path().to_str().unwrap())
.await
.expect("TLS material load for mixed root/domain layout should succeed");
let acceptor = snapshot
.build_tls_acceptor(temp_dir.path().to_str().unwrap())
let acceptor = build_acceptor_from_loaded(snapshot.server, temp_dir.path())
.await
.expect("multi-cert TLS acceptor should still build when root pair also exists");
rustfs/src/storage/helper.rs
+2 -2
View File
@@ -27,10 +27,10 @@ use rustfs_io_metrics::record_s3_op;
use rustfs_notify::{EventArgsBuilder, notifier_global};
use rustfs_s3_ops::{S3Operation, operation_matches_event_name};
use rustfs_s3_types::EventName;
use rustfs_utils::{
use rustfs_targets::{
extract_params_header, extract_req_params, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
http::headers::AMZ_REQUEST_ID,
};
use rustfs_utils::http::headers::AMZ_REQUEST_ID;
use s3s::{S3Request, S3Response, S3Result};
use serde_json::Value;
use std::future::Future;