gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-10 15:16:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(tls): centralize runtime foundation (#3065)

Browse Source 
* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* feat(tls-runtime): add TLS debug state and admin handler

* refactor(tls-runtime): unify TLS debug consumer status view

* fix(tls): address PR3065 review feedback

* refactor(tls): align debug status payload types

* refactor(targets): harden TLS hot reload paths

* fix(targets): resolve review-4348251652 findings

* fix(targets): finalize tls runtime review follow-ups

* fix(targets): harden tls reload and review follow-ups

* fix(targets): align tls reload handling across targets

* fix(targets): finalize tls reload state and metrics updates

* chore(deps): trim unused TLS deps

* style(targets): normalize TLS reload formatting

* refactor(targets): introduce tls runtime adapter path

* chore: update workspace manifests for tls refactor

* fix(tls): stabilize material reload and audit workflow

* fix(targets): refresh tls fingerprint flow across sinks

* fix(tls): align runtime coordinator and http reader updates

* fix(sftp): simplify protocol error mapping

* fix(tls): harmonize material loading behavior

* fix(server): finalize tls material wiring in startup flow

* fix(protos): tighten tls generation cache and deps
This commit is contained in:
houseme
2026-05-24 14:41:15 +08:00
committed by GitHub
parent 8be787387c
commit d74e6eb042
67 changed files with 4978 additions and 1725 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/tls-runtime/src/certs.rs
+451
View File
@@ -0,0 +1,451 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustls::RootCertStore;
use rustls::server::{
ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni, WebPkiClientVerifier, danger::ClientCertVerifier,
};
use rustls::sign::CertifiedKey;
use rustls_pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
use std::collections::HashMap;
use std::io::Error;
use std::path::PathBuf;
use std::sync::Arc;
use std::{fs, io};
use tracing::{debug, warn};
#[derive(Debug, Clone)]
pub struct CertDirectoryLoadOptions {
dir_path: PathBuf,
cert_filename: String,
key_filename: String,
}
impl CertDirectoryLoadOptions {
pub fn builder(
dir_path: impl Into<PathBuf>,
cert_filename: impl Into<String>,
key_filename: impl Into<String>,
) -> CertDirectoryLoadOptionsBuilder {
CertDirectoryLoadOptionsBuilder {
dir_path: dir_path.into(),
cert_filename: cert_filename.into(),
key_filename: key_filename.into(),
}
}
fn validate(&self) -> io::Result<()> {
if self.cert_filename.is_empty() {
return Err(certs_error("certificate filename cannot be empty".to_string()));
}
if self.key_filename.is_empty() {
return Err(certs_error("private key filename cannot be empty".to_string()));
}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct CertDirectoryLoadOptionsBuilder {
dir_path: PathBuf,
cert_filename: String,
key_filename: String,
}
impl CertDirectoryLoadOptionsBuilder {
pub fn cert_filename(mut self, cert_filename: impl Into<String>) -> Self {
self.cert_filename = cert_filename.into();
self
}
pub fn key_filename(mut self, key_filename: impl Into<String>) -> Self {
self.key_filename = key_filename.into();
self
}
pub fn build(self) -> CertDirectoryLoadOptions {
CertDirectoryLoadOptions {
dir_path: self.dir_path,
cert_filename: self.cert_filename,
key_filename: self.key_filename,
}
}
}
#[derive(Debug, Clone)]
pub struct WebPkiClientVerifierOptions {
tls_path: PathBuf,
enabled: bool,
client_ca_cert_filename: String,
fallback_ca_cert_filename: String,
}
impl WebPkiClientVerifierOptions {
pub fn builder(
tls_path: impl Into<PathBuf>,
client_ca_cert_filename: impl Into<String>,
fallback_ca_cert_filename: impl Into<String>,
) -> WebPkiClientVerifierOptionsBuilder {
WebPkiClientVerifierOptionsBuilder {
tls_path: tls_path.into(),
enabled: false,
client_ca_cert_filename: client_ca_cert_filename.into(),
fallback_ca_cert_filename: fallback_ca_cert_filename.into(),
}
}
}
#[derive(Debug, Clone)]
pub struct WebPkiClientVerifierOptionsBuilder {
tls_path: PathBuf,
enabled: bool,
client_ca_cert_filename: String,
fallback_ca_cert_filename: String,
}
impl WebPkiClientVerifierOptionsBuilder {
pub fn enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
pub fn client_ca_cert_filename(mut self, client_ca_cert_filename: impl Into<String>) -> Self {
self.client_ca_cert_filename = client_ca_cert_filename.into();
self
}
pub fn fallback_ca_cert_filename(mut self, fallback_ca_cert_filename: impl Into<String>) -> Self {
self.fallback_ca_cert_filename = fallback_ca_cert_filename.into();
self
}
pub fn build(self) -> WebPkiClientVerifierOptions {
WebPkiClientVerifierOptions {
tls_path: self.tls_path,
enabled: self.enabled,
client_ca_cert_filename: self.client_ca_cert_filename,
fallback_ca_cert_filename: self.fallback_ca_cert_filename,
}
}
}
pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
let mut reader = io::BufReader::new(cert_file);
let certs = CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| certs_error(format!("certificate file {filename} format error:{e:?}")))?;
if certs.is_empty() {
return Err(certs_error(format!("No valid certificate was found in the certificate file {filename}")));
}
Ok(certs)
}
pub fn load_cert_bundle_der_bytes(path: &str) -> io::Result<Vec<Vec<u8>>> {
let pem = fs::read(path)?;
let mut reader = io::BufReader::new(&pem[..]);
let certs = CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| certs_error(format!("Failed to parse PEM certs from {path}: {e}")))?;
Ok(certs.into_iter().map(|c| c.to_vec()).collect())
}
pub fn build_webpki_client_verifier(options: WebPkiClientVerifierOptions) -> io::Result<Option<Arc<dyn ClientCertVerifier>>> {
if !options.enabled {
return Ok(None);
}
let tls_path = &options.tls_path;
let ca_path = mtls_ca_bundle_path(&options).ok_or_else(|| {
Error::other(format!(
"mTLS is enabled but missing {}/{} (or fallback {}/{})",
tls_path.display(),
options.client_ca_cert_filename,
tls_path.display(),
options.fallback_ca_cert_filename
))
})?;
let ca_path = ca_path
.to_str()
.ok_or_else(|| Error::other(format!("Invalid UTF-8 in mTLS CA path: {ca_path:?}")))?;
let der_list = load_cert_bundle_der_bytes(ca_path)?;
let mut store = RootCertStore::empty();
for der in der_list {
store
.add(der.into())
.map_err(|e| Error::other(format!("Invalid client CA cert: {e}")))?;
}
let verifier = WebPkiClientVerifier::builder(Arc::new(store))
.build()
.map_err(|e| Error::other(format!("Build client cert verifier failed: {e}")))?;
Ok(Some(verifier))
}
fn mtls_ca_bundle_path(options: &WebPkiClientVerifierOptions) -> Option<PathBuf> {
let p1 = options.tls_path.join(&options.client_ca_cert_filename);
if p1.exists() {
return Some(p1);
}
let p2 = options.tls_path.join(&options.fallback_ca_cert_filename);
if p2.exists() {
return Some(p2);
}
None
}
pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {filename}: {e}")))?;
let mut reader = io::BufReader::new(keyfile);
PrivateKeyDer::from_pem_reader(&mut reader)
.map_err(|e| certs_error(format!("failed to parse private key in {filename}: {e}")))
}
pub fn certs_error(err: String) -> Error {
Error::other(err)
}
fn is_discoverable_cert_domain_dir(domain_name: &str) -> bool {
!domain_name.starts_with('.')
}
pub fn load_all_certs_from_directory(
options: CertDirectoryLoadOptions,
) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
options.validate()?;
let mut cert_key_pairs = HashMap::new();
let dir = options.dir_path.as_path();
if !dir.exists() || !dir.is_dir() {
return Err(certs_error(format!(
"The certificate directory does not exist or is not a directory: {}",
dir.display()
)));
}
let root_cert_path = dir.join(&options.cert_filename);
let root_key_path = dir.join(&options.key_filename);
if root_cert_path.exists() && root_key_path.exists() {
debug!("find the root directory certificate: {:?}", root_cert_path);
let root_cert_str = root_cert_path
.to_str()
.ok_or_else(|| certs_error(format!("Invalid UTF-8 in root certificate path: {root_cert_path:?}")))?;
let root_key_str = root_key_path
.to_str()
.ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {root_key_path:?}")))?;
match load_cert_key_pair(root_cert_str, root_key_str) {
Ok((certs, key)) => {
cert_key_pairs.insert("default".to_string(), (certs, key));
}
Err(e) => {
warn!("unable to load root directory certificate: {}", e);
}
}
}
for entry in fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
let domain_name: &str = path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| certs_error(format!("invalid domain name directory:{path:?}")))?;
if !is_discoverable_cert_domain_dir(domain_name) {
debug!("skip internal certificate directory: {:?}", path);
continue;
}
let cert_path = path.join(&options.cert_filename);
let key_path = path.join(&options.key_filename);
if cert_path.exists() && key_path.exists() {
debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path);
let cert_path = match cert_path.to_str() {
Some(path) => path,
None => {
warn!("skip domain certificate load, invalid UTF-8 path: {:?}", cert_path);
continue;
}
};
let key_path = match key_path.to_str() {
Some(path) => path,
None => {
warn!("skip domain key load, invalid UTF-8 path: {:?}", key_path);
continue;
}
};
match load_cert_key_pair(cert_path, key_path) {
Ok((certs, key)) => {
cert_key_pairs.insert(domain_name.to_string(), (certs, key));
}
Err(e) => {
warn!("unable to load the certificate for {} domain name: {}", domain_name, e);
}
}
}
}
}
if cert_key_pairs.is_empty() {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("No valid certificate/private key pair found in directory {}", dir.display()),
));
}
Ok(cert_key_pairs)
}
fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)> {
let certs = load_certs(cert_path)?;
let key = load_private_key(key_path)?;
Ok((certs, key))
}
pub fn create_multi_cert_resolver(
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
) -> io::Result<impl ResolvesServerCert> {
#[derive(Debug)]
struct MultiCertResolver {
cert_resolver: ResolvesServerCertUsingSni,
default_cert: Option<Arc<CertifiedKey>>,
}
impl ResolvesServerCert for MultiCertResolver {
fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
if let Some(cert) = self.cert_resolver.resolve(client_hello) {
return Some(cert);
}
self.default_cert.clone()
}
}
let mut resolver = ResolvesServerCertUsingSni::new();
let mut default_cert = None;
for (domain, (certs, key)) in cert_key_pairs {
let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
.map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?;
let certified_key = CertifiedKey::new(certs, signing_key);
if domain == "default" {
default_cert = Some(Arc::new(certified_key.clone()));
} else {
resolver
.add(&domain, certified_key)
.map_err(|e| certs_error(format!("failed to add a domain name certificate:{domain},err: {e:?}")))?;
}
}
Ok(MultiCertResolver {
cert_resolver: resolver,
default_cert,
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use std::io::ErrorKind;
use std::path::PathBuf;
use tempfile::TempDir;
fn default_load_options(path: impl Into<PathBuf>) -> CertDirectoryLoadOptions {
CertDirectoryLoadOptions::builder(path, "rustfs_cert.pem", "rustfs_key.pem").build()
}
fn write_test_cert_pair(dir: &std::path::Path) {
let rcgen::CertifiedKey { cert, signing_key } =
rcgen::generate_simple_self_signed(vec!["example.com".to_string()]).expect("cert should generate");
fs::write(dir.join("rustfs_cert.pem"), cert.pem()).expect("cert should write");
fs::write(dir.join("rustfs_key.pem"), signing_key.serialize_pem()).expect("key should write");
}
#[test]
fn test_certs_error_function() {
let error_msg = "Test error message";
let error = certs_error(error_msg.to_string());
assert_eq!(error.kind(), ErrorKind::Other);
assert_eq!(error.to_string(), error_msg);
}
#[test]
fn test_load_certs_file_not_found() {
let result = load_certs("non_existent_file.pem");
assert!(result.is_err());
let error = result.expect_err("missing cert should error");
assert_eq!(error.kind(), ErrorKind::Other);
assert!(error.to_string().contains("failed to open"));
}
#[test]
fn test_load_private_key_file_not_found() {
let result = load_private_key("non_existent_key.pem");
assert!(result.is_err());
let error = result.expect_err("missing key should error");
assert_eq!(error.kind(), ErrorKind::Other);
assert!(error.to_string().contains("failed to open"));
}
#[test]
fn test_load_all_certs_from_directory_empty() {
let temp_dir = TempDir::new().expect("tempdir should create");
let result = load_all_certs_from_directory(default_load_options(temp_dir.path()));
assert!(result.is_err());
let error = result.expect_err("empty directory should error");
assert_eq!(error.kind(), ErrorKind::NotFound);
assert!(error.to_string().contains("No valid certificate/private key pair found"));
}
#[test]
fn test_load_all_certs_skips_kubernetes_secret_projection_dirs() {
let temp_dir = TempDir::new().expect("tempdir should create");
write_test_cert_pair(temp_dir.path());
let domain_dir = temp_dir.path().join("example.com");
fs::create_dir(&domain_dir).expect("domain dir should create");
write_test_cert_pair(&domain_dir);
for internal_dir_name in ["..data", "..2026_04_28_18_33_53.4209048473"] {
let internal_dir = temp_dir.path().join(internal_dir_name);
fs::create_dir(&internal_dir).expect("internal dir should create");
write_test_cert_pair(&internal_dir);
}
let certs = load_all_certs_from_directory(default_load_options(temp_dir.path())).expect("certs should load");
assert!(certs.contains_key("default"));
assert!(certs.contains_key("example.com"));
assert!(!certs.contains_key("..data"));
assert_eq!(certs.len(), 2);
}
}
crates/tls-runtime/src/config.rs
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::time::Duration;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReloadDetectMode {
Poll,
Watch, // TODO: implement fs::watch-based reload
Hybrid, // TODO: implement poll + fs::watch hybrid
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReloadApplyHint {
Lazy,
SoftReconnect,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TlsReloadOptions {
pub enabled: bool,
pub detect_mode: ReloadDetectMode,
pub interval: Duration,
pub debounce: Duration,
pub min_stable_age: Duration,
pub apply_hint: ReloadApplyHint,
}
impl Default for TlsReloadOptions {
fn default() -> Self {
Self {
enabled: true,
detect_mode: ReloadDetectMode::Poll,
interval: Duration::from_secs(15),
debounce: Duration::from_secs(2),
min_stable_age: Duration::from_secs(1),
apply_hint: ReloadApplyHint::Lazy,
}
}
}
crates/tls-runtime/src/coordinator.rs
+226
View File
@@ -0,0 +1,226 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::config::{ReloadDetectMode, TlsReloadOptions};
use crate::error::TlsRuntimeError;
use crate::material::TlsMaterialSnapshot;
use crate::metrics::{
TLS_RUNTIME_FOUNDATION_CONSUMER, record_tls_generation, record_tls_publication_fail, record_tls_reload_result,
record_tls_reload_skipped,
};
use crate::source::TlsSource;
use crate::state::{
TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeConsumerSection, TlsRuntimeOutboundSection,
TlsRuntimeRuntimeSection, TlsRuntimeServerSection, TlsRuntimeStatusSnapshot, detect_mode_label,
};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::task::JoinHandle;
use tracing::{debug, info, warn};
pub trait TlsConsumer<M>: Send + Sync + 'static {
fn on_publish(&self, generation: TlsGeneration, state: Arc<TlsPublishedState<M>>) -> Result<(), TlsRuntimeError>;
}
#[derive(Debug)]
pub struct TlsReloadCoordinator {
source: TlsSource,
options: TlsReloadOptions,
}
impl TlsReloadCoordinator {
pub fn new(source: TlsSource, options: TlsReloadOptions) -> Self {
Self { source, options }
}
pub fn source(&self) -> &TlsSource {
&self.source
}
pub fn options(&self) -> &TlsReloadOptions {
&self.options
}
pub async fn status_snapshot(&self, runtime_state: &TlsReloadRuntimeState<TlsMaterialSnapshot>) -> TlsRuntimeStatusSnapshot {
let current = runtime_state.current.load();
let last_attempt = runtime_state.last_attempt_unix_ms();
let last_success = runtime_state.last_success_unix_ms();
TlsRuntimeStatusSnapshot {
runtime: TlsRuntimeRuntimeSection {
generation: current.generation.0,
reload_enabled: self.options.enabled,
detect_mode: detect_mode_label(self.options.detect_mode),
last_attempt_time: (last_attempt != 0).then_some(last_attempt),
last_success_time: (last_success != 0).then_some(last_success),
last_error: runtime_state.last_error.read().await.clone(),
source_path: self.source.base_dir.display().to_string(),
},
outbound: TlsRuntimeOutboundSection {
has_roots: !current.material.outbound.root_ca_pem.is_empty(),
has_mtls_identity: current.material.outbound.mtls_identity.is_some(),
},
server: TlsRuntimeServerSection {
has_material: current.material.server.is_some(),
},
consumer: TlsRuntimeConsumerSection { stale_generation: false },
}
}
pub async fn load_initial_snapshot(&self) -> Result<TlsMaterialSnapshot, TlsRuntimeError> {
TlsMaterialSnapshot::load(&self.source).await
}
pub async fn publish_initial_state(&self, snapshot: TlsMaterialSnapshot) -> Arc<TlsPublishedState<TlsMaterialSnapshot>> {
let published = Arc::new(TlsPublishedState {
generation: TlsGeneration(1),
fingerprint: snapshot.fingerprint.clone(),
material: Arc::new(snapshot),
loaded_at_unix_ms: unix_time_ms(),
});
record_tls_generation(TLS_RUNTIME_FOUNDATION_CONSUMER, published.generation.0);
published
}
pub async fn reload_once<C>(
&self,
runtime_state: &TlsReloadRuntimeState<TlsMaterialSnapshot>,
consumer: &C,
) -> Result<Option<Arc<TlsPublishedState<TlsMaterialSnapshot>>>, TlsRuntimeError>
where
C: TlsConsumer<TlsMaterialSnapshot>,
{
runtime_state.mark_attempt(unix_time_ms());
let started_at = std::time::Instant::now();
let snapshot = self.load_initial_snapshot().await?;
let current = runtime_state.current.load();
if current.fingerprint == snapshot.fingerprint {
debug!(source = %self.source.base_dir.display(), "TLS material unchanged; skipping publication");
record_tls_reload_skipped(TLS_RUNTIME_FOUNDATION_CONSUMER, "unchanged");
return Ok(None);
}
let published = Arc::new(TlsPublishedState {
generation: runtime_state.bump_generation(),
fingerprint: snapshot.fingerprint.clone(),
material: Arc::new(snapshot),
loaded_at_unix_ms: unix_time_ms(),
});
if let Err(err) = consumer.on_publish(published.generation, published.clone()) {
record_tls_publication_fail(TLS_RUNTIME_FOUNDATION_CONSUMER);
return Err(err);
}
runtime_state.current.store(published.clone());
runtime_state.last_good.store(published.clone());
runtime_state.mark_success(unix_time_ms());
*runtime_state.last_error.write().await = None;
record_tls_reload_result(
TLS_RUNTIME_FOUNDATION_CONSUMER,
"ok",
Some(started_at.elapsed().as_secs_f64()),
Some(published.generation.0),
);
Ok(Some(published))
}
pub fn spawn_poll_loop<C>(
self: Arc<Self>,
runtime_state: Arc<TlsReloadRuntimeState<TlsMaterialSnapshot>>,
consumer: Arc<C>,
) -> Option<JoinHandle<()>>
where
C: TlsConsumer<TlsMaterialSnapshot>,
{
if !self.options.enabled {
debug!(source = %self.source.base_dir.display(), "TLS reload disabled; poll loop not started");
return None;
}
if !matches!(self.options.detect_mode, ReloadDetectMode::Poll | ReloadDetectMode::Hybrid) {
debug!(source = %self.source.base_dir.display(), "TLS poll loop skipped for non-poll detect mode");
return None;
}
let interval_duration = self.options.interval;
info!(
source = %self.source.base_dir.display(),
interval_secs = interval_duration.as_secs(),
"TLS poll reload loop enabled"
);
Some(tokio::spawn(async move {
let mut interval = tokio::time::interval(interval_duration);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await;
loop {
interval.tick().await;
if let Err(err) = self.reload_once(runtime_state.as_ref(), consumer.as_ref()).await {
warn!(
source = %self.source.base_dir.display(),
error = %err,
"TLS reload failed (will retry)"
);
*runtime_state.last_error.write().await = Some(err.to_string());
}
}
}))
}
}
fn unix_time_ms() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64
}
#[cfg(test)]
mod tests {
use super::*;
use crate::source::TlsSource;
use std::sync::Arc;
struct NopConsumer;
impl TlsConsumer<crate::material::TlsMaterialSnapshot> for NopConsumer {
fn on_publish(
&self,
_generation: TlsGeneration,
_state: Arc<TlsPublishedState<crate::material::TlsMaterialSnapshot>>,
) -> Result<(), TlsRuntimeError> {
Ok(())
}
}
#[tokio::test]
async fn reload_once_skips_when_fingerprint_unchanged() {
let temp = tempfile::tempdir().expect("tempdir");
let source = TlsSource::from_directory(temp.path().to_path_buf());
let options = TlsReloadOptions::default();
let coordinator = TlsReloadCoordinator::new(source.clone(), options);
// Load the actual snapshot from the (empty) temp dir so its fingerprint
// matches what reload_once will observe on the next load.
let initial_snapshot = coordinator.load_initial_snapshot().await.expect("initial load");
let initial = coordinator.publish_initial_state(initial_snapshot).await;
let runtime_state = TlsReloadRuntimeState::new(initial);
let consumer = NopConsumer;
let result = coordinator.reload_once(&runtime_state, &consumer).await;
// Fingerprint has not changed → should skip and return Ok(None).
assert!(result.is_ok(), "reload_once should succeed: {:?}", result.err());
assert!(result.unwrap().is_none(), "should skip when fingerprint unchanged");
}
}
crates/tls-runtime/src/debug.rs
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::state::TlsRuntimeStatusSnapshot;
use serde::Serialize;
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsConsumerStatusItem {
pub consumer: &'static str,
pub generation: u64,
pub has_root_ca: bool,
pub has_mtls_identity: bool,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsDebugStatusResponse {
pub foundation: TlsRuntimeStatusSnapshot,
pub consumers: Vec<TlsConsumerStatusItem>,
}
#[derive(Debug, Clone)]
pub struct TlsDebugStatusResponseBuilder {
foundation: TlsRuntimeStatusSnapshot,
consumers: Vec<TlsConsumerStatusItem>,
}
impl TlsDebugStatusResponse {
pub fn builder(foundation: TlsRuntimeStatusSnapshot) -> TlsDebugStatusResponseBuilder {
TlsDebugStatusResponseBuilder {
foundation,
consumers: Vec::new(),
}
}
}
impl TlsDebugStatusResponseBuilder {
pub fn push_consumers<I>(mut self, sources: I) -> Self
where
I: IntoIterator<Item = TlsConsumerStatusItem>,
{
self.consumers.extend(sources);
self
}
pub fn build(self) -> TlsDebugStatusResponse {
TlsDebugStatusResponse {
foundation: self.foundation,
consumers: self.consumers,
}
}
}
#[cfg(test)]
mod tests {
use super::{TlsConsumerStatusItem, TlsDebugStatusResponse};
use crate::state::{
TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection,
TlsRuntimeStatusSnapshot,
};
#[test]
fn builder_produces_structured_response() {
let foundation = TlsRuntimeStatusSnapshot {
runtime: TlsRuntimeRuntimeSection {
generation: 5,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: Some(1),
last_success_time: Some(2),
last_error: None,
source_path: "/tmp/tls".to_string(),
},
outbound: TlsRuntimeOutboundSection {
has_roots: true,
has_mtls_identity: false,
},
server: TlsRuntimeServerSection { has_material: true },
consumer: TlsRuntimeConsumerSection { stale_generation: false },
};
let response = TlsDebugStatusResponse::builder(foundation)
.push_consumers([TlsConsumerStatusItem {
consumer: "test_consumer",
generation: 7,
has_root_ca: true,
has_mtls_identity: false,
}])
.build();
let json = serde_json::to_value(response).expect("response should serialize");
assert!(json.get("foundation").is_some());
assert!(json.get("consumers").is_some());
assert!(json["consumers"].is_array());
}
}
crates/tls-runtime/src/error.rs
+32
View File
@@ -0,0 +1,32 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::path::PathBuf;
use thiserror::Error;
#[derive(Debug, Error)]
pub enum TlsRuntimeError {
#[error("TLS source path is empty")]
EmptySourcePath,
#[error("TLS directory does not exist: {path}")]
DirectoryNotFound { path: PathBuf },
#[error("TLS path is not a directory: {path}")]
NotADirectory { path: PathBuf },
#[error("TLS material error: {0}")]
Material(String),
#[error("TLS publication error: {0}")]
Publication(String),
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
}
crates/tls-runtime/src/fingerprint.rs
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use sha2::{Digest, Sha256};
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct TlsFingerprint {
pub server_sha256: Option<[u8; 32]>,
pub public_ca_sha256: Option<[u8; 32]>,
pub client_ca_sha256: Option<[u8; 32]>,
pub client_cert_sha256: Option<[u8; 32]>,
pub client_key_sha256: Option<[u8; 32]>,
}
fn digest_bytes(bytes: &[u8]) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(bytes);
hasher.finalize().into()
}
impl TlsFingerprint {
pub fn from_optional_bytes(
server: Option<&[u8]>,
public_ca: Option<&[u8]>,
client_ca: Option<&[u8]>,
client_cert: Option<&[u8]>,
client_key: Option<&[u8]>,
) -> Self {
Self {
server_sha256: server.map(digest_bytes),
public_ca_sha256: public_ca.map(digest_bytes),
client_ca_sha256: client_ca.map(digest_bytes),
client_cert_sha256: client_cert.map(digest_bytes),
client_key_sha256: client_key.map(digest_bytes),
}
}
}
crates/tls-runtime/src/lib.rs
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod certs;
pub mod config;
pub mod coordinator;
pub mod debug;
pub mod error;
pub mod fingerprint;
pub mod material;
pub mod metrics;
pub mod outbound;
pub mod server;
pub mod source;
pub mod state;
pub use certs::{
CertDirectoryLoadOptions, WebPkiClientVerifierOptions, build_webpki_client_verifier, create_multi_cert_resolver,
load_all_certs_from_directory, load_cert_bundle_der_bytes, load_certs, load_private_key,
};
pub use config::{ReloadApplyHint, ReloadDetectMode, TlsReloadOptions};
pub use coordinator::{TlsConsumer, TlsReloadCoordinator};
pub use debug::{TlsConsumerStatusItem, TlsDebugStatusResponse, TlsDebugStatusResponseBuilder};
pub use error::TlsRuntimeError;
pub use fingerprint::TlsFingerprint;
pub use material::{OutboundTlsMaterial, ServerTlsMaterial, TlsMaterialSnapshot};
pub use metrics::{
TLS_OUTBOUND_GLOBAL_CONSUMER, TLS_RUNTIME_FOUNDATION_CONSUMER, init_tls_metrics, record_tls_consumer_stale_generation,
record_tls_generation, record_tls_publication_fail, record_tls_reload_result, record_tls_reload_skipped,
};
pub use outbound::{
GlobalOutboundTlsStateSummary, GlobalPublishedOutboundTlsState, load_global_outbound_tls_generation,
load_global_outbound_tls_state, publish_global_outbound_tls_state, summarize_global_outbound_tls_state,
};
pub use server::{ReloadableServerCertResolver, spawn_server_cert_reload_loop};
pub use source::{TlsFileLayout, TlsSource, TlsSourceKind};
pub use state::OutboundOnlySnapshotArgs;
pub use state::{TlsGeneration, TlsPublishedState, TlsReloadRuntimeState, TlsRuntimeStatusSnapshot};
pub use state::{TlsRuntimeConsumerSection, TlsRuntimeOutboundSection, TlsRuntimeRuntimeSection, TlsRuntimeServerSection};
#[cfg(test)]
mod tests {
use super::*;
use rcgen::generate_simple_self_signed;
use std::collections::HashMap;
#[test]
fn tls_source_requires_existing_directory() {
let source = TlsSource::from_directory("/definitely/missing/rustfs/tls-runtime-test");
let err = source.validate_directory().expect_err("missing directory should fail");
assert!(matches!(err, TlsRuntimeError::DirectoryNotFound { .. }));
}
#[test]
fn fingerprint_changes_when_server_material_changes() {
let cert_a = generate_simple_self_signed(vec!["a.example.com".to_string()]).expect("cert A should generate");
let cert_b = generate_simple_self_signed(vec!["b.example.com".to_string()]).expect("cert B should generate");
let single_a = ServerTlsMaterial::SingleCert {
certs: vec![cert_a.cert.der().clone()],
key: rustls::pki_types::PrivateKeyDer::try_from(cert_a.signing_key.serialize_der()).expect("key A should convert"),
};
let single_b = ServerTlsMaterial::SingleCert {
certs: vec![cert_b.cert.der().clone()],
key: rustls::pki_types::PrivateKeyDer::try_from(cert_b.signing_key.serialize_der()).expect("key B should convert"),
};
let bytes_a = match &single_a {
ServerTlsMaterial::SingleCert { certs, key } => {
let mut bytes = Vec::new();
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
bytes
}
ServerTlsMaterial::MultiCert { .. } => unreachable!(),
};
let bytes_b = match &single_b {
ServerTlsMaterial::SingleCert { certs, key } => {
let mut bytes = Vec::new();
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
bytes
}
ServerTlsMaterial::MultiCert { .. } => unreachable!(),
};
let fp_a = TlsFingerprint::from_optional_bytes(Some(&bytes_a), None, None, None, None);
let fp_b = TlsFingerprint::from_optional_bytes(Some(&bytes_b), None, None, None, None);
assert_ne!(fp_a, fp_b);
}
#[tokio::test]
async fn coordinator_can_publish_initial_state() {
let source = TlsSource::from_directory(std::env::temp_dir());
let coordinator = TlsReloadCoordinator::new(source.clone(), TlsReloadOptions::default());
let snapshot = TlsMaterialSnapshot {
source,
server: Some(ServerTlsMaterial::MultiCert {
cert_key_pairs: HashMap::new(),
}),
outbound: OutboundTlsMaterial {
root_ca_pem: Vec::new(),
mtls_identity: None,
},
fingerprint: TlsFingerprint::default(),
};
let published = coordinator.publish_initial_state(snapshot).await;
assert_eq!(published.generation, TlsGeneration(1));
}
}
crates/tls-runtime/src/material.rs
+200
View File
@@ -0,0 +1,200 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory, load_certs, load_private_key};
use crate::error::TlsRuntimeError;
use crate::fingerprint::TlsFingerprint;
use crate::source::TlsSource;
use rustfs_common::MtlsIdentityPem;
use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
use std::collections::HashMap;
use std::io::Cursor;
use std::io::ErrorKind;
use std::path::Path;
#[derive(Debug, Clone)]
pub struct OutboundTlsMaterial {
pub root_ca_pem: Vec<u8>,
pub mtls_identity: Option<MtlsIdentityPem>,
}
#[derive(Debug)]
pub enum ServerTlsMaterial {
SingleCert {
certs: Vec<CertificateDer<'static>>,
key: PrivateKeyDer<'static>,
},
MultiCert {
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
},
}
#[derive(Debug)]
pub struct TlsMaterialSnapshot {
pub source: TlsSource,
pub server: Option<ServerTlsMaterial>,
pub outbound: OutboundTlsMaterial,
pub fingerprint: TlsFingerprint,
}
impl TlsMaterialSnapshot {
pub async fn load(source: &TlsSource) -> Result<Self, TlsRuntimeError> {
let base_dir = source.validate_directory()?.to_path_buf();
let server = load_server_material(source, &base_dir)?;
let public_ca_path = base_dir.join(&source.layout.public_ca_filename);
let client_ca_path = base_dir.join(&source.fallback_ca_filename);
let client_cert_path = base_dir.join(&source.layout.client_cert_filename);
let client_key_path = base_dir.join(&source.layout.client_key_filename);
let public_ca_pem = tokio::fs::read(&public_ca_path).await.ok();
let client_ca_pem = tokio::fs::read(&client_ca_path).await.ok();
let root_ca_pem = combine_optional_pem(public_ca_pem.as_deref(), client_ca_pem.as_deref());
let mtls_identity = match (
tokio::fs::read(&client_cert_path).await.ok(),
tokio::fs::read(&client_key_path).await.ok(),
) {
(Some(cert_pem), Some(key_pem)) => {
let mut cert_reader = Cursor::new(&cert_pem);
if CertificateDer::pem_reader_iter(&mut cert_reader).next().is_none() {
return Err(TlsRuntimeError::Material("no valid certificate in client cert PEM".to_string()));
}
let mut key_reader = Cursor::new(&key_pem);
PrivateKeyDer::from_pem_reader(&mut key_reader)
.map_err(|e| TlsRuntimeError::Material(format!("invalid client key PEM: {e}")))?;
Some(MtlsIdentityPem { cert_pem, key_pem })
}
_ => None,
};
let outbound = OutboundTlsMaterial {
root_ca_pem: root_ca_pem.clone(),
mtls_identity: mtls_identity.clone(),
};
let server_fingerprint_bytes = server.as_ref().map(serialize_server_material_for_fingerprint);
let fingerprint = TlsFingerprint::from_optional_bytes(
server_fingerprint_bytes.as_deref(),
public_ca_pem.as_deref(),
client_ca_pem.as_deref(),
mtls_identity.as_ref().map(|identity| identity.cert_pem.as_slice()),
mtls_identity.as_ref().map(|identity| identity.key_pem.as_slice()),
);
Ok(Self {
source: source.clone(),
server,
outbound,
fingerprint,
})
}
}
fn load_server_material(source: &TlsSource, base_dir: &Path) -> Result<Option<ServerTlsMaterial>, TlsRuntimeError> {
let root_cert = base_dir.join(&source.layout.server_cert_filename);
let root_key = base_dir.join(&source.layout.server_key_filename);
let has_root_pair = root_cert.exists() && root_key.exists();
let cert_key_pairs = load_all_certs_from_directory(
CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename)
.build(),
);
match cert_key_pairs {
Ok(cert_key_pairs) if cert_key_pairs.len() > 1 || cert_key_pairs.keys().any(|key| key != "default") => {
Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs }))
}
Ok(cert_key_pairs) if !cert_key_pairs.is_empty() => {
if let Some((certs, key)) = cert_key_pairs.get("default") {
return Ok(Some(ServerTlsMaterial::SingleCert {
certs: certs.clone(),
key: key.clone_key(),
}));
}
Ok(Some(ServerTlsMaterial::MultiCert { cert_key_pairs }))
}
Ok(_) => Ok(None),
Err(_err) if has_root_pair => {
let root_cert_path = path_to_utf8_str(&root_cert, "root TLS certificate")?;
let root_key_path = path_to_utf8_str(&root_key, "root TLS private key")?;
let certs = load_certs(root_cert_path)
.map_err(|e| TlsRuntimeError::Material(format!("load root TLS certificate {}: {e}", root_cert.display())))?;
let key = load_private_key(root_key_path)
.map_err(|e| TlsRuntimeError::Material(format!("load root TLS private key {}: {e}", root_key.display())))?;
Ok(Some(ServerTlsMaterial::SingleCert { certs, key }))
}
Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
Err(err) => Err(TlsRuntimeError::Material(format!(
"discover server TLS certificates under '{}': {err}",
base_dir.display()
))),
}
}
fn path_to_utf8_str<'a>(path: &'a Path, description: &str) -> Result<&'a str, TlsRuntimeError> {
path.to_str()
.ok_or_else(|| TlsRuntimeError::Material(format!("{description} path '{}' is not valid UTF-8", path.display())))
}
fn combine_optional_pem(primary: Option<&[u8]>, fallback: Option<&[u8]>) -> Vec<u8> {
let mut combined = Vec::new();
for pem in [primary, fallback].into_iter().flatten() {
if pem.iter().all(|&b| b.is_ascii_whitespace()) {
continue;
}
combined.extend_from_slice(pem);
if !combined.ends_with(b"\n") {
combined.push(b'\n');
}
}
combined
}
fn serialize_server_material_for_fingerprint(material: &ServerTlsMaterial) -> Vec<u8> {
match material {
ServerTlsMaterial::SingleCert { certs, key } => {
let mut bytes = Vec::new();
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
bytes
}
ServerTlsMaterial::MultiCert { cert_key_pairs } => {
let mut entries = cert_key_pairs.iter().collect::<Vec<_>>();
entries.sort_by_key(|(left, _)| *left);
let mut bytes = Vec::new();
for (domain, (certs, key)) in entries {
bytes.extend_from_slice(domain.as_bytes());
for cert in certs {
bytes.extend_from_slice(cert.as_ref());
}
bytes.extend_from_slice(key.secret_der());
}
bytes
}
}
}
pub(crate) fn server_material_fingerprint(material: &ServerTlsMaterial) -> TlsFingerprint {
let bytes = serialize_server_material_for_fingerprint(material);
TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None)
}
crates/tls-runtime/src/metrics.rs
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
pub const TLS_RUNTIME_FOUNDATION_CONSUMER: &str = "tls_runtime_foundation";
pub const TLS_OUTBOUND_GLOBAL_CONSUMER: &str = "outbound_global";
const CONSUMER_LABEL: &str = "consumer";
const RESULT_LABEL: &str = "result";
const REASON_LABEL: &str = "reason";
const TLS_OUTBOUND_PUBLICATIONS_TOTAL: &str = "rustfs_tls_outbound_publications_total";
const TLS_OUTBOUND_GENERATION: &str = "rustfs_tls_outbound_generation";
const TLS_OUTBOUND_HAS_ROOT_CA: &str = "rustfs_tls_outbound_has_root_ca";
const TLS_OUTBOUND_HAS_MTLS_IDENTITY: &str = "rustfs_tls_outbound_has_mtls_identity";
const TLS_GENERATION: &str = "rustfs_tls_generation";
const TLS_RELOAD_TOTAL: &str = "rustfs_tls_reload_total";
const TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_tls_reload_duration_seconds";
const TLS_RELOAD_GENERATION: &str = "rustfs_tls_reload_generation";
const TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_tls_reload_skipped_total";
const TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_tls_publication_fail_total";
const TLS_CONSUMER_STALE_GENERATION_TOTAL: &str = "rustfs_tls_consumer_stale_generation_total";
pub fn record_outbound_tls_publication(generation: u64, has_root_ca: bool, has_mtls_identity: bool) {
counter!(TLS_OUTBOUND_PUBLICATIONS_TOTAL, "result" => "ok").increment(1);
gauge!(TLS_OUTBOUND_GENERATION).set(generation as f64);
gauge!(TLS_OUTBOUND_HAS_ROOT_CA).set(if has_root_ca { 1.0 } else { 0.0 });
gauge!(TLS_OUTBOUND_HAS_MTLS_IDENTITY).set(if has_mtls_identity { 1.0 } else { 0.0 });
record_tls_generation(TLS_OUTBOUND_GLOBAL_CONSUMER, generation);
}
pub fn record_tls_generation(consumer: &'static str, generation: u64) {
gauge!(TLS_GENERATION, CONSUMER_LABEL => consumer).set(generation as f64);
}
pub fn record_tls_reload_result(
consumer: &'static str,
result: &'static str,
duration_secs: Option<f64>,
generation: Option<u64>,
) {
counter!(TLS_RELOAD_TOTAL, CONSUMER_LABEL => consumer, RESULT_LABEL => result).increment(1);
if let Some(duration_secs) = duration_secs {
histogram!(TLS_RELOAD_DURATION_SECONDS, CONSUMER_LABEL => consumer).record(duration_secs);
}
if let Some(generation) = generation {
gauge!(TLS_RELOAD_GENERATION, CONSUMER_LABEL => consumer).set(generation as f64);
record_tls_generation(consumer, generation);
}
}
pub fn record_tls_reload_skipped(consumer: &'static str, reason: &'static str) {
counter!(TLS_RELOAD_SKIPPED_TOTAL, CONSUMER_LABEL => consumer, REASON_LABEL => reason).increment(1);
}
pub fn record_tls_publication_fail(consumer: &'static str) {
counter!(TLS_PUBLICATION_FAIL_TOTAL, CONSUMER_LABEL => consumer).increment(1);
}
pub fn record_tls_consumer_stale_generation(consumer: &'static str) {
counter!(TLS_CONSUMER_STALE_GENERATION_TOTAL, CONSUMER_LABEL => consumer).increment(1);
}
pub fn init_tls_metrics() {
describe_counter!(TLS_OUTBOUND_PUBLICATIONS_TOTAL, "Total TLS outbound publications, labeled by result.");
describe_gauge!(TLS_OUTBOUND_GENERATION, "Current outbound TLS generation.");
describe_gauge!(TLS_OUTBOUND_HAS_ROOT_CA, "Whether outbound TLS roots are configured.");
describe_gauge!(TLS_OUTBOUND_HAS_MTLS_IDENTITY, "Whether outbound mTLS identity is configured.");
describe_gauge!(TLS_GENERATION, "Current TLS generation by consumer.");
describe_counter!(TLS_RELOAD_TOTAL, "Total TLS reload attempts, labeled by consumer and result.");
describe_histogram!(TLS_RELOAD_DURATION_SECONDS, "TLS reload duration by consumer (seconds).");
describe_gauge!(TLS_RELOAD_GENERATION, "TLS generation after reload by consumer.");
describe_counter!(TLS_RELOAD_SKIPPED_TOTAL, "Total skipped TLS reloads, labeled by consumer and reason.");
describe_counter!(TLS_PUBLICATION_FAIL_TOTAL, "Total TLS publication failures by consumer.");
describe_counter!(TLS_CONSUMER_STALE_GENERATION_TOTAL, "Total stale TLS consumer generations observed.");
}
crates/tls-runtime/src/outbound.rs
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::material::OutboundTlsMaterial;
use crate::metrics::record_outbound_tls_publication;
use crate::state::TlsGeneration;
use rustfs_common::{
GLOBAL_MTLS_IDENTITY, GLOBAL_ROOT_CERT, MtlsIdentityPem, get_global_outbound_tls_generation, set_global_mtls_identity,
set_global_outbound_tls_generation, set_global_root_cert,
};
#[derive(Debug, Clone)]
pub struct GlobalPublishedOutboundTlsState {
pub generation: TlsGeneration,
pub root_ca_pem: Option<Vec<u8>>,
pub mtls_identity: Option<MtlsIdentityPem>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GlobalOutboundTlsStateSummary {
pub generation: TlsGeneration,
pub has_root_ca: bool,
pub has_mtls_identity: bool,
}
pub async fn publish_global_outbound_tls_state(generation: TlsGeneration, material: &OutboundTlsMaterial) {
if !material.root_ca_pem.is_empty() {
set_global_root_cert(material.root_ca_pem.clone()).await;
} else {
*GLOBAL_ROOT_CERT.write().await = None;
}
set_global_mtls_identity(material.mtls_identity.clone()).await;
set_global_outbound_tls_generation(generation.0);
record_outbound_tls_publication(generation.0, !material.root_ca_pem.is_empty(), material.mtls_identity.is_some());
}
pub async fn load_global_outbound_tls_state() -> GlobalPublishedOutboundTlsState {
GlobalPublishedOutboundTlsState {
generation: TlsGeneration(get_global_outbound_tls_generation()),
root_ca_pem: GLOBAL_ROOT_CERT.read().await.clone(),
mtls_identity: GLOBAL_MTLS_IDENTITY.read().await.clone(),
}
}
pub fn load_global_outbound_tls_generation() -> TlsGeneration {
TlsGeneration(get_global_outbound_tls_generation())
}
pub async fn summarize_global_outbound_tls_state() -> GlobalOutboundTlsStateSummary {
let state = load_global_outbound_tls_state().await;
GlobalOutboundTlsStateSummary {
generation: state.generation,
has_root_ca: state.root_ca_pem.as_ref().is_some_and(|pem| !pem.is_empty()),
has_mtls_identity: state.mtls_identity.is_some(),
}
}
crates/tls-runtime/src/server.rs
+326
View File
@@ -0,0 +1,326 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::certs::{CertDirectoryLoadOptions, load_all_certs_from_directory};
use crate::config::TlsReloadOptions;
use crate::error::TlsRuntimeError;
use crate::material::{ServerTlsMaterial, server_material_fingerprint};
use crate::metrics::{record_tls_generation, record_tls_publication_fail, record_tls_reload_result, record_tls_reload_skipped};
use crate::source::TlsSource;
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
use rustls::sign::CertifiedKey;
use std::collections::HashMap;
use std::io;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, RwLock};
use tokio::sync::watch;
use tokio::task::JoinHandle;
use tokio::time::MissedTickBehavior;
use tracing::{debug, info, warn};
#[derive(Debug)]
struct ResolverState {
cert_resolver: ResolvesServerCertUsingSni,
default_cert: Option<Arc<CertifiedKey>>,
cert_count: usize,
fingerprint: crate::fingerprint::TlsFingerprint,
}
impl ResolverState {
fn load_from_source(source: &TlsSource) -> Result<Self, TlsRuntimeError> {
let base_dir = source.validate_directory()?;
let cert_key_pairs = load_all_certs_from_directory(
CertDirectoryLoadOptions::builder(base_dir, &source.layout.server_cert_filename, &source.layout.server_key_filename)
.build(),
)?;
if cert_key_pairs.is_empty() {
return Err(TlsRuntimeError::Material("No valid certificates found in directory".to_string()));
}
Self::from_cert_key_pairs(cert_key_pairs)
}
fn from_cert_key_pairs(
cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
) -> Result<Self, TlsRuntimeError> {
let cert_count = cert_key_pairs.len();
let mut cert_resolver = ResolvesServerCertUsingSni::new();
let mut default_cert = None;
let mut entries = cert_key_pairs.into_iter().collect::<Vec<_>>();
entries.sort_by(|(left_domain, _), (right_domain, _)| left_domain.cmp(right_domain));
let material = ServerTlsMaterial::MultiCert {
cert_key_pairs: entries
.iter()
.map(|(domain, (certs, key))| (domain.clone(), (certs.clone(), key.clone_key())))
.collect(),
};
let fingerprint = server_material_fingerprint(&material);
for (domain, (certs, key)) in entries {
let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
.map_err(|e| io::Error::other(format!("unsupported private key type for {domain}: {e:?}")))?;
let certified_key = CertifiedKey::new(certs, signing_key);
if domain.as_str() == "default" {
default_cert = Some(Arc::new(certified_key.clone()));
} else {
cert_resolver
.add(&domain, certified_key)
.map_err(|e| io::Error::other(format!("failed to add certificate for {domain}: {e:?}")))?;
}
}
Ok(Self {
cert_resolver,
default_cert,
cert_count,
fingerprint,
})
}
}
#[derive(Debug)]
pub struct ReloadableServerCertResolver {
source: TlsSource,
current: RwLock<ResolverState>,
generation: AtomicU64,
}
impl ReloadableServerCertResolver {
pub fn load_from_source(source: TlsSource) -> Result<Arc<Self>, TlsRuntimeError> {
let state = ResolverState::load_from_source(&source)?;
record_tls_generation("server_resolver", 1);
Ok(Arc::new(Self {
source,
current: RwLock::new(state),
generation: AtomicU64::new(1),
}))
}
pub fn load_from_directory(cert_dir: &str) -> Result<Arc<Self>, TlsRuntimeError> {
Self::load_from_source(TlsSource::from_directory(cert_dir))
}
pub fn reload(&self) -> Result<Option<usize>, TlsRuntimeError> {
let new_state = ResolverState::load_from_source(&self.source)?;
match self.current.write() {
Ok(mut guard) => {
if guard.fingerprint == new_state.fingerprint {
return Ok(None);
}
let cert_count = new_state.cert_count;
*guard = new_state;
self.generation.fetch_add(1, Ordering::Relaxed);
Ok(Some(cert_count))
}
Err(poisoned) => {
let mut guard = poisoned.into_inner();
if guard.fingerprint == new_state.fingerprint {
return Ok(None);
}
let cert_count = new_state.cert_count;
*guard = new_state;
self.generation.fetch_add(1, Ordering::Relaxed);
Ok(Some(cert_count))
}
}
}
pub fn generation(&self) -> u64 {
self.generation.load(Ordering::Relaxed)
}
}
impl ResolvesServerCert for ReloadableServerCertResolver {
fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> {
let guard = match self.current.read() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
guard
.cert_resolver
.resolve(client_hello)
.or_else(|| guard.default_cert.clone())
}
}
pub fn spawn_server_cert_reload_loop(
protocol: &'static str,
resolver: Arc<ReloadableServerCertResolver>,
options: TlsReloadOptions,
mut shutdown_rx: watch::Receiver<bool>,
) -> Option<JoinHandle<()>> {
if !options.enabled {
debug!(protocol, "TLS certificate hot reload is disabled");
return None;
}
info!(
protocol,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate hot reload enabled, checking every {}s",
options.interval.as_secs()
);
Some(tokio::spawn(async move {
let mut interval = tokio::time::interval(options.interval);
interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
interval.tick().await;
loop {
tokio::select! {
changed = shutdown_rx.changed() => {
match changed {
Ok(()) => {
if *shutdown_rx.borrow() {
info!(protocol, cert_dir = %resolver.source.base_dir.display(), "TLS certificate hot reload task stopped");
break;
}
continue;
}
Err(_) => {
info!(
protocol,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate hot reload task stopped because the shutdown channel closed"
);
break;
}
}
}
_ = interval.tick() => {}
}
match resolver.reload() {
Ok(Some(cert_count)) => {
record_tls_reload_result(protocol, "ok", None, Some(resolver.generation()));
info!(
protocol,
cert_dir = %resolver.source.base_dir.display(),
cert_count,
"TLS certificates reloaded successfully"
);
}
Ok(None) => {
record_tls_reload_skipped(protocol, "unchanged");
debug!(
protocol,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate material unchanged; skipping reload"
);
}
Err(e) => {
record_tls_publication_fail(protocol);
warn!(
protocol,
cert_dir = %resolver.source.base_dir.display(),
"TLS certificate reload failed (will retry): {}",
e
);
}
}
}
}))
}
#[cfg(test)]
mod tests {
use super::*;
use rcgen::generate_simple_self_signed;
use std::fs;
use tempfile::TempDir;
fn cert_key_pair(san: &str) -> (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>) {
let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate");
(
vec![cert.cert.der().clone()],
PrivateKeyDer::try_from(cert.signing_key.serialize_der()).expect("key should convert"),
)
}
fn clone_cert_key_pair(
cert_key_pair: &(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>),
) -> (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>) {
(cert_key_pair.0.clone(), cert_key_pair.1.clone_key())
}
fn write_default_cert(dir: &std::path::Path, san: &str) {
let cert = generate_simple_self_signed(vec![san.to_string()]).expect("cert should generate");
fs::write(dir.join(rustfs_config::RUSTFS_TLS_CERT), cert.cert.pem()).expect("cert should write");
fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), cert.signing_key.serialize_pem()).expect("key should write");
}
#[test]
fn reload_replaces_default_certificate() {
let temp_dir = TempDir::new().expect("tempdir should create");
write_default_cert(temp_dir.path(), "localhost");
let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8"))
.expect("resolver should load");
let before = {
let guard = resolver.current.read().expect("lock should acquire");
guard.default_cert.as_ref().expect("default cert should exist").clone()
};
write_default_cert(temp_dir.path(), "rotated.local");
let cert_count = resolver.reload().expect("reload should succeed");
assert_eq!(cert_count, Some(1));
let after = {
let guard = resolver.current.read().expect("lock should acquire");
guard.default_cert.as_ref().expect("default cert should exist").clone()
};
assert_ne!(before.cert[0].as_ref(), after.cert[0].as_ref());
}
#[test]
fn reload_skips_when_material_is_unchanged() {
let temp_dir = TempDir::new().expect("tempdir should create");
write_default_cert(temp_dir.path(), "localhost");
let resolver = ReloadableServerCertResolver::load_from_directory(temp_dir.path().to_str().expect("path should utf8"))
.expect("resolver should load");
let outcome = resolver.reload().expect("reload should succeed");
assert_eq!(outcome, None);
}
#[test]
fn resolver_state_fingerprint_is_stable_across_domain_ordering() {
let default_cert = cert_key_pair("localhost");
let api_cert = cert_key_pair("api.example.com");
let web_cert = cert_key_pair("web.example.com");
let mut first = HashMap::new();
first.insert("default".to_string(), clone_cert_key_pair(&default_cert));
first.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert));
first.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert));
let mut second = HashMap::new();
second.insert("web.example.com".to_string(), clone_cert_key_pair(&web_cert));
second.insert("default".to_string(), clone_cert_key_pair(&default_cert));
second.insert("api.example.com".to_string(), clone_cert_key_pair(&api_cert));
let first_state = ResolverState::from_cert_key_pairs(first).expect("first state should build");
let second_state = ResolverState::from_cert_key_pairs(second).expect("second state should build");
assert_eq!(first_state.cert_count, 3);
assert_eq!(second_state.cert_count, 3);
assert_eq!(first_state.fingerprint, second_state.fingerprint);
}
}
crates/tls-runtime/src/source.rs
+94
View File
@@ -0,0 +1,94 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::error::TlsRuntimeError;
use rustfs_config::{
RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_PUBLIC_CERT,
RUSTFS_TLS_CERT, RUSTFS_TLS_KEY,
};
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TlsSourceKind {
Directory,
ExplicitFiles,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TlsFileLayout {
pub server_cert_filename: String,
pub server_key_filename: String,
pub public_ca_filename: String,
pub client_ca_filename: String,
pub client_cert_filename: String,
pub client_key_filename: String,
}
impl Default for TlsFileLayout {
fn default() -> Self {
Self {
server_cert_filename: RUSTFS_TLS_CERT.to_string(),
server_key_filename: RUSTFS_TLS_KEY.to_string(),
public_ca_filename: RUSTFS_PUBLIC_CERT.to_string(),
client_ca_filename: RUSTFS_CLIENT_CA_CERT_FILENAME.to_string(),
client_cert_filename: RUSTFS_CLIENT_CERT_FILENAME.to_string(),
client_key_filename: RUSTFS_CLIENT_KEY_FILENAME.to_string(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TlsSource {
pub kind: TlsSourceKind,
pub base_dir: PathBuf,
pub layout: TlsFileLayout,
pub fallback_ca_filename: String,
pub trust_system_ca: bool,
pub trust_leaf_as_ca: bool,
pub server_mtls_enabled: bool,
}
impl TlsSource {
pub fn from_directory(base_dir: impl Into<PathBuf>) -> Self {
Self {
kind: TlsSourceKind::Directory,
base_dir: base_dir.into(),
layout: TlsFileLayout::default(),
fallback_ca_filename: RUSTFS_CA_CERT.to_string(),
trust_system_ca: false,
trust_leaf_as_ca: false,
server_mtls_enabled: false,
}
}
pub fn validate_directory(&self) -> Result<&Path, TlsRuntimeError> {
if self.base_dir.as_os_str().is_empty() {
return Err(TlsRuntimeError::EmptySourcePath);
}
if !self.base_dir.exists() {
return Err(TlsRuntimeError::DirectoryNotFound {
path: self.base_dir.clone(),
});
}
if !self.base_dir.is_dir() {
return Err(TlsRuntimeError::NotADirectory {
path: self.base_dir.clone(),
});
}
Ok(self.base_dir.as_path())
}
}
crates/tls-runtime/src/state.rs
+234
View File
@@ -0,0 +1,234 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::fingerprint::TlsFingerprint;
use arc_swap::ArcSwap;
use serde::Serialize;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::sync::RwLock;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TlsGeneration(pub u64);
#[derive(Debug)]
pub struct TlsPublishedState<M> {
pub generation: TlsGeneration,
pub material: Arc<M>,
pub fingerprint: TlsFingerprint,
pub loaded_at_unix_ms: u64,
}
#[derive(Debug)]
pub struct TlsReloadRuntimeState<M> {
pub current: ArcSwap<TlsPublishedState<M>>,
pub last_good: ArcSwap<TlsPublishedState<M>>,
pub last_attempt_unix_ms: AtomicU64,
pub last_success_unix_ms: AtomicU64,
pub last_error: RwLock<Option<String>>,
}
impl<M> TlsReloadRuntimeState<M> {
pub fn new(initial: Arc<TlsPublishedState<M>>) -> Self {
Self {
current: ArcSwap::from(initial.clone()),
last_good: ArcSwap::from(initial),
last_attempt_unix_ms: AtomicU64::new(0),
last_success_unix_ms: AtomicU64::new(0),
last_error: RwLock::new(None),
}
}
pub fn current_generation(&self) -> TlsGeneration {
self.current.load().generation
}
pub fn bump_generation(&self) -> TlsGeneration {
TlsGeneration(self.current_generation().0.saturating_add(1))
}
pub fn mark_attempt(&self, unix_ms: u64) {
self.last_attempt_unix_ms.store(unix_ms, Ordering::Relaxed);
}
pub fn mark_success(&self, unix_ms: u64) {
self.last_success_unix_ms.store(unix_ms, Ordering::Relaxed);
}
pub fn last_attempt_unix_ms(&self) -> u64 {
self.last_attempt_unix_ms.load(Ordering::Relaxed)
}
pub fn last_success_unix_ms(&self) -> u64 {
self.last_success_unix_ms.load(Ordering::Relaxed)
}
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeStatusSnapshot {
pub runtime: TlsRuntimeRuntimeSection,
pub outbound: TlsRuntimeOutboundSection,
pub server: TlsRuntimeServerSection,
pub consumer: TlsRuntimeConsumerSection,
}
impl TlsRuntimeStatusSnapshot {
pub fn is_complete(&self) -> bool {
self.server.has_material || self.outbound.has_roots || self.outbound.has_mtls_identity
}
pub fn from_outbound_only(args: OutboundOnlySnapshotArgs) -> Self {
Self {
runtime: TlsRuntimeRuntimeSection {
generation: args.generation,
reload_enabled: args.reload_enabled,
detect_mode: args.detect_mode,
last_attempt_time: args.last_attempt_time,
last_success_time: args.last_success_time,
last_error: args.last_error,
source_path: args.source_path,
},
outbound: TlsRuntimeOutboundSection {
has_roots: args.has_roots,
has_mtls_identity: args.has_mtls_identity,
},
server: TlsRuntimeServerSection { has_material: false },
consumer: TlsRuntimeConsumerSection { stale_generation: false },
}
}
}
#[derive(Debug, Clone)]
pub struct OutboundOnlySnapshotArgs {
pub source_path: String,
pub generation: u64,
pub reload_enabled: bool,
pub detect_mode: &'static str,
pub last_attempt_time: Option<u64>,
pub last_success_time: Option<u64>,
pub last_error: Option<String>,
pub has_roots: bool,
pub has_mtls_identity: bool,
}
pub fn detect_mode_label(mode: crate::config::ReloadDetectMode) -> &'static str {
match mode {
crate::config::ReloadDetectMode::Poll => "poll",
crate::config::ReloadDetectMode::Watch => "watch",
crate::config::ReloadDetectMode::Hybrid => "hybrid",
}
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeRuntimeSection {
pub generation: u64,
pub reload_enabled: bool,
pub detect_mode: &'static str,
pub last_attempt_time: Option<u64>,
pub last_success_time: Option<u64>,
pub last_error: Option<String>,
pub source_path: String,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeOutboundSection {
pub has_roots: bool,
pub has_mtls_identity: bool,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeServerSection {
pub has_material: bool,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct TlsRuntimeConsumerSection {
pub stale_generation: bool,
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
fn make_published(generation: u64, fingerprint_bytes: &[u8]) -> Arc<TlsPublishedState<String>> {
Arc::new(TlsPublishedState {
generation: TlsGeneration(generation),
material: Arc::new("test".to_string()),
fingerprint: TlsFingerprint::from_optional_bytes(Some(fingerprint_bytes), None, None, None, None),
loaded_at_unix_ms: 0,
})
}
#[test]
fn runtime_state_tracks_generation_and_timestamps() {
let initial = make_published(1, b"aaa");
let state = TlsReloadRuntimeState::new(initial);
assert_eq!(state.current_generation(), TlsGeneration(1));
assert_eq!(state.bump_generation(), TlsGeneration(2));
assert_eq!(state.last_attempt_unix_ms(), 0);
assert_eq!(state.last_success_unix_ms(), 0);
state.mark_attempt(100);
assert_eq!(state.last_attempt_unix_ms(), 100);
state.mark_success(200);
assert_eq!(state.last_success_unix_ms(), 200);
}
#[test]
fn bump_generation_saturates_at_max() {
let initial = Arc::new(TlsPublishedState {
generation: TlsGeneration(u64::MAX),
material: Arc::new("max".to_string()),
fingerprint: TlsFingerprint::default(),
loaded_at_unix_ms: 0,
});
let state = TlsReloadRuntimeState::new(initial);
assert_eq!(state.bump_generation(), TlsGeneration(u64::MAX));
}
#[test]
fn status_snapshot_is_complete_with_outbound_roots() {
let snap = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: "/tmp".to_string(),
generation: 1,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: None,
last_success_time: None,
last_error: None,
has_roots: true,
has_mtls_identity: false,
});
assert!(snap.is_complete());
}
#[test]
fn status_snapshot_is_not_complete_when_empty() {
let snap = TlsRuntimeStatusSnapshot::from_outbound_only(OutboundOnlySnapshotArgs {
source_path: "/tmp".to_string(),
generation: 1,
reload_enabled: true,
detect_mode: "poll",
last_attempt_time: None,
last_success_time: None,
last_error: None,
has_roots: false,
has_mtls_identity: false,
});
assert!(!snap.is_complete());
}
}