refactor(tls): centralize runtime foundation (#3065)

* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* refactor(targets): move notify net helpers from utils

* refactor(tls): centralize runtime foundation

* feat(tls-runtime): add TLS debug state and admin handler

* refactor(tls-runtime): unify TLS debug consumer status view

* fix(tls): address PR3065 review feedback

* refactor(tls): align debug status payload types

* refactor(targets): harden TLS hot reload paths

* fix(targets): resolve review-4348251652 findings

* fix(targets): finalize tls runtime review follow-ups

* fix(targets): harden tls reload and review follow-ups

* fix(targets): align tls reload handling across targets

* fix(targets): finalize tls reload state and metrics updates

* chore(deps): trim unused TLS deps

* style(targets): normalize TLS reload formatting

* refactor(targets): introduce tls runtime adapter path

* chore: update workspace manifests for tls refactor

* fix(tls): stabilize material reload and audit workflow

* fix(targets): refresh tls fingerprint flow across sinks

* fix(tls): align runtime coordinator and http reader updates

* fix(sftp): simplify protocol error mapping

* fix(tls): harmonize material loading behavior

* fix(server): finalize tls material wiring in startup flow

* fix(protos): tighten tls generation cache and deps
This commit is contained in:
houseme
2026-05-24 14:41:15 +08:00
committed by GitHub
parent 8be787387c
commit d74e6eb042
67 changed files with 4978 additions and 1725 deletions
+7 -2
View File
@@ -14,20 +14,23 @@ documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
[dependencies]
rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] }
rustfs-ecstore = { workspace = true }
rustfs-utils = { workspace = true, features = ["notify", "tls"] }
rustfs-tls-runtime = { workspace = true }
rustfs-s3-types = { workspace = true }
async-trait = { workspace = true }
async-nats = { workspace = true }
deadpool-postgres = { workspace = true }
hyper = { workspace = true }
hyper-rustls = { workspace = true }
lapin = { workspace = true }
libc = { workspace = true }
pulsar = { workspace = true }
regex = { workspace = true }
reqwest = { workspace = true }
rumqttc = { workspace = true }
redis = { workspace = true }
rustls = { workspace = true }
rustls-native-certs = { workspace = true }
rustls-pki-types = { workspace = true }
s3s = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
snap = { workspace = true }
@@ -45,6 +48,8 @@ mysql_async = { workspace = true }
chrono = { workspace = true }
parking_lot = { workspace = true }
hashbrown = { workspace = true }
arc-swap = { workspace = true }
metrics = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
+2 -2
View File
@@ -64,8 +64,8 @@ pub async fn check_mqtt_broker_available_with_tls(
use crate::target::mqtt::build_mqtt_options;
use rumqttc::{AsyncClient, QoS};
let url = rustfs_utils::parse_url(broker_url)
.map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
let url =
crate::parse_url(broker_url).map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
let url = url.url();
// build_mqtt_options returns TargetError directly; Configuration variants propagate as-is.
+2
View File
@@ -20,6 +20,7 @@ pub mod control_plane;
pub mod domain;
pub mod error;
pub mod manifest;
mod net;
pub mod plugin;
pub mod runtime;
pub mod store;
@@ -48,6 +49,7 @@ pub use manifest::{
TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginPackaging,
TargetPluginRuntimeTransport, builtin_target_marketplace_manifest, installable_target_marketplace_manifest,
};
pub use net::*;
pub use plugin::{
BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetPluginRegistry,
TargetRequestValidator, boxed_target,
+642
View File
@@ -0,0 +1,642 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use hashbrown::HashMap;
use hyper::HeaderMap;
use regex::Regex;
use s3s::{S3Request, S3Response};
use serde::{Deserialize, Serialize};
use std::net::IpAddr;
use std::path::Path;
use std::sync::LazyLock;
use thiserror::Error;
use url::Url;
static HOST_LABEL_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$").unwrap());
/// NetError represents errors that can occur in network operations.
#[derive(Error, Debug)]
pub enum NetError {
#[error("invalid argument")]
InvalidArgument,
#[error("invalid hostname")]
InvalidHost,
#[error("missing '[' in host")]
MissingBracket,
#[error("parse error: {0}")]
ParseError(String),
#[error("unexpected scheme: {0}")]
UnexpectedScheme(String),
#[error("scheme appears with empty host")]
SchemeWithEmptyHost,
}
/// Host represents a network host with IP/name and port.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Host {
pub name: String,
pub port: Option<u16>,
}
impl Host {
pub fn is_empty(&self) -> bool {
self.name.is_empty()
}
pub fn equal(&self, other: &Host) -> bool {
self.to_string() == other.to_string()
}
}
impl std::fmt::Display for Host {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.port {
Some(p) => write!(f, "{}:{}", self.name, p),
None => write!(f, "{}", self.name),
}
}
}
/// Extract request parameters from S3Request, mainly header information.
pub fn extract_req_params<T>(req: &S3Request<T>) -> HashMap<String, String> {
extract_params_header(&req.headers)
}
/// Extract request parameters from hyper::HeaderMap, mainly header information.
/// This function is useful when you have a raw HTTP request and need to extract parameters.
#[deprecated(since = "0.1.0", note = "Use extract_params_header instead")]
pub fn extract_req_params_header(head: &HeaderMap) -> HashMap<String, String> {
extract_params_header(head)
}
/// Extract parameters from hyper::HeaderMap, mainly header information.
pub fn extract_params_header(head: &HeaderMap) -> HashMap<String, String> {
let mut params = HashMap::new();
for (key, value) in head.iter() {
if let Ok(val_str) = value.to_str() {
params.insert(key.as_str().to_string(), val_str.to_string());
}
}
params
}
/// Extract response elements from S3Response, mainly header information.
pub fn extract_resp_elements<T>(resp: &S3Response<T>) -> HashMap<String, String> {
extract_params_header(&resp.headers)
}
/// Get host from header information.
pub fn get_request_host(headers: &HeaderMap) -> String {
headers
.get("host")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get Port from header information.
/// Priority:
/// 1. x-forwarded-port
/// 2. host header (parse port)
/// 3. x-forwarded-proto inferred default (http=80, https=443) when host has no explicit port
/// 4. port header
pub fn get_request_port(headers: &HeaderMap) -> u16 {
if let Some(port) = headers
.get("x-forwarded-port")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u16>().ok())
{
return port;
}
if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) {
if let Some(idx) = host.rfind(':') {
let valid_colon = match host.rfind(']') {
Some(close_bracket_idx) => idx > close_bracket_idx,
None => true,
};
if valid_colon
&& let Ok(port) = host[idx + 1..].parse::<u16>()
&& port > 0
{
return port;
}
}
if let Some(proto) = headers.get("x-forwarded-proto").and_then(|v| v.to_str().ok()) {
match proto {
"http" => return 80,
"https" => return 443,
_ => {}
}
}
}
headers
.get("port")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u16>().ok())
.unwrap_or(0)
}
/// Get content-length from header information.
pub fn get_request_content_length(headers: &HeaderMap) -> u64 {
headers
.get("content-length")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0)
}
/// Get referer from header information.
pub fn get_request_referer(headers: &HeaderMap) -> String {
headers
.get("referer")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get user-agent from header information.
pub fn get_request_user_agent(headers: &HeaderMap) -> String {
headers
.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// parse_host parses a string into a Host, with validation similar to Go's ParseHost.
pub fn parse_host(s: &str) -> Result<Host, NetError> {
if s.is_empty() {
return Err(NetError::InvalidArgument);
}
let is_valid_host = |host: &str| -> bool {
if host.is_empty() {
return true;
}
if host.parse::<IpAddr>().is_ok() {
return true;
}
if !(1..=253).contains(&host.len()) {
return false;
}
for (i, label) in host.split('.').enumerate() {
if i + 1 == host.split('.').count() && label.is_empty() {
continue;
}
if !(1..=63).contains(&label.len()) || !HOST_LABEL_REGEX.is_match(label) {
return false;
}
}
true
};
let (host, port) = if let Some(rest) = s.strip_prefix('[') {
let Some(end) = rest.find(']') else {
return Err(NetError::MissingBracket);
};
let host = rest[..end].to_string();
let port_str = &rest[end + 1..];
let port = if let Some(port_str) = port_str.strip_prefix(':') {
if port_str.is_empty() {
None
} else {
Some(port_str.parse().map_err(|_| NetError::ParseError(port_str.to_string()))?)
}
} else if port_str.is_empty() {
None
} else {
return Err(NetError::InvalidHost);
};
(host, port)
} else {
if s.contains(']') {
return Err(NetError::MissingBracket);
}
let (host_str, port_str) = if s.matches(':').count() > 1 {
(s, "")
} else {
s.rsplit_once(':').map_or((s, ""), |(h, p)| (h, p))
};
let port = if !port_str.is_empty() {
Some(port_str.parse().map_err(|_| NetError::ParseError(port_str.to_string()))?)
} else {
None
};
(trim_ipv6(host_str)?, port)
};
let trimmed_host = host.split('%').next().unwrap_or(&host);
if !is_valid_host(trimmed_host) {
return Err(NetError::InvalidHost);
}
Ok(Host { name: host, port })
}
fn trim_ipv6(host: &str) -> Result<String, NetError> {
if host.ends_with(']') {
if !host.starts_with('[') {
return Err(NetError::MissingBracket);
}
Ok(host[1..host.len() - 1].to_string())
} else {
Ok(host.to_string())
}
}
/// URL is a wrapper around url::Url for custom handling.
#[derive(Debug, Clone)]
pub struct ParsedURL(pub Url);
impl ParsedURL {
pub fn is_empty(&self) -> bool {
self.0.as_str() == "" || (self.0.scheme() == "about" && self.0.path() == "blank")
}
pub fn hostname(&self) -> String {
self.0.host_str().unwrap_or("").to_string()
}
pub fn port(&self) -> String {
match self.0.port() {
Some(p) => p.to_string(),
None => match self.0.scheme() {
"http" => "80".to_string(),
"https" => "443".to_string(),
_ => "".to_string(),
},
}
}
pub fn scheme(&self) -> &str {
self.0.scheme()
}
pub fn url(&self) -> &Url {
&self.0
}
}
impl std::fmt::Display for ParsedURL {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut url = self.0.clone();
if let Some(host) = url.host_str().map(|h| h.to_string())
&& let Some(port) = url.port()
&& ((url.scheme() == "http" && port == 80) || (url.scheme() == "https" && port == 443))
{
let _ = url.set_host(Some(&host));
let _ = url.set_port(None);
}
let mut s = url.to_string();
if s.ends_with('/') && url.path() == "/" {
s.pop();
}
write!(f, "{s}")
}
}
impl serde::Serialize for ParsedURL {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.to_string())
}
}
impl<'de> serde::Deserialize<'de> for ParsedURL {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let s: String = serde::Deserialize::deserialize(deserializer)?;
if s.is_empty() {
Ok(ParsedURL(Url::parse("about:blank").unwrap()))
} else {
parse_url(&s).map_err(serde::de::Error::custom)
}
}
}
/// parse_url parses a string into a ParsedURL, with host validation and path cleaning.
pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
if let Some(scheme_end) = s.find("://")
&& s[scheme_end + 3..].starts_with('/')
{
let scheme = &s[..scheme_end];
if !scheme.is_empty() {
return Err(NetError::SchemeWithEmptyHost);
}
}
let mut uu = Url::parse(s).map_err(|e| NetError::ParseError(e.to_string()))?;
if uu.host_str().is_none_or(|h| h.is_empty()) {
if uu.scheme() != "" {
return Err(NetError::SchemeWithEmptyHost);
}
} else {
let port_str = uu.port().map(|p| p.to_string()).unwrap_or_else(|| match uu.scheme() {
"http" => "80".to_string(),
"https" => "443".to_string(),
_ => "".to_string(),
});
if !port_str.is_empty() {
let host_port = format!("{}:{}", uu.host_str().unwrap(), port_str);
parse_host(&host_port)?;
}
}
if !uu.path().is_empty() {
let mut cleaned_path = String::new();
for comp in Path::new(uu.path()).components() {
use std::path::Component;
match comp {
Component::RootDir => cleaned_path.push('/'),
Component::Normal(s) => {
if !cleaned_path.ends_with('/') {
cleaned_path.push('/');
}
cleaned_path.push_str(&s.to_string_lossy());
}
_ => {}
}
}
if s.ends_with('/') && !cleaned_path.ends_with('/') {
cleaned_path.push('/');
}
if cleaned_path.is_empty() {
cleaned_path.push('/');
}
uu.set_path(&cleaned_path);
}
Ok(ParsedURL(uu))
}
#[allow(dead_code)]
pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
let u = parse_url(s)?;
match u.0.scheme() {
"http" | "https" => Ok(u),
_ => Err(NetError::UnexpectedScheme(u.0.scheme().to_string())),
}
}
#[allow(dead_code)]
pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool {
if err.kind() == std::io::ErrorKind::TimedOut {
return !expect_timeouts;
}
let err_str = err.to_string().to_lowercase();
err_str.contains("connection reset by peer")
|| err_str.contains("connection timed out")
|| err_str.contains("broken pipe")
|| err_str.contains("use of closed network connection")
}
#[allow(dead_code)]
pub fn is_conn_reset_err(err: &std::io::Error) -> bool {
err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET))
}
#[allow(dead_code)]
pub fn is_conn_refused_err(err: &std::io::Error) -> bool {
err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED))
}
#[cfg(test)]
mod tests {
use super::*;
use hyper::header::HeaderValue;
#[test]
fn test_get_request_port() {
let mut headers = HeaderMap::new();
assert_eq!(get_request_port(&headers), 0);
headers.insert("port", HeaderValue::from_static("8080"));
assert_eq!(get_request_port(&headers), 8080);
headers.remove("port");
headers.insert("host", HeaderValue::from_static("example.com:9000"));
assert_eq!(get_request_port(&headers), 9000);
headers.insert("host", HeaderValue::from_static("example.com"));
assert_eq!(get_request_port(&headers), 0);
headers.insert("host", HeaderValue::from_static("[::1]:9001"));
assert_eq!(get_request_port(&headers), 9001);
headers.insert("host", HeaderValue::from_static("[::1]"));
assert_eq!(get_request_port(&headers), 0);
headers.insert("x-forwarded-port", HeaderValue::from_static("7000"));
assert_eq!(get_request_port(&headers), 7000);
headers.remove("x-forwarded-port");
headers.insert("host", HeaderValue::from_static("example.com"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("http"));
assert_eq!(get_request_port(&headers), 80);
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
assert_eq!(get_request_port(&headers), 443);
headers.insert("x-forwarded-proto", HeaderValue::from_static("ftp"));
assert_eq!(get_request_port(&headers), 0);
headers.insert("host", HeaderValue::from_static("example.com:0"));
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
assert_eq!(get_request_port(&headers), 443);
headers.remove("x-forwarded-proto");
assert_eq!(get_request_port(&headers), 0);
}
#[test]
fn parse_host_with_empty_string_returns_error() {
let result = parse_host("");
assert!(matches!(result, Err(NetError::InvalidArgument)));
}
#[test]
fn parse_host_with_valid_ipv4() {
let result = parse_host("192.168.1.1:8080");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "192.168.1.1");
assert_eq!(host.port, Some(8080));
}
#[test]
fn parse_host_with_valid_hostname() {
let result = parse_host("example.com:443");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "example.com");
assert_eq!(host.port, Some(443));
}
#[test]
fn parse_host_with_ipv6_brackets() {
let result = parse_host("[::1]:8080");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "::1");
assert_eq!(host.port, Some(8080));
}
#[test]
fn parse_host_with_bare_ipv6_without_port() {
let result = parse_host("::1");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "::1");
assert_eq!(host.port, None);
}
#[test]
fn parse_host_with_ipv6_zone_without_port() {
let result = parse_host("fe80::1%eth0");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "fe80::1%eth0");
assert_eq!(host.port, None);
}
#[test]
fn parse_host_with_bracketed_ipv6_zone_and_port() {
let result = parse_host("[fe80::1%eth0]:9000");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "fe80::1%eth0");
assert_eq!(host.port, Some(9000));
}
#[test]
fn parse_host_with_bracketed_ipv6_without_port() {
let result = parse_host("[::1]");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "::1");
assert_eq!(host.port, None);
}
#[test]
fn parse_host_with_invalid_ipv6_missing_bracket() {
let result = parse_host("::1]:8080");
assert!(matches!(result, Err(NetError::MissingBracket)));
}
#[test]
fn parse_host_with_invalid_hostname() {
let result = parse_host("invalid..host:80");
assert!(matches!(result, Err(NetError::InvalidHost)));
}
#[test]
fn parse_host_without_port() {
let result = parse_host("example.com");
assert!(result.is_ok());
let host = result.unwrap();
assert_eq!(host.name, "example.com");
assert_eq!(host.port, None);
}
#[test]
fn host_is_empty_when_name_is_empty() {
let host = Host {
name: "".to_string(),
port: None,
};
assert!(host.is_empty());
}
#[test]
fn host_is_not_empty_when_name_present() {
let host = Host {
name: "example.com".to_string(),
port: Some(80),
};
assert!(!host.is_empty());
}
#[test]
fn host_to_string_with_port() {
let host = Host {
name: "example.com".to_string(),
port: Some(80),
};
assert_eq!(host.to_string(), "example.com:80");
}
#[test]
fn host_to_string_without_port() {
let host = Host {
name: "example.com".to_string(),
port: None,
};
assert_eq!(host.to_string(), "example.com");
}
#[test]
fn parse_url_with_valid_http_url() {
let result = parse_url("http://example.com/path");
assert!(result.is_ok());
let parsed = result.unwrap();
assert_eq!(parsed.hostname(), "example.com");
assert_eq!(parsed.port(), "80");
assert_eq!(parsed.scheme(), "http");
assert_eq!(parsed.to_string(), "http://example.com/path");
}
#[test]
fn parse_url_with_explicit_default_https_port() {
let result = parse_url("https://example.com:443/path");
assert!(result.is_ok());
let parsed = result.unwrap();
assert_eq!(parsed.to_string(), "https://example.com/path");
}
#[test]
fn parse_url_with_empty_host_returns_error() {
let result = parse_url("http:///path");
assert!(matches!(result, Err(NetError::SchemeWithEmptyHost)));
}
#[test]
fn parse_url_with_invalid_host_returns_error() {
let result = parse_url("http://invalid..host/path");
assert!(matches!(result, Err(NetError::InvalidHost)));
}
#[test]
fn parse_url_normalizes_path() {
let result = parse_url("http://example.com//path/../path/");
assert!(result.is_ok());
let parsed = result.unwrap();
assert_eq!(parsed.to_string(), "http://example.com/path/");
}
}
+1
View File
@@ -15,6 +15,7 @@
pub mod adapter;
pub mod sidecar;
pub mod sidecar_protocol;
pub mod tls;
use crate::Target;
use crate::arn::TargetID;
+217
View File
@@ -0,0 +1,217 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! `TlsReloadAdapter<M>` — the single entry-point that connects a target to
//! the TLS reload coordinator. Each target holds an `Option<TlsReloadAdapter<M>>`
//! and calls `current_material()` on the hot path. When `None`, the target
//! falls back to its legacy inline fingerprint logic.
use super::config::TlsReloadOptions;
use super::coordinator::TargetTlsReloadCoordinator;
use super::state::{TargetTlsRuntimeState, TargetTlsStatusSnapshot};
use super::r#trait::ReloadableTargetTls;
use std::sync::Arc;
use tracing::warn;
/// Bridges a `ReloadableTargetTls` implementor and the reload coordinator.
///
/// Created via [`TlsReloadAdapter::try_register`]. Holds the coordinator-
/// managed runtime state and exposes a zero-cost `current_material()` accessor
/// for the send hot-path.
pub struct TlsReloadAdapter<M> {
runtime_state: Arc<TargetTlsRuntimeState<M>>,
options: TlsReloadOptions,
}
impl<M> std::fmt::Debug for TlsReloadAdapter<M> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TlsReloadAdapter")
.field("target_label", &self.runtime_state.inputs.target_label)
.finish_non_exhaustive()
}
}
impl<M> Clone for TlsReloadAdapter<M> {
fn clone(&self) -> Self {
Self {
runtime_state: Arc::clone(&self.runtime_state),
options: self.options.clone(),
}
}
}
impl<M: Send + Sync + 'static> TlsReloadAdapter<M> {
/// Registers `target` with the coordinator and returns an adapter.
///
/// On success the coordinator has:
/// - built initial TLS material
/// - spawned a background poll loop
///
/// On failure returns `None` (the caller should keep its inline fallback
/// path intact — the target continues to work, just without coordinator
/// support).
pub async fn try_register<T: ReloadableTargetTls<Material = M>>(
target: Arc<T>,
options: TlsReloadOptions,
coordinator: &TargetTlsReloadCoordinator,
) -> Option<Self> {
let label = target.tls_input_set().target_label.clone();
match coordinator.register(target, options.clone()).await {
Ok(runtime_state) => {
tracing::info!(target = %label, "TLS reload adapter registered");
Some(Self { runtime_state, options })
}
Err(err) => {
warn!(target = %label, error = %err, "TLS reload adapter registration failed; target will use inline fallback");
None
}
}
}
/// Hot-path accessor: returns the current TLS material managed by the
/// coordinator. The returned `Arc<M>` is cheap to clone.
#[inline]
pub fn current_material(&self) -> Arc<M> {
Arc::clone(&self.runtime_state.current.load().material)
}
/// Returns the active generation counter.
#[inline]
pub fn generation(&self) -> u64 {
self.runtime_state.current.load().generation.0
}
/// Returns a read-only status snapshot for admin/observability.
pub fn status_snapshot(&self) -> TargetTlsStatusSnapshot {
TargetTlsReloadCoordinator::build_status_snapshot(&self.runtime_state, &self.options)
}
/// Returns the underlying runtime state (for `close()` cleanup etc.).
pub fn runtime_state(&self) -> &Arc<TargetTlsRuntimeState<M>> {
&self.runtime_state
}
/// Unregisters from the coordinator (stops the poll loop).
pub async fn unregister(&self, coordinator: &TargetTlsReloadCoordinator) {
let label = &self.runtime_state.inputs.target_label;
if let Err(err) = coordinator.unregister(label).await {
warn!(target = %label, error = %err, "Failed to unregister TLS reload adapter");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::TargetError;
use async_trait::async_trait;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
struct FakeTarget {
label: String,
build_calls: AtomicUsize,
should_fail: AtomicBool,
}
impl FakeTarget {
fn new(label: &str) -> Self {
Self {
label: label.to_string(),
build_calls: AtomicUsize::new(0),
should_fail: AtomicBool::new(false),
}
}
}
#[async_trait]
impl ReloadableTargetTls for FakeTarget {
type Material = String;
fn tls_input_set(&self) -> super::super::state::TargetTlsInputSet {
super::super::state::TargetTlsInputSet {
ca_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
target_label: self.label.clone(),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
self.build_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("fail".to_string()));
}
Ok("material".to_string())
}
async fn apply_tls_material(
&self,
_generation: super::super::fingerprint::TargetTlsGeneration,
_material: Arc<Self::Material>,
_mode: super::super::config::ReloadApplyMode,
) -> Result<(), TargetError> {
Ok(())
}
}
#[tokio::test]
async fn try_register_returns_adapter_on_success() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:fake"));
let options = TlsReloadOptions::default();
let adapter = TlsReloadAdapter::try_register(target.clone(), options, &coordinator).await;
assert!(adapter.is_some());
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
let a = adapter.unwrap();
assert_eq!(*a.current_material(), "material");
}
#[tokio::test]
async fn try_register_returns_none_on_failure() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:fail"));
target.should_fail.store(true, Ordering::SeqCst);
let options = TlsReloadOptions::default();
let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await;
assert!(adapter.is_none());
}
#[tokio::test]
async fn adapter_is_clone_and_shares_state() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:clone"));
let options = TlsReloadOptions::default();
let a = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
let b = a.clone();
assert_eq!(*a.current_material(), *b.current_material());
assert_eq!(a.generation(), b.generation());
}
#[tokio::test]
async fn status_snapshot_contains_label() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(FakeTarget::new("test:snap"));
let options = TlsReloadOptions::default();
let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
let snap = adapter.status_snapshot();
assert_eq!(snap.target_label, "test:snap");
assert!(snap.reload_enabled);
}
}
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Target-level TLS reload configuration.
//!
//! Re-exports the shared types from `rustfs_tls_runtime::config` so that
//! targets and their callers use a single source of truth.
pub use rustfs_tls_runtime::config::{ReloadApplyHint as ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};
@@ -0,0 +1,656 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Target TLS reload coordinator. Manages per-target background poll loops
//! that periodically check TLS material fingerprints and drive safe reload.
use super::config::{ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};
#[cfg(test)]
use super::fingerprint::TargetTlsFingerprint;
use super::fingerprint::{TargetTlsGeneration, build_target_tls_fingerprint};
use super::metrics::{record_target_tls_publication_fail, record_target_tls_reload_result, record_target_tls_reload_skipped};
#[cfg(test)]
use super::state::TargetTlsInputSet;
use super::state::{TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
use super::r#trait::ReloadableTargetTls;
use super::validate::validate_tls_material;
use crate::error::TargetError;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tokio::task::JoinHandle;
use tracing::{debug, info, warn};
struct TargetReloadEntry {
#[expect(dead_code)]
target_label: String,
cancel_tx: tokio::sync::mpsc::Sender<()>,
poll_handle: JoinHandle<()>,
}
/// The top-level coordinator that manages TLS reload for all registered targets.
///
/// Typically one instance per process, held alongside `TargetRuntimeManager`.
/// Each registered target gets its own background poll loop that periodically
/// checks TLS fingerprints and drives the build/apply cycle.
pub struct TargetTlsReloadCoordinator {
entries: RwLock<HashMap<String, TargetReloadEntry>>,
}
impl Default for TargetTlsReloadCoordinator {
fn default() -> Self {
Self::new()
}
}
impl TargetTlsReloadCoordinator {
pub fn new() -> Self {
Self {
entries: RwLock::new(HashMap::new()),
}
}
/// Register a target for coordinated TLS reload. Spawns a background poll loop.
///
/// Returns the initial runtime state that the target should hold for
/// accessing the current TLS material via `ArcSwap`.
pub async fn register<T: ReloadableTargetTls>(
&self,
target: Arc<T>,
options: TlsReloadOptions,
) -> Result<Arc<TargetTlsRuntimeState<T::Material>>, TargetError> {
if !options.enabled {
return Err(TargetError::Configuration("TLS reload is disabled".to_string()));
}
let inputs = target.tls_input_set();
let target_label = inputs.target_label.clone();
// Build initial material
let initial_material = Arc::new(target.build_tls_material().await?);
let initial_fingerprint =
build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: unix_time_ms(),
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state.clone(), inputs));
if options.detect_mode == ReloadDetectMode::Poll || options.detect_mode == ReloadDetectMode::Hybrid {
let (cancel_tx, cancel_rx) = tokio::sync::mpsc::channel(1);
let poll_handle = tokio::spawn(spawn_target_poll_loop(target, Arc::clone(&runtime_state), options, cancel_rx));
let mut entries = self.entries.write().await;
entries.insert(
target_label.clone(),
TargetReloadEntry {
target_label: target_label.clone(),
cancel_tx,
poll_handle,
},
);
info!(target = %target_label, "Registered target for TLS reload coordinator");
}
Ok(runtime_state)
}
/// Unregister a target and stop its poll loop.
pub async fn unregister(&self, target_label: &str) -> Result<(), TargetError> {
let mut entries = self.entries.write().await;
if let Some(entry) = entries.remove(target_label) {
let _ = entry.cancel_tx.send(()).await;
entry.poll_handle.abort();
info!(target = %target_label, "Unregistered target from TLS reload coordinator");
}
Ok(())
}
/// Force an immediate reload check for a specific target.
/// Used by admin endpoints and test harnesses.
pub async fn force_reload<T: ReloadableTargetTls>(
&self,
target: &T,
runtime_state: &TargetTlsRuntimeState<T::Material>,
options: &TlsReloadOptions,
) -> Result<TargetTlsGeneration, TargetError> {
reload_target_once(target, runtime_state, options).await
}
/// Stop all poll loops.
pub async fn shutdown(&self) {
let mut entries = self.entries.write().await;
for (label, entry) in entries.drain() {
let _ = entry.cancel_tx.send(()).await;
entry.poll_handle.abort();
debug!(target = %label, "Stopped TLS reload poll loop");
}
}
/// Collect status snapshots from all registered targets.
/// The caller must provide the runtime states separately since the
/// coordinator does not hold type-erased references to them.
pub fn build_status_snapshot<M>(
runtime_state: &TargetTlsRuntimeState<M>,
options: &TlsReloadOptions,
) -> TargetTlsStatusSnapshot {
let current = runtime_state.current.load();
let last_attempt = runtime_state.last_attempt_unix_ms();
let last_success = runtime_state.last_success_unix_ms();
let last_error = runtime_state.last_error.read().clone();
TargetTlsStatusSnapshot {
target_label: runtime_state.inputs.target_label.clone(),
generation: current.generation.0,
reload_enabled: options.enabled,
detect_mode: match options.detect_mode {
ReloadDetectMode::Poll => "poll",
ReloadDetectMode::Watch => "watch",
ReloadDetectMode::Hybrid => "hybrid",
},
apply_mode: match options.apply_hint {
ReloadApplyMode::Lazy => "lazy",
ReloadApplyMode::SoftReconnect => "soft_reconnect",
},
last_attempt_time: if last_attempt > 0 { Some(last_attempt) } else { None },
last_success_time: if last_success > 0 { Some(last_success) } else { None },
last_error,
ca_path: runtime_state.inputs.ca_path.clone(),
client_cert_path: runtime_state.inputs.client_cert_path.clone(),
client_key_path: runtime_state.inputs.client_key_path.clone(),
}
}
}
/// Background poll loop for a single target.
async fn spawn_target_poll_loop<T: ReloadableTargetTls>(
target: Arc<T>,
runtime_state: Arc<TargetTlsRuntimeState<T::Material>>,
options: TlsReloadOptions,
mut cancel_rx: tokio::sync::mpsc::Receiver<()>,
) {
let mut interval = tokio::time::interval(options.interval);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
interval.tick().await; // skip the immediate first tick
let label = &runtime_state.inputs.target_label;
let debounce = options.debounce;
debug!(target = %label, interval_secs = options.interval.as_secs(), "TLS reload poll loop started");
loop {
tokio::select! {
biased;
_ = cancel_rx.recv() => {
info!(target = %label, "TLS reload poll loop stopped");
return;
}
_ = interval.tick() => {
// Enforce minimum stable age: if the last attempt was too recent
// (e.g. a rapid succession of ticks), wait one debounce period
// before reading files again to avoid picking up half-written certs.
let last_attempt = runtime_state.last_attempt_unix_ms();
if last_attempt > 0 {
let elapsed_since_last = unix_time_ms().saturating_sub(last_attempt);
if elapsed_since_last < debounce.as_millis() as u64 {
continue;
}
}
if let Err(err) = reload_target_once(target.as_ref(), runtime_state.as_ref(), &options).await {
warn!(target = %label, error = %err, "TLS reload poll check failed (will retry)");
}
}
}
}
}
/// Single reload cycle: read → compare → validate → build → apply → publish.
///
/// Returns the new generation on success, or an error on failure.
/// On failure the current generation and material are untouched.
async fn reload_target_once<T: ReloadableTargetTls>(
target: &T,
runtime_state: &TargetTlsRuntimeState<T::Material>,
options: &TlsReloadOptions,
) -> Result<TargetTlsGeneration, TargetError> {
let now = unix_time_ms();
runtime_state.mark_attempt(now);
let started_at = std::time::Instant::now();
let label = &runtime_state.inputs.target_label;
// 1. Read TLS files and compute fingerprint
let inputs = &runtime_state.inputs;
let next_fingerprint =
build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
// 2. Compare with current — skip if unchanged
let current = runtime_state.current.load();
if current.fingerprint == next_fingerprint {
record_target_tls_reload_skipped(label, "unchanged");
return Ok(current.generation);
}
// 3. Validate TLS files (cert/key pairing, CA parseable)
if let Err(err) = validate_tls_material(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path) {
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
// Also call target-specific validation
if let Err(err) = target.validate_tls_files().await {
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
// 4. Build new material (does not touch current state yet)
let new_material = match target.build_tls_material().await {
Ok(m) => Arc::new(m),
Err(err) => {
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
};
// 5. Bump generation and apply
let new_generation = runtime_state.bump_generation();
if let Err(err) = target
.apply_tls_material(new_generation, Arc::clone(&new_material), options.apply_hint)
.await
{
*runtime_state.last_error.write() = Some(err.to_string());
record_target_tls_publication_fail(label);
return Err(err);
}
// 6. Publish new state
let published = Arc::new(TargetTlsPublishedState {
generation: new_generation,
fingerprint: next_fingerprint,
material: new_material,
loaded_at_unix_ms: now,
});
runtime_state.current.store(published.clone());
runtime_state.last_good.store(published);
runtime_state.mark_success(now);
*runtime_state.last_error.write() = None;
record_target_tls_reload_result(label, "ok", started_at.elapsed().as_secs_f64(), new_generation.0);
debug!(target = %label, generation = new_generation.0, "TLS reload successful");
Ok(new_generation)
}
fn unix_time_ms() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
struct MockTarget {
inputs: TargetTlsInputSet,
build_calls: AtomicUsize,
apply_calls: AtomicUsize,
validate_calls: AtomicUsize,
should_fail_build: AtomicBool,
should_fail_apply: AtomicBool,
should_fail_validate: AtomicBool,
}
impl MockTarget {
fn new(label: &str) -> Self {
Self {
inputs: TargetTlsInputSet {
ca_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
target_label: label.to_string(),
},
build_calls: AtomicUsize::new(0),
apply_calls: AtomicUsize::new(0),
validate_calls: AtomicUsize::new(0),
should_fail_build: AtomicBool::new(false),
should_fail_apply: AtomicBool::new(false),
should_fail_validate: AtomicBool::new(false),
}
}
}
#[async_trait::async_trait]
impl ReloadableTargetTls for MockTarget {
type Material = String;
fn tls_input_set(&self) -> TargetTlsInputSet {
self.inputs.clone()
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
self.build_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail_build.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("build failed".to_string()));
}
Ok("mock-material".to_string())
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
_material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
self.apply_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail_apply.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("apply failed".to_string()));
}
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
self.validate_calls.fetch_add(1, Ordering::SeqCst);
if self.should_fail_validate.load(Ordering::SeqCst) {
return Err(TargetError::Configuration("validate failed".to_string()));
}
Ok(())
}
}
fn default_options() -> TlsReloadOptions {
TlsReloadOptions {
enabled: true,
detect_mode: ReloadDetectMode::Poll,
interval: std::time::Duration::from_secs(1),
debounce: std::time::Duration::from_secs(1),
min_stable_age: std::time::Duration::from_millis(100),
apply_hint: ReloadApplyMode::Lazy,
}
}
#[tokio::test]
async fn register_builds_initial_material() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:webhook"));
let options = TlsReloadOptions {
detect_mode: ReloadDetectMode::Watch, // no poll loop for this test
..default_options()
};
let state = coordinator.register(target.clone(), options).await.unwrap();
assert_eq!(state.current.load().generation, TargetTlsGeneration(1));
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn register_disabled_returns_error() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:webhook"));
let options = TlsReloadOptions {
enabled: false,
..default_options()
};
let result = coordinator.register(target, options).await;
assert!(result.is_err());
}
#[tokio::test]
async fn shutdown_stops_all_loops() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:webhook"));
let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
assert_eq!(coordinator.entries.read().await.len(), 1);
coordinator.shutdown().await;
assert!(coordinator.entries.read().await.is_empty());
}
#[tokio::test]
async fn force_reload_calls_build_and_apply() {
let target = MockTarget::new("test:webhook");
let initial_material = Arc::new("initial".to_string());
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: TargetTlsFingerprint::default(),
material: initial_material,
loaded_at_unix_ms: 0,
});
let inputs = target.tls_input_set();
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, inputs));
let options = default_options();
// Force reload should succeed since MockTarget uses empty paths
// and the fingerprint won't change from default
let result = reload_target_once(&target, &runtime_state, &options).await.unwrap();
// Since fingerprint is unchanged (empty paths), generation stays at 1
assert_eq!(result, TargetTlsGeneration(1));
// Build should NOT be called because fingerprint unchanged
assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn build_failure_preserves_old_generation() {
let target = MockTarget::new("test:webhook");
target.should_fail_build.store(true, Ordering::SeqCst);
// Use a non-default fingerprint so the reload will detect a change
// (empty paths → default fingerprint ≠ initial fingerprint)
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
// Empty paths produce default fingerprint which differs from initial →
// validate passes (empty paths), then build is called and fails.
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_err());
// Generation should remain at 1
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn status_snapshot_reflects_state() {
let target = MockTarget::new("test:webhook");
let initial_material = Arc::new("initial".to_string());
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: TargetTlsFingerprint::default(),
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
let snapshot = TargetTlsReloadCoordinator::build_status_snapshot(runtime_state.as_ref(), &options);
assert_eq!(snapshot.target_label, "test:webhook");
assert_eq!(snapshot.generation, 1);
assert!(snapshot.reload_enabled);
assert_eq!(snapshot.detect_mode, "poll");
assert_eq!(snapshot.apply_mode, "lazy");
assert!(snapshot.last_attempt_time.is_none());
assert!(snapshot.last_error.is_none());
}
#[tokio::test]
async fn apply_failure_preserves_old_generation() {
let target = MockTarget::new("test:webhook");
target.should_fail_apply.store(true, Ordering::SeqCst);
// Use a non-default fingerprint so reload detects a change
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([42; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(3),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_err());
// Generation should remain at 3
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(3));
assert!(runtime_state.last_error.read().is_some());
}
#[tokio::test]
async fn validate_failure_prevents_build() {
let target = MockTarget::new("test:kafka");
target.should_fail_validate.store(true, Ordering::SeqCst);
// Non-default fingerprint to trigger reload
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([99; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_err());
// Build should NOT have been called
assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
// Validate was called
assert!(target.validate_calls.load(Ordering::SeqCst) > 0);
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
}
#[tokio::test]
async fn error_is_cleared_on_successful_reload() {
let target = MockTarget::new("test:nats");
let initial_material = Arc::new("initial".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(1),
fingerprint: initial_fingerprint,
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
let options = default_options();
// First: fail the reload
target.should_fail_build.store(true, Ordering::SeqCst);
let _ = reload_target_once(&target, &runtime_state, &options).await;
assert!(runtime_state.last_error.read().is_some());
// Now succeed (fingerprint still different from default)
target.should_fail_build.store(false, Ordering::SeqCst);
let result = reload_target_once(&target, &runtime_state, &options).await;
assert!(result.is_ok());
assert!(runtime_state.last_error.read().is_none());
assert!(runtime_state.last_success_unix_ms() > 0);
}
#[tokio::test]
async fn last_good_is_never_overwritten_by_failed_reload() {
let target = MockTarget::new("test:amqp");
let initial_material = Arc::new("good".to_string());
let initial_fingerprint = TargetTlsFingerprint {
ca_sha256: Some([5; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let initial_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(2),
fingerprint: initial_fingerprint.clone(),
material: initial_material.clone(),
loaded_at_unix_ms: 100,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
// Verify last_good matches initial
let good = runtime_state.last_good.load();
assert_eq!(good.generation, TargetTlsGeneration(2));
// Fail a reload
target.should_fail_build.store(true, Ordering::SeqCst);
let _ = reload_target_once(&target, &runtime_state, &default_options()).await;
// last_good should still be the initial state
let good_after = runtime_state.last_good.load();
assert_eq!(good_after.generation, TargetTlsGeneration(2));
assert_eq!(good_after.fingerprint, initial_fingerprint);
}
#[tokio::test]
async fn unregister_stops_target_poll_loop() {
let coordinator = TargetTlsReloadCoordinator::new();
let target = Arc::new(MockTarget::new("test:pulsar"));
let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
assert_eq!(coordinator.entries.read().await.len(), 1);
coordinator.unregister("test:pulsar").await.unwrap();
assert!(coordinator.entries.read().await.is_empty());
}
#[tokio::test]
async fn bump_generation_saturates_at_max() {
let target = MockTarget::new("test:saturation");
let initial_material = Arc::new("initial".to_string());
let max_gen_state = Arc::new(TargetTlsPublishedState {
generation: TargetTlsGeneration(u64::MAX),
fingerprint: TargetTlsFingerprint::default(),
material: initial_material,
loaded_at_unix_ms: 0,
});
let runtime_state = Arc::new(TargetTlsRuntimeState::new(max_gen_state, target.tls_input_set()));
let bumped = runtime_state.bump_generation();
assert_eq!(bumped, TargetTlsGeneration(u64::MAX)); // saturating add
}
}
@@ -0,0 +1,160 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! TLS fingerprint types for per-target certificate hot-reload detection.
use crate::error::TargetError;
/// SHA256 digest per TLS file component used to detect certificate changes.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TargetTlsFingerprint {
pub ca_sha256: Option<[u8; 32]>,
pub client_cert_sha256: Option<[u8; 32]>,
pub client_key_sha256: Option<[u8; 32]>,
}
/// Monotonically increasing generation counter bumped on each successful reload.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TargetTlsGeneration(pub u64);
/// Combined TLS state held per-target for tracking reload progress.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TargetTlsState {
pub generation: TargetTlsGeneration,
pub fingerprint: Option<TargetTlsFingerprint>,
}
impl TargetTlsState {
/// Compares `next_fingerprint` with the current one. If different, bumps
/// generation and stores the new fingerprint. Returns `true` when changed.
pub fn refresh(&mut self, next_fingerprint: TargetTlsFingerprint) -> bool {
if self.fingerprint.as_ref() == Some(&next_fingerprint) {
return false;
}
self.generation = TargetTlsGeneration(self.generation.0.saturating_add(1));
self.fingerprint = Some(next_fingerprint);
true
}
/// Checks whether `candidate` differs from the stored fingerprint without
/// mutating state. Use this to gate a rebuild, then call `refresh` only
/// after the rebuild succeeds.
pub fn needs_update(&self, candidate: &TargetTlsFingerprint) -> bool {
self.fingerprint.as_ref() != Some(candidate)
}
/// Resets state to default (generation 0, no fingerprint).
pub fn reset(&mut self) {
*self = Self::default();
}
}
/// Reads the three TLS material files from disk and returns a fingerprint
/// computed from their SHA256 digests. Empty paths produce `None` digests.
pub async fn build_target_tls_fingerprint(
ca_path: &str,
client_cert_path: &str,
client_key_path: &str,
) -> Result<TargetTlsFingerprint, TargetError> {
async fn load_optional_digest(path: &str) -> Result<Option<[u8; 32]>, TargetError> {
if path.is_empty() {
return Ok(None);
}
let bytes = tokio::fs::read(path)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read TLS material '{path}': {e}")))?;
let digest = rustfs_tls_runtime::TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None).server_sha256;
Ok(digest)
}
Ok(TargetTlsFingerprint {
ca_sha256: load_optional_digest(ca_path).await?,
client_cert_sha256: load_optional_digest(client_cert_path).await?,
client_key_sha256: load_optional_digest(client_key_path).await?,
})
}
#[cfg(test)]
mod tests {
use super::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState};
#[test]
fn refresh_increments_generation_only_when_fingerprint_changes() {
let mut state = TargetTlsState::default();
let first = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let second = TargetTlsFingerprint {
ca_sha256: Some([2; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
assert!(state.refresh(first.clone()));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(!state.refresh(first));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(state.refresh(second));
assert_eq!(state.generation, TargetTlsGeneration(2));
}
#[test]
fn reset_clears_generation_and_fingerprint() {
let mut state = TargetTlsState {
generation: TargetTlsGeneration(5),
fingerprint: Some(TargetTlsFingerprint {
ca_sha256: Some([9; 32]),
client_cert_sha256: None,
client_key_sha256: None,
}),
};
state.reset();
assert_eq!(state, TargetTlsState::default());
}
#[test]
fn fingerprint_eq_when_all_fields_match() {
let a = TargetTlsFingerprint {
ca_sha256: Some([42; 32]),
client_cert_sha256: Some([1; 32]),
client_key_sha256: None,
};
let b = TargetTlsFingerprint {
ca_sha256: Some([42; 32]),
client_cert_sha256: Some([1; 32]),
client_key_sha256: None,
};
assert_eq!(a, b);
}
#[test]
fn fingerprint_ne_when_ca_differs() {
let a = TargetTlsFingerprint {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let b = TargetTlsFingerprint {
ca_sha256: Some([2; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
assert_ne!(a, b);
}
}
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Target-level TLS reload metrics.
use ::metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
const TARGET_TLS_RELOAD_TOTAL: &str = "rustfs_target_tls_reload_total";
const TARGET_TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_target_tls_reload_skipped_total";
const TARGET_TLS_GENERATION: &str = "rustfs_target_tls_generation";
const TARGET_TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_target_tls_reload_duration_seconds";
const TARGET_TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_target_tls_publication_fail_total";
const TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL: &str = "rustfs_target_tls_active_generation_mismatch_total";
/// Describes all target TLS metrics. Call once during initialization.
pub fn init_target_tls_metrics() {
describe_counter!(TARGET_TLS_RELOAD_TOTAL, "Total number of TLS reload attempts per target");
describe_counter!(
TARGET_TLS_RELOAD_SKIPPED_TOTAL,
"Number of TLS reloads skipped per target (unchanged, etc.)"
);
describe_gauge!(TARGET_TLS_GENERATION, "Current TLS generation per target");
describe_histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "Duration of TLS reload attempts per target");
describe_counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "Number of TLS reload publication failures per target");
describe_counter!(
TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL,
"Number of times a target's active connection used a stale TLS generation"
);
}
/// Records a TLS reload result (success or failure).
pub fn record_target_tls_reload_result(target: &str, result: &str, duration_secs: f64, generation: u64) {
counter!(TARGET_TLS_RELOAD_TOTAL, "target_id" => target.to_string(), "result" => result.to_string()).increment(1);
histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "target_id" => target.to_string(), "result" => result.to_string())
.record(duration_secs);
gauge!(TARGET_TLS_GENERATION, "target_id" => target.to_string()).set(generation as f64);
}
/// Records a skipped reload (typically because fingerprint was unchanged).
pub fn record_target_tls_reload_skipped(target: &str, reason: &str) {
counter!(TARGET_TLS_RELOAD_SKIPPED_TOTAL, "target_id" => target.to_string(), "reason" => reason.to_string()).increment(1);
}
/// Records a TLS reload publication failure.
pub fn record_target_tls_publication_fail(target: &str) {
counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "target_id" => target.to_string()).increment(1);
}
/// Records that a target used a stale TLS generation (active ≠ latest published).
pub fn record_target_tls_stale_generation(target: &str) {
counter!(TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL, "target_id" => target.to_string()).increment(1);
}
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified TLS hot-reload infrastructure for notification targets.
//!
//! This module provides:
//! - Fingerprint-based change detection (`fingerprint`)
//! - Per-target reload configuration (`config`)
//! - Runtime state tracking with atomic timestamps (`state`)
//! - The `ReloadableTargetTls` trait protocol (`trait`)
//! - TLS material validation helpers (`validate`)
//! - The reload coordinator with background poll loops (`coordinator`)
//! - Target-level reload metrics (`metrics`)
pub mod adapter;
pub mod config;
pub mod coordinator;
pub mod fingerprint;
pub mod metrics;
pub mod state;
pub mod r#trait;
pub mod validate;
pub use adapter::TlsReloadAdapter;
pub use coordinator::TargetTlsReloadCoordinator;
pub use fingerprint::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState, build_target_tls_fingerprint};
pub use metrics::init_target_tls_metrics;
pub use state::{TargetTlsInputSet, TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
pub use r#trait::ReloadableTargetTls;
pub use validate::validate_tls_material;
+127
View File
@@ -0,0 +1,127 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Per-target TLS reload runtime state with atomic timestamps and error tracking.
use super::fingerprint::{TargetTlsFingerprint, TargetTlsGeneration};
use ::arc_swap::ArcSwap;
use serde::Serialize;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
/// Describes which TLS files a target reads.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TargetTlsInputSet {
pub ca_path: String,
pub client_cert_path: String,
pub client_key_path: String,
/// Human-readable label for logging and metrics (e.g. "webhook:primary").
pub target_label: String,
}
impl TargetTlsInputSet {
/// Returns `true` when no TLS paths are configured (no CA, cert, or key).
pub fn is_empty(&self) -> bool {
self.ca_path.is_empty() && self.client_cert_path.is_empty() && self.client_key_path.is_empty()
}
}
/// Immutable snapshot of a successfully published TLS material generation.
pub struct TargetTlsPublishedState<M> {
pub generation: TargetTlsGeneration,
pub fingerprint: TargetTlsFingerprint,
pub material: Arc<M>,
pub loaded_at_unix_ms: u64,
}
/// Per-target TLS reload runtime state. Owns the current published material
/// and tracks timestamps and the last error for observability.
pub struct TargetTlsRuntimeState<M> {
/// The currently active TLS material generation.
pub current: ArcSwap<TargetTlsPublishedState<M>>,
/// The last known-good generation (never overwritten by a failed reload).
pub last_good: ArcSwap<TargetTlsPublishedState<M>>,
/// Unix-millis timestamp of the last reload *attempt* (success or failure).
pub last_attempt_unix_ms: AtomicU64,
/// Unix-millis timestamp of the last *successful* reload.
pub last_success_unix_ms: AtomicU64,
/// Last reload error message, if any.
pub last_error: parking_lot::RwLock<Option<String>>,
/// The TLS file paths this state watches.
pub inputs: TargetTlsInputSet,
}
impl<M> TargetTlsRuntimeState<M> {
/// Creates a new runtime state with the given initial published state.
pub fn new(initial: Arc<TargetTlsPublishedState<M>>, inputs: TargetTlsInputSet) -> Self {
Self {
current: ArcSwap::from(initial.clone()),
last_good: ArcSwap::from(initial),
last_attempt_unix_ms: AtomicU64::new(0),
last_success_unix_ms: AtomicU64::new(0),
last_error: parking_lot::RwLock::new(None),
inputs,
}
}
/// Returns the generation of the currently active material.
pub fn current_generation(&self) -> TargetTlsGeneration {
self.current.load().generation
}
/// Atomically bumps and returns the next generation.
pub fn bump_generation(&self) -> TargetTlsGeneration {
// Load the current generation from the arc-swap, compute next,
// and return it. The caller is responsible for publishing the new state.
let current = self.current.load();
TargetTlsGeneration(current.generation.0.saturating_add(1))
}
/// Records the timestamp of a reload attempt.
pub fn mark_attempt(&self, unix_ms: u64) {
self.last_attempt_unix_ms.store(unix_ms, Ordering::Release);
}
/// Records the timestamp of a successful reload.
pub fn mark_success(&self, unix_ms: u64) {
self.last_success_unix_ms.store(unix_ms, Ordering::Release);
}
/// Returns the last attempt timestamp.
pub fn last_attempt_unix_ms(&self) -> u64 {
self.last_attempt_unix_ms.load(Ordering::Acquire)
}
/// Returns the last success timestamp.
pub fn last_success_unix_ms(&self) -> u64 {
self.last_success_unix_ms.load(Ordering::Acquire)
}
}
/// Read-only status snapshot for admin/debug visibility.
#[derive(Debug, Clone, Serialize)]
pub struct TargetTlsStatusSnapshot {
pub target_label: String,
pub generation: u64,
pub reload_enabled: bool,
pub detect_mode: &'static str,
pub apply_mode: &'static str,
pub last_attempt_time: Option<u64>,
pub last_success_time: Option<u64>,
pub last_error: Option<String>,
/// TLS file paths this target watches (for admin diagnostics).
pub ca_path: String,
pub client_cert_path: String,
pub client_key_path: String,
}
+68
View File
@@ -0,0 +1,68 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The `ReloadableTargetTls` trait — the public protocol each TLS-capable
//! target implements to participate in coordinated hot-reload.
use crate::error::TargetError;
use async_trait::async_trait;
use std::sync::Arc;
use super::config::ReloadApplyMode;
use super::fingerprint::TargetTlsGeneration;
use super::state::TargetTlsInputSet;
/// Protocol that each TLS-capable target implements so the reload coordinator
/// can drive certificate hot-reload without knowing the target's internals.
///
/// The target is responsible for:
/// - Declaring which TLS files it reads (`tls_input_set`)
/// - Building a new client/pool/connector from current files (`build_tls_material`)
/// - Atomically swapping the active connection state (`apply_tls_material`)
///
/// The coordinator is responsible for:
/// - Deciding *when* to check
/// - Detecting *whether* material changed
/// - Ensuring *safety* (validate, build-then-apply, fallback on failure)
#[async_trait]
pub trait ReloadableTargetTls: Send + Sync + 'static {
/// The rebuilt connection/client/pool object this target uses.
type Material: Send + Sync + 'static;
/// Returns the TLS file paths this target reads.
fn tls_input_set(&self) -> TargetTlsInputSet;
/// Build a fresh TLS material object from current files on disk.
///
/// Called by the coordinator on the reload path only — never on the send hot path.
async fn build_tls_material(&self) -> Result<Self::Material, TargetError>;
/// Atomically apply new TLS material, replacing the current active connection state.
///
/// On success, the target's internal state must point to the new material.
/// On failure, the target must keep its current state unchanged.
async fn apply_tls_material(
&self,
generation: TargetTlsGeneration,
material: Arc<Self::Material>,
mode: ReloadApplyMode,
) -> Result<(), TargetError>;
/// Optional pre-check: validate that TLS files on disk are self-consistent
/// (cert/key pair parseable, CA loadable) before attempting `build_tls_material`.
/// Default implementation returns `Ok(())`.
async fn validate_tls_files(&self) -> Result<(), TargetError> {
Ok(())
}
}
@@ -0,0 +1,58 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! TLS material validation helpers used by the reload coordinator before
//! attempting to build new client/pool objects.
use crate::error::TargetError;
use rustfs_tls_runtime::{load_certs, load_private_key};
/// Validates that a client certificate and private key file can be loaded
/// and paired together. Returns `Ok(())` if both files parse successfully,
/// or `Ok(())` if both paths are empty (no mTLS configured).
pub fn validate_cert_key_pairing(cert_path: &str, key_path: &str) -> Result<(), TargetError> {
if cert_path.is_empty() && key_path.is_empty() {
return Ok(());
}
if cert_path.is_empty() || key_path.is_empty() {
return Err(TargetError::Configuration(
"Client certificate and key must both be specified or both be empty".to_string(),
));
}
load_certs(cert_path).map_err(|e| TargetError::Configuration(format!("Invalid client certificate '{cert_path}': {e}")))?;
load_private_key(key_path).map_err(|e| TargetError::Configuration(format!("Invalid client key '{key_path}': {e}")))?;
Ok(())
}
/// Validates that a CA certificate file can be loaded. Returns `Ok(())`
/// if the path is empty (no custom CA) or if the file parses successfully.
pub fn validate_ca_file(ca_path: &str) -> Result<(), TargetError> {
if ca_path.is_empty() {
return Ok(());
}
load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Invalid CA certificate '{ca_path}': {e}")))?;
Ok(())
}
/// Validates all three TLS material files in one call.
pub fn validate_tls_material(ca_path: &str, cert_path: &str, key_path: &str) -> Result<(), TargetError> {
validate_ca_file(ca_path)?;
validate_cert_key_pairing(cert_path, key_path)
}
+104 -8
View File
@@ -22,11 +22,15 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload_with_records, is_connectivity_error, open_target_queue_store,
persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
open_target_queue_store, persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
@@ -37,6 +41,7 @@ use lapin::{
};
use parking_lot::Mutex;
use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fmt;
@@ -196,16 +201,24 @@ async fn build_tls_config(args: &AMQPArgs) -> Result<OwnedTLSConfig, TargetError
let cert_chain = if args.tls_ca.is_empty() {
None
} else {
Some(
tokio::fs::read_to_string(&args.tls_ca)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?,
)
let certs_der = load_cert_bundle_der_bytes(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CA}: {e}")))?;
if certs_der.is_empty() {
return Err(TargetError::Configuration(format!(
"{AMQP_TLS_CA} did not contain any parsable certificates"
)));
}
let pem = tokio::fs::read_to_string(&args.tls_ca)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?;
Some(pem)
};
let identity = if args.tls_client_cert.is_empty() {
None
} else {
let _ = load_cert_bundle_der_bytes(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CLIENT_CERT}: {e}")))?;
let pem = tokio::fs::read(&args.tls_client_cert)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_CERT}: {e}")))?;
@@ -290,6 +303,9 @@ where
id: TargetID,
args: AMQPArgs,
connection: Arc<Mutex<Option<Arc<AMQPConnection>>>>,
tls_state: Arc<Mutex<TargetTlsState>>,
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
tls_adapter: Option<TlsReloadAdapter<AMQPConnection>>,
connect_lock: Arc<AsyncMutex<()>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
delivery_counters: Arc<TargetDeliveryCounters>,
@@ -305,6 +321,8 @@ where
id: self.id.clone(),
args: self.args.clone(),
connection: Arc::clone(&self.connection),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
connect_lock: Arc::clone(&self.connect_lock),
store: self.store.as_ref().map(|s| s.boxed_clone()),
delivery_counters: Arc::clone(&self.delivery_counters),
@@ -329,6 +347,8 @@ where
id: target_id,
args,
connection: Arc::new(Mutex::new(None)),
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
tls_adapter: None,
connect_lock: Arc::new(AsyncMutex::new(())),
store: queue_store,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -341,6 +361,27 @@ where
}
async fn get_or_connect(&self) -> Result<Arc<AMQPConnection>, TargetError> {
// When a TLS reload adapter is attached, it drives connection rebuilds
// in the background. The inline per-send fingerprint check is skipped.
if let Some(adapter) = &self.tls_adapter {
let material = adapter.current_material();
if material.connection.status().connected() && material.channel.status().connected() {
return Ok(material);
}
self.clear_connection_handle();
} else {
let next_fingerprint =
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
self.clear_connection_handle();
self.tls_state.lock().refresh(next_fingerprint);
}
}
if let Some(connection) = self.connection.lock().clone()
&& connection.connection.status().connected()
&& connection.channel.status().connected()
@@ -362,10 +403,19 @@ where
Ok(connection)
}
fn clear_connection(&self) {
fn clear_connection_handle(&self) {
*self.connection.lock() = None;
}
fn clear_connection_cache(&self) {
self.clear_connection_handle();
self.tls_state.lock().reset();
}
fn clear_connection(&self) {
self.clear_connection_cache();
}
async fn send_body(&self, body: &[u8]) -> Result<(), TargetError> {
let connection = self.get_or_connect().await?;
let publish = connection
@@ -407,6 +457,46 @@ where
}
}
/// Coordinated TLS hot-reload implementation for AMQP targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the connection without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for AMQPTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = AMQPConnection;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("amqp:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
connect_amqp(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.connection.lock();
*guard = Some(material);
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[async_trait]
impl<E> Target<E> for AMQPTarget<E>
where
@@ -458,6 +548,12 @@ where
.await
.map_err(|e| map_lapin_error(e, "Failed to close AMQP connection"))?;
}
self.tls_state.lock().reset();
// If a TLS reload adapter is attached, reset its error tracking
// so that a future re-init does not inherit stale failure state.
if let Some(adapter) = &self.tls_adapter {
*adapter.runtime_state().last_error.write() = None;
}
info!(target_id = %self.id, "AMQP target closed");
Ok(())
}
+108 -2
View File
@@ -16,16 +16,21 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
validate::validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, open_target_queue_store,
persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
open_target_queue_store, persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{marker::PhantomData, sync::Arc, time::Duration};
@@ -104,6 +109,11 @@ where
args: KafkaArgs,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
tls_state: Arc<Mutex<TargetTlsState>>,
/// Adapter that bridges this target to the TLS reload coordinator.
/// When `Some`, the target uses coordinator-managed material; when `None`,
/// it falls back to inline fingerprint-based change detection.
tls_adapter: Option<TlsReloadAdapter<Arc<AsyncProducer>>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: PhantomData<E>,
}
@@ -144,6 +154,8 @@ where
args,
store: queue_store,
producer: Arc::new(Mutex::new(None)),
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
tls_adapter: None,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: PhantomData,
})
@@ -164,9 +176,27 @@ where
if self.args.tls_enable {
let mut security = SecurityConfig::new();
if !self.args.tls_ca.is_empty() {
let certs = load_cert_bundle_der_bytes(&self.args.tls_ca)
.map_err(|e| Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_ca"))?;
if certs.is_empty() {
return Err(TargetError::Configuration(
"Kafka tls_ca did not contain any parsable certificates".to_string(),
));
}
security = security.with_ca_cert(self.args.tls_ca.clone());
}
if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
let certs = load_cert_bundle_der_bytes(&self.args.tls_client_cert).map_err(|e| {
Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_cert")
})?;
if certs.is_empty() {
return Err(TargetError::Configuration(
"Kafka tls_client_cert did not contain any parsable certificates".to_string(),
));
}
let _ = load_private_key(&self.args.tls_client_key).map_err(|e| {
Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_key")
})?;
security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone());
}
config = config.with_security(security);
@@ -178,6 +208,31 @@ where
}
async fn get_or_build_producer(&self) -> Result<Arc<AsyncProducer>, TargetError> {
// Adapter-managed path: use the material directly from the TLS reload adapter.
if let Some(adapter) = &self.tls_adapter {
let producer: Arc<AsyncProducer> = (*adapter.current_material()).clone();
// Ensure the producer is also stored locally so that close() can drain it.
{
let mut guard = self.producer.lock().await;
*guard = Some(Arc::clone(&producer));
}
return Ok(producer);
}
// Inline fingerprint fallback path (no coordinator).
let next_fingerprint =
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock().await;
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
let mut cached = self.producer.lock().await;
*cached = None;
self.tls_state.lock().await.refresh(next_fingerprint);
}
let mut cached = self.producer.lock().await;
if let Some(producer) = cached.as_ref() {
return Ok(Arc::clone(producer));
@@ -191,6 +246,7 @@ where
async fn invalidate_cached_producer(&self) {
let mut cached = self.producer.lock().await;
*cached = None;
self.tls_state.lock().await.reset();
}
/// Serializes the event and builds a QueuedPayload
@@ -230,6 +286,8 @@ where
args: self.args.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
producer: Arc::clone(&self.producer),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: PhantomData,
})
@@ -292,6 +350,13 @@ where
}
async fn close(&self) -> Result<(), TargetError> {
{
let mut guard = self.producer.lock().await;
*guard = None;
}
self.tls_state.lock().await.reset();
info!("Kafka target closed: {}", self.id);
Ok(())
}
@@ -318,6 +383,47 @@ where
}
}
/// Coordinated TLS hot-reload implementation for Kafka targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the producer without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for KafkaTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Arc<AsyncProducer>;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("kafka:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
let producer = self.build_producer().await?;
Ok(Arc::new(producer))
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.producer.lock().await;
*guard = Some((*material).clone());
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[cfg(test)]
mod tests {
use super::*;
+49
View File
@@ -37,6 +37,13 @@ pub mod pulsar;
pub mod redis;
pub mod webhook;
#[cfg(test)]
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsFingerprint as TargetTlsFingerprintState;
#[cfg(test)]
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsGeneration;
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsState;
pub(crate) use crate::runtime::tls::fingerprint::build_target_tls_fingerprint;
/// A read-only snapshot of delivery counters for a target.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TargetDeliverySnapshot {
@@ -531,6 +538,48 @@ pub(crate) fn ensure_rustls_provider_installed() {
}
}
#[cfg(test)]
mod tls_state_tests {
use super::{TargetTlsFingerprintState, TargetTlsGeneration, TargetTlsState};
#[test]
fn refresh_increments_generation_only_when_fingerprint_changes() {
let mut state = TargetTlsState::default();
let first = TargetTlsFingerprintState {
ca_sha256: Some([1; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
let second = TargetTlsFingerprintState {
ca_sha256: Some([2; 32]),
client_cert_sha256: None,
client_key_sha256: None,
};
assert!(state.refresh(first.clone()));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(!state.refresh(first));
assert_eq!(state.generation, TargetTlsGeneration(1));
assert!(state.refresh(second));
assert_eq!(state.generation, TargetTlsGeneration(2));
}
#[test]
fn reset_clears_generation_and_fingerprint() {
let mut state = TargetTlsState {
generation: TargetTlsGeneration(5),
fingerprint: Some(TargetTlsFingerprintState {
ca_sha256: Some([9; 32]),
client_cert_sha256: None,
client_key_sha256: None,
}),
};
state.reset();
assert_eq!(state, TargetTlsState::default());
}
}
#[cfg(test)]
mod tests {
use super::*;
+96 -15
View File
@@ -16,6 +16,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TargetTlsState, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -23,6 +27,7 @@ use crate::{
persist_queued_payload_to_store,
},
};
use arc_swap::ArcSwap;
use async_trait::async_trait;
use hyper_rustls::ConfigBuilderExt;
use rumqttc::{
@@ -32,6 +37,7 @@ use rumqttc::{
use rustfs_config::{
EnableState, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_WS_PATH_ALLOWLIST,
};
use rustfs_tls_runtime::{load_certs, load_private_key};
use rustls::ClientConfig;
use serde::Serialize;
use serde::de::DeserializeOwned;
@@ -185,8 +191,7 @@ fn validate_path_is_absolute(path: &str, field: &str) -> Result<(), TargetError>
}
fn build_root_store(ca_path: &str, trust_leaf_as_ca: bool) -> Result<rustls::RootCertStore, TargetError> {
let certs =
rustfs_utils::load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
let certs = load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
let mut store = rustls::RootCertStore::empty();
if trust_leaf_as_ca {
@@ -222,9 +227,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
if tls.client_cert_path.is_empty() {
builder.with_no_client_auth()
} else {
let certs = rustfs_utils::load_certs(&tls.client_cert_path)
let certs = load_certs(&tls.client_cert_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
let key = rustfs_utils::load_private_key(&tls.client_key_path)
let key = load_private_key(&tls.client_key_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
builder
.with_client_auth_cert(certs, key)
@@ -237,9 +242,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
if tls.client_cert_path.is_empty() {
builder.with_no_client_auth()
} else {
let certs = rustfs_utils::load_certs(&tls.client_cert_path)
let certs = load_certs(&tls.client_cert_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
let key = rustfs_utils::load_private_key(&tls.client_key_path)
let key = load_private_key(&tls.client_key_path)
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
builder
.with_client_auth_cert(certs, key)
@@ -490,6 +495,12 @@ where
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: Arc<AtomicBool>,
bg_task_manager: Arc<BgTaskManager>,
/// TLS fingerprint tracking for inline fallback path.
tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
tls_adapter: Option<TlsReloadAdapter<MqttOptions>>,
/// Updated MqttOptions from coordinator for use on next reconnection.
pending_mqtt_options: Arc<ArcSwap<MqttOptions>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: PhantomData<E>,
}
@@ -519,6 +530,17 @@ where
initial_cancel_rx: Mutex::new(Some(cancel_rx)),
});
// Build the initial MqttOptions for TLS reload support.
let initial_mqtt_options = build_mqtt_options(
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
&args.broker,
Some(args.username.as_str()),
Some(args.password.as_str()),
&args.tls,
args.keep_alive,
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?;
info!(target_id = %target_id, "MQTT target created");
Ok(MQTTTarget::<E> {
id: target_id,
@@ -527,6 +549,9 @@ where
store: queue_store,
connected: Arc::new(AtomicBool::new(false)),
bg_task_manager,
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
tls_adapter: None,
pending_mqtt_options: Arc::new(ArcSwap::from(Arc::new(initial_mqtt_options))),
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: PhantomData,
})
@@ -544,20 +569,15 @@ where
let connected_arc = Arc::clone(&self.connected);
let target_id_clone = self.id.clone();
let args_clone = self.args.clone();
let pending_mqtt_options = Arc::clone(&self.pending_mqtt_options);
let _ = bg_task_manager
.init_cell
.get_or_try_init(|| async {
debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
let mqtt_options = build_mqtt_options(
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
&args_clone.broker,
Some(args_clone.username.as_str()),
Some(args_clone.password.as_str()),
&args_clone.tls,
args_clone.keep_alive,
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?;
// Use the latest MqttOptions (may have been updated by TLS reload coordinator).
let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
@@ -662,12 +682,66 @@ where
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: self.connected.clone(),
bg_task_manager: self.bg_task_manager.clone(),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
pending_mqtt_options: Arc::clone(&self.pending_mqtt_options),
delivery_counters: self.delivery_counters.clone(),
_phantom: PhantomData,
})
}
}
/// Coordinated TLS hot-reload implementation for MQTT targets.
///
/// MQTT uses `MqttOptions` as the material type. The coordinator rebuilds
/// `MqttOptions` on TLS file changes, and `apply_tls_material` stores it in
/// an `ArcSwap` for use on the next reconnection. The running event loop is
/// not interrupted; rumqttc handles reconnection internally.
#[async_trait]
impl<E> ReloadableTargetTls for MQTTTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = MqttOptions;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls.ca_path.clone(),
client_cert_path: self.args.tls.client_cert_path.clone(),
client_key_path: self.args.tls.client_key_path.clone(),
target_label: format!("mqtt:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_mqtt_options(
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
&self.args.broker,
Some(self.args.username.as_str()),
Some(self.args.password.as_str()),
&self.args.tls,
self.args.keep_alive,
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
// Store the new MqttOptions for use on next reconnection.
// The running event loop is not interrupted; rumqttc handles reconnection.
self.pending_mqtt_options.store(material);
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
}
}
async fn run_mqtt_event_loop(
mut eventloop: EventLoop,
connected_status: Arc<AtomicBool>,
@@ -968,6 +1042,13 @@ where
}
}
self.tls_state.lock().reset();
// If a TLS reload adapter is attached, reset its error tracking
// so that a future re-init does not inherit stale failure state.
if let Some(adapter) = &self.tls_adapter {
*adapter.runtime_state().last_error.write() = None;
}
self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "MQTT target close method finished.");
Ok(())
+162 -62
View File
@@ -16,6 +16,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
validate::validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -26,6 +30,7 @@ use crate::{
use async_trait::async_trait;
use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable};
use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY};
use rustfs_tls_runtime::{load_certs, load_private_key};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::marker::PhantomData;
@@ -478,6 +483,11 @@ where
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
/// Lazily-initialized MySQL connection pool
pool: Arc<Mutex<Option<Pool>>>,
/// TLS fingerprint tracking for hot reload (inline fallback path)
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
/// When present, the adapter provides coordinator-managed TLS material;
/// otherwise the inline fingerprint path is used as a fallback.
tls_adapter: Option<TlsReloadAdapter<Pool>>,
/// Success/failure counters exposed via `delivery_snapshot`
delivery_counters: Arc<TargetDeliveryCounters>,
/// Zero-sized marker for the event type `E`
@@ -489,6 +499,9 @@ where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
/// Creates a new MySqlTarget.
///
/// The target starts without a TLS reload coordinator. Use
/// `TlsReloadAdapter::try_register` to opt into coordinated TLS hot-reload.
pub fn new(id: String, args: MySqlArgs) -> Result<Self, TargetError> {
args.validate()?;
@@ -511,6 +524,8 @@ where
store: queue_store,
// Pool is lazily initialized on first use to avoid unnecessary connections at startup and allow for better error handling
pool: Arc::new(Mutex::new(None)),
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
tls_adapter: None,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: PhantomData,
})
@@ -518,6 +533,10 @@ where
/// Returns or lazily initializes the MySQL connection pool.
///
/// When `tls_adapter` is present (coordinator-managed), the pool
/// is sourced from the coordinator's published material.
/// Otherwise, the inline fingerprint-based path is used as a fallback.
///
/// # Errors
///
/// | Scenario | Error variant |
@@ -528,6 +547,31 @@ where
/// | Existing table has incompatible schema | `Initialization` |
/// | DSN parse failure / invalid config | `Configuration` |
async fn get_or_init_pool(&self) -> Result<Pool, TargetError> {
// Adapter-managed path: use the material directly from the coordinator.
if let Some(adapter) = &self.tls_adapter {
let pool: Pool = (*adapter.current_material()).clone();
// Ensure the pool is also stored locally so that close() can drain it.
{
let mut guard = self.pool.lock().await;
*guard = Some(pool.clone());
}
return Ok(pool);
}
// Inline fingerprint fallback path (no coordinator).
let next_fingerprint =
super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
let mut guard = self.pool.lock().await;
*guard = None;
self.tls_state.lock().refresh(next_fingerprint);
}
{
let guard = self.pool.lock().await;
if let Some(pool) = guard.as_ref() {
@@ -535,68 +579,7 @@ where
}
}
let dsn = MySqlDsn::parse(&self.args.dsn_string)?;
let mut builder = OptsBuilder::default()
.user(Some(dsn.user.clone()))
.pass(Some(dsn.password.clone()))
.ip_or_hostname(dsn.host.clone())
.tcp_port(dsn.port)
.db_name(Some(dsn.database.clone()));
if dsn.tls {
super::ensure_rustls_provider_installed();
let mut ssl_opts = SslOpts::default();
if !self.args.tls_ca.is_empty() {
ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(self.args.tls_ca.clone()).into()]);
}
if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
let identity = mysql_async::ClientIdentity::new(
PathBuf::from(self.args.tls_client_cert.clone()).into(),
PathBuf::from(self.args.tls_client_key.clone()).into(),
);
ssl_opts = ssl_opts.with_client_identity(Some(identity));
}
builder = builder.ssl_opts(Some(ssl_opts));
} else {
warn!(
"MySQL target '{}' is configured without TLS. This is insecure and should not be used in production.",
self.id
);
}
// When max_open_connections is 0, no explicit upper bound is set —
// mysql_async uses its default pool constraints (10100).
if self.args.max_open_connections > 0 {
let constraints = PoolConstraints::new(1, self.args.max_open_connections).ok_or_else(|| {
TargetError::Configuration(format!(
"MySQL max_open_connections must be >= 1, got {}",
self.args.max_open_connections
))
})?;
builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
}
let opts = Opts::from(builder);
let pool = Pool::new(opts);
// Uses a double-check pattern: the mutex guard is only held for
// short reads/writes to the pool cache. All I/O (connecting,
// DDL, schema validation) happens outside the lock so that
// concurrent callers are not blocked by a slow MySQL server.
let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
let ddl = format!(
"CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
quote_table_name(&self.args.table)?
);
conn.query_drop(ddl)
.await
.map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
validate_existing_schema(&mut conn, &self.args.table).await?;
let pool = build_mysql_pool_from_args(&self.args).await?;
// Double-check: another caller may have initialized the pool
// while we were doing I/O.
@@ -653,12 +636,87 @@ where
args: self.args.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
pool: Arc::clone(&self.pool),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: PhantomData,
})
}
}
/// Builds a MySQL connection pool from the given args, including TLS setup,
/// DDL table creation, and schema validation.
///
/// This is a standalone function so it can be called both from
/// `get_or_init_pool` (inline fallback) and from `build_tls_material`
/// (coordinator path).
async fn build_mysql_pool_from_args(args: &MySqlArgs) -> Result<Pool, TargetError> {
let dsn = MySqlDsn::parse(&args.dsn_string)?;
let mut builder = OptsBuilder::default()
.user(Some(dsn.user.clone()))
.pass(Some(dsn.password.clone()))
.ip_or_hostname(dsn.host.clone())
.tcp_port(dsn.port)
.db_name(Some(dsn.database.clone()));
if dsn.tls {
super::ensure_rustls_provider_installed();
let mut ssl_opts = SslOpts::default();
if !args.tls_ca.is_empty() {
let _ =
load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_ca: {e}")))?;
ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(args.tls_ca.clone()).into()]);
}
if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
let _ = load_certs(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_cert: {e}")))?;
let _ = load_private_key(&args.tls_client_key)
.map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_key: {e}")))?;
let identity = mysql_async::ClientIdentity::new(
PathBuf::from(args.tls_client_cert.clone()).into(),
PathBuf::from(args.tls_client_key.clone()).into(),
);
ssl_opts = ssl_opts.with_client_identity(Some(identity));
}
builder = builder.ssl_opts(Some(ssl_opts));
} else {
warn!("MySQL target is configured without TLS. This is insecure and should not be used in production.");
}
// When max_open_connections is 0, no explicit upper bound is set —
// mysql_async uses its default pool constraints (10100).
if args.max_open_connections > 0 {
let constraints = PoolConstraints::new(1, args.max_open_connections).ok_or_else(|| {
TargetError::Configuration(format!("MySQL max_open_connections must be >= 1, got {}", args.max_open_connections))
})?;
builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
}
let opts = Opts::from(builder);
let pool = Pool::new(opts);
// Uses a double-check pattern: the mutex guard is only held for
// short reads/writes to the pool cache. All I/O (connecting,
// DDL, schema validation) happens outside the lock so that
// concurrent callers are not blocked by a slow MySQL server.
let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
let ddl = format!(
"CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
quote_table_name(&args.table)?
);
conn.query_drop(ddl)
.await
.map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
validate_existing_schema(&mut conn, &args.table).await?;
Ok(pool)
}
/// Maps a mysql_async error to `TargetError`:
/// - `Io`/`Driver` → `NotConnected` (connection lost, fixed-delay retry)
/// - `Server(1213|1205|1040)` → `Timeout` (deadlock/lock timeout/too
@@ -793,6 +851,8 @@ where
.map_err(|err| TargetError::Network(format!("Failed to disconnect MySQL pool: {err}")))?;
}
// Adapter cleanup is done by the coordinator; no local state to reset.
info!("MySQL target closed: {}", self.id);
Ok(())
}
@@ -828,6 +888,46 @@ where
}
}
/// Coordinated TLS hot-reload implementation for MySQL targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the connection pool without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for MySqlTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Pool;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("mysql:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_mysql_pool_from_args(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.pool.lock().await;
*guard = Some((*material).clone());
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[cfg(test)]
mod tests {
use super::*;
+143 -9
View File
@@ -16,10 +16,15 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
@@ -28,9 +33,10 @@ use serde::Serialize;
use serde::de::DeserializeOwned;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use tracing::{info, instrument};
use tokio::sync::Mutex;
use tracing::{info, instrument, warn};
#[derive(Debug, Clone)]
pub struct NATSArgs {
@@ -168,7 +174,12 @@ where
{
id: TargetID,
args: NATSArgs,
client: Mutex<Option<async_nats::Client>>,
client: Arc<Mutex<Option<async_nats::Client>>>,
tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
/// Adapter that bridges this target to the TLS reload coordinator.
/// When `Some`, the target uses coordinator-managed material; when `None`,
/// it falls back to inline fingerprint-based change detection.
tls_adapter: Option<TlsReloadAdapter<async_nats::Client>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: AtomicBool,
delivery_counters: Arc<TargetDeliveryCounters>,
@@ -183,7 +194,9 @@ where
Box::new(NATSTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
client: Mutex::new(self.client.lock().unwrap().clone()),
client: Arc::clone(&self.client),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
delivery_counters: Arc::clone(&self.delivery_counters),
@@ -207,7 +220,9 @@ where
Ok(Self {
id: target_id,
args,
client: Mutex::new(None),
client: Arc::new(Mutex::new(None)),
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
tls_adapter: None,
store: queue_store,
connected: AtomicBool::new(false),
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -215,11 +230,42 @@ where
})
}
async fn invalidate_cached_client_connection(&self) {
*self.client.lock().await = None;
}
async fn get_or_connect(&self) -> Result<async_nats::Client, TargetError> {
if let Some(client) = self.client.lock().unwrap().clone() {
// Adapter-managed path: use the material directly from the TLS reload adapter.
if let Some(adapter) = &self.tls_adapter {
let client: async_nats::Client = (*adapter.current_material()).clone();
// Ensure the client is also stored locally so that close() can drain it.
{
let mut guard = self.client.lock().await;
*guard = Some(client.clone());
}
return Ok(client);
}
// Inline fingerprint fallback path (no coordinator).
let next_fingerprint =
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
self.invalidate_cached_client_connection().await;
self.tls_state.lock().refresh(next_fingerprint);
}
{
let guard = self.client.lock().await;
if let Some(client) = guard.as_ref() {
return Ok(client.clone());
}
}
let client = connect_nats(&self.args).await?;
client
.flush()
@@ -227,7 +273,7 @@ where
.map_err(|e| TargetError::Network(format!("Failed to flush NATS connection: {e}")))?;
self.connected.store(true, Ordering::SeqCst);
let mut guard = self.client.lock().unwrap();
let mut guard = self.client.lock().await;
let shared = guard.get_or_insert_with(|| client.clone()).clone();
Ok(shared)
}
@@ -294,7 +340,11 @@ where
}
async fn close(&self) -> Result<(), TargetError> {
let client = self.client.lock().unwrap().take();
let client = {
let mut guard = self.client.lock().await;
guard.take()
};
self.tls_state.lock().reset();
self.connected.store(false, Ordering::SeqCst);
if let Some(client) = client {
client
@@ -335,3 +385,87 @@ where
self.delivery_counters.record_final_failure();
}
}
/// Coordinated TLS hot-reload implementation for NATS targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the NATS client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for NATSTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = async_nats::Client;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("nats:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
connect_nats(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
let mut guard = self.client.lock().await;
*guard = Some((*material).clone());
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn base_args() -> NATSArgs {
NATSArgs {
enable: true,
address: "nats://127.0.0.1:4222".to_string(),
subject: "rustfs.events".to_string(),
username: String::new(),
password: String::new(),
token: String::new(),
credentials_file: String::new(),
tls_ca: String::new(),
tls_client_cert: String::new(),
tls_client_key: String::new(),
tls_required: false,
queue_dir: String::new(),
queue_limit: 0,
target_type: TargetType::NotifyEvent,
}
}
#[test]
fn validate_nats_rejects_multiple_auth_methods() {
let args = NATSArgs {
token: "abc".to_string(),
username: "user".to_string(),
password: "pass".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
#[test]
fn validate_nats_rejects_relative_queue_dir() {
let args = NATSArgs {
queue_dir: "relative/path".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
}
+86 -27
View File
@@ -29,6 +29,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -38,12 +42,10 @@ use crate::{
use async_trait::async_trait;
use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod};
use rustfs_config::{POSTGRES_DSN_STRING, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY};
use rustls_pki_types::pem::PemObject;
use rustls_pki_types::{CertificateDer, PrivateKeyDer};
use rustfs_tls_runtime::{load_certs, load_private_key};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fmt;
use std::io::BufReader;
use std::path::Path;
use std::sync::Arc;
use tokio_postgres::Config;
@@ -425,11 +427,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
let _ = root_store.add(cert);
}
} else {
let pem = std::fs::read(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CA}: {e}")))?;
let mut reader = BufReader::new(pem.as_slice());
for cert in CertificateDer::pem_reader_iter(&mut reader) {
let cert = cert.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
let certs =
load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
for cert in certs {
root_store
.add(cert)
.map_err(|e| TargetError::Configuration(format!("failed to add CA cert: {e}")))?;
@@ -439,16 +439,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
let builder = rustls::ClientConfig::builder().with_root_certificates(root_store);
let client_config = if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
let cert_pem = std::fs::read(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
let key_pem = std::fs::read(&args.tls_client_key)
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
let certs: Vec<_> = CertificateDer::pem_reader_iter(&mut BufReader::new(cert_pem.as_slice()))
.collect::<Result<_, _>>()
let certs = load_certs(&args.tls_client_cert)
.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
let key = PrivateKeyDer::from_pem_reader(&mut BufReader::new(key_pem.as_slice()))
let key = load_private_key(&args.tls_client_key)
.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
builder
@@ -548,13 +541,22 @@ fn resolve_payload_key(payload: &serde_json::Value, meta: &QueuedPayloadMeta) ->
/// so that `clone_box` does not duplicate connection state. The optional
/// `QueueStore` provides at-least-once delivery semantics consistent with the
/// other built-in targets.
///
/// When `tls_adapter` is `Some`, the target participates in the
/// coordinated TLS hot-reload system driven by `TlsReloadAdapter`,
/// and the inline fingerprint check in `send_body` is skipped. When `None`,
/// the legacy inline fingerprint check is used as a fallback.
pub struct PostgresTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
id: TargetID,
args: PostgresArgs,
pool: Pool,
pool: Arc<parking_lot::Mutex<Pool>>,
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
/// When present, the adapter provides coordinator-managed TLS material;
/// otherwise the inline fingerprint path is used as a fallback.
tls_adapter: Option<TlsReloadAdapter<Pool>>,
namespace_sql: String,
access_sql: String,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
@@ -570,7 +572,9 @@ where
Box::new(PostgresTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
pool: self.pool.clone(),
pool: Arc::clone(&self.pool),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
namespace_sql: self.namespace_sql.clone(),
access_sql: self.access_sql.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
@@ -599,7 +603,9 @@ where
namespace_sql: namespace_upsert_sql(&args.schema, &args.table),
access_sql: access_insert_sql(&args.schema, &args.table),
args,
pool,
pool: Arc::new(parking_lot::Mutex::new(pool)),
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
tls_adapter: None,
store: queue_store,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: std::marker::PhantomData,
@@ -611,8 +617,25 @@ where
/// Identifier validation has already happened in `PostgresArgs::validate()`,
/// so `qualified_table` cannot produce a malformed SQL string here.
async fn send_body(&self, body: &[u8], event_id: &str, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
let client = self
.pool
// When a TLS reload adapter is attached, it drives pool rebuilds in
// the background. The inline per-send fingerprint check is skipped.
if self.tls_adapter.is_none() {
let next_fingerprint =
super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
.await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
};
if tls_changed {
let new_pool = build_pool(&self.args)?;
*self.pool.lock() = new_pool;
self.tls_state.lock().refresh(next_fingerprint);
}
}
let pool = self.pool.lock().clone();
let client = pool
.get()
.await
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
@@ -645,8 +668,8 @@ where
/// Probes the table from `init()`. Failure is non-fatal when a queue is
/// configured: events buffer in the store until the schema is fixed.
async fn probe_table(&self) -> Result<(), TargetError> {
let client = self
.pool
let pool = self.pool.lock().clone();
let client = pool
.get()
.await
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed during init probe"))?;
@@ -659,6 +682,41 @@ where
}
}
#[async_trait]
impl<E> ReloadableTargetTls for PostgresTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Pool;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: self.args.tls_client_cert.clone(),
client_key_path: self.args.tls_client_key.clone(),
target_label: format!("postgres:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_pool(&self.args)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
*self.pool.lock() = (*material).clone();
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
}
}
#[async_trait]
impl<E> Target<E> for PostgresTarget<E>
where
@@ -674,8 +732,8 @@ where
}
match tokio::time::timeout(std::time::Duration::from_secs(10), async {
let client = self
.pool
let pool = self.pool.lock().clone();
let client = pool
.get()
.await
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
@@ -728,7 +786,8 @@ where
}
async fn close(&self) -> Result<(), TargetError> {
self.pool.close();
self.pool.lock().close();
// Adapter cleanup is done by the coordinator; no local state to reset.
info!(target_id = %self.id, "PostgreSQL target closed");
Ok(())
}
+148 -2
View File
@@ -16,14 +16,20 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
use pulsar::{Authentication, Producer, Pulsar, TokioExecutor};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::path::Path;
@@ -136,6 +142,13 @@ pub async fn connect_pulsar(args: &PulsarArgs) -> Result<Pulsar<TokioExecutor>,
}
if !args.tls_ca.is_empty() {
let certs = load_cert_bundle_der_bytes(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to parse Pulsar tls_ca: {e}")))?;
if certs.is_empty() {
return Err(TargetError::Configuration(
"Pulsar tls_ca did not contain any parsable certificates".to_string(),
));
}
builder = builder
.with_certificate_chain_file(&args.tls_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to load Pulsar tls_ca: {e}")))?;
@@ -158,6 +171,9 @@ where
id: TargetID,
args: PulsarArgs,
client: Mutex<Option<Pulsar<TokioExecutor>>>,
tls_state: Mutex<TargetTlsState>,
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
tls_adapter: Option<TlsReloadAdapter<Pulsar<TokioExecutor>>>,
producer: AsyncMutex<Option<Producer<TokioExecutor>>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: AtomicBool,
@@ -174,6 +190,8 @@ where
id: self.id.clone(),
args: self.args.clone(),
client: Mutex::new(self.client.lock().unwrap().clone()),
tls_state: Mutex::new(self.tls_state.lock().unwrap().clone()),
tls_adapter: self.tls_adapter.clone(),
producer: AsyncMutex::new(None),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
@@ -199,6 +217,8 @@ where
id: target_id,
args,
client: Mutex::new(None),
tls_state: Mutex::new(TargetTlsState::default()),
tls_adapter: None,
producer: AsyncMutex::new(None),
store: queue_store,
connected: AtomicBool::new(false),
@@ -207,7 +227,36 @@ where
})
}
fn clear_cached_client_connection(&self) {
self.client.lock().unwrap().take();
}
fn clear_cached_client(&self) {
self.clear_cached_client_connection();
self.tls_state.lock().unwrap().reset();
}
async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
// When a TLS reload adapter is attached, it drives client rebuilds
// in the background. The inline per-send fingerprint check is skipped.
if let Some(adapter) = &self.tls_adapter {
let material = adapter.current_material();
{
let mut guard = self.client.lock().unwrap();
*guard = Some((*material).clone());
}
} else {
let next_fingerprint = build_target_tls_fingerprint(&self.args.tls_ca, "", "").await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock().unwrap();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
self.clear_cached_client_connection();
self.tls_state.lock().unwrap().refresh(next_fingerprint);
}
}
if let Some(client) = self.client.lock().unwrap().clone() {
return Ok(client);
}
@@ -262,6 +311,56 @@ where
}
}
/// Coordinated TLS hot-reload implementation for Pulsar targets.
///
/// Pulsar only uses a CA certificate (no client cert/key).
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for PulsarTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Pulsar<TokioExecutor>;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls_ca.clone(),
client_cert_path: String::new(),
client_key_path: String::new(),
target_label: format!("pulsar:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
connect_pulsar(&self.args).await
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
// Pulsar client is Clone, so we clone from the Arc and store it.
{
let mut guard = self.client.lock().unwrap();
*guard = Some((*material).clone());
}
// Producer is bound to the old client; clear it so next send rebuilds.
{
let mut producer = self.producer.lock().await;
*producer = None;
}
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
// Pulsar only uses CA, no client cert/key.
validate_tls_material(&self.args.tls_ca, "", "")
}
}
#[async_trait]
impl<E> Target<E> for PulsarTarget<E>
where
@@ -321,8 +420,13 @@ where
.map_err(|e| TargetError::Network(format!("Failed to close Pulsar producer: {e}")))?;
}
*producer = None;
self.client.lock().unwrap().take();
self.clear_cached_client();
self.connected.store(false, Ordering::SeqCst);
// If a TLS reload adapter is attached, reset its error tracking
// so that a future re-init does not inherit stale failure state.
if let Some(adapter) = &self.tls_adapter {
*adapter.runtime_state().last_error.write() = None;
}
info!(target_id = %self.id, "Pulsar target closed");
Ok(())
}
@@ -355,3 +459,45 @@ where
self.delivery_counters.record_final_failure();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn base_args() -> PulsarArgs {
PulsarArgs {
enable: true,
broker: "pulsar://127.0.0.1:6650".to_string(),
topic: "persistent://public/default/rustfs-events".to_string(),
auth_token: String::new(),
username: String::new(),
password: String::new(),
tls_ca: String::new(),
tls_allow_insecure: false,
tls_hostname_verification: true,
queue_dir: String::new(),
queue_limit: 0,
target_type: TargetType::NotifyEvent,
}
}
#[test]
fn validate_pulsar_rejects_mixed_auth_methods() {
let args = PulsarArgs {
auth_token: "token".to_string(),
username: "user".to_string(),
password: "pass".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
#[test]
fn validate_pulsar_rejects_relative_queue_dir() {
let args = PulsarArgs {
queue_dir: "relative/path".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
}
+116 -9
View File
@@ -16,6 +16,10 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
@@ -31,9 +35,12 @@ use redis::{
io::tcp::{TcpSettings, socket2},
};
use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY};
use rustls::pki_types::CertificateDer;
use rustls::pki_types::pem::PemObject;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fmt;
use std::io::BufReader;
use std::path::Path;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
@@ -297,7 +304,8 @@ where
{
id: TargetID,
args: RedisArgs,
publisher_client: Client,
/// Redis client, wrapped in a lock so TLS hot-reload can atomically replace it.
publisher_client: Arc<parking_lot::Mutex<Client>>,
publisher: Arc<Mutex<Option<ConnectionManager>>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
/// Business-level liveness flag.
@@ -306,6 +314,12 @@ where
/// publish exhausted retries, or the target was explicitly closed). Temporary reconnectable
/// errors only invalidate the cached publisher so that a later request can lazily rebuild it.
connected: Arc<AtomicBool>,
/// TLS fingerprint tracking for hot reload (inline fallback path).
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
/// Adapter that bridges this target to the TLS reload coordinator.
/// When `Some`, the target uses coordinator-managed material; when `None`,
/// it falls back to inline fingerprint-based change detection.
tls_adapter: Option<TlsReloadAdapter<Client>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: std::marker::PhantomData<E>,
}
@@ -334,10 +348,12 @@ where
Ok(Self {
id: target_id,
args,
publisher_client,
publisher_client: Arc::new(parking_lot::Mutex::new(publisher_client)),
publisher: Arc::new(Mutex::new(None)),
store: queue_store,
connected: Arc::new(AtomicBool::new(false)),
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
tls_adapter: None,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: std::marker::PhantomData,
})
@@ -347,23 +363,61 @@ where
Box::new(Self {
id: self.id.clone(),
args: self.args.clone(),
publisher_client: self.publisher_client.clone(),
publisher_client: Arc::clone(&self.publisher_client),
publisher: Arc::clone(&self.publisher),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: Arc::clone(&self.connected),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: std::marker::PhantomData,
})
}
async fn get_or_create_publisher(&self) -> Result<ConnectionManager, TargetError> {
// Adapter-managed path: use the material directly from the TLS reload adapter.
if let Some(adapter) = &self.tls_adapter {
let client: Client = (*adapter.current_material()).clone();
// Ensure the client is also stored locally so close() can drain it.
*self.publisher_client.lock() = client.clone();
let manager = client
.get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
.map_err(map_redis_error)?;
*self.publisher.lock().await = Some(manager.clone());
return Ok(manager);
}
// Inline fingerprint fallback path (no coordinator).
let secure_scheme = matches!(self.args.url.scheme(), "rediss" | "valkeys");
if secure_scheme {
let next_fingerprint = super::build_target_tls_fingerprint(
&self.args.tls.ca_path,
&self.args.tls.client_cert_path,
&self.args.tls.client_key_path,
)
.await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.needs_update(&next_fingerprint)
};
if tls_changed {
let new_client = build_redis_client(&self.args)?;
*self.publisher_client.lock() = new_client;
self.invalidate_cached_publisher().await;
self.tls_state.lock().refresh(next_fingerprint);
}
}
let mut guard = self.publisher.lock().await;
if let Some(manager) = guard.clone() {
return Ok(manager);
}
let manager = self
.publisher_client
let client = self.publisher_client.lock().clone();
let manager = client
.get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
.map_err(map_redis_error)?;
@@ -475,7 +529,8 @@ where
return Ok(false);
}
match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&self.publisher_client, &self.args)).await {
let client = self.publisher_client.lock().clone();
match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&client, &self.args)).await {
Ok(Ok(())) => {
self.connected.store(true, Ordering::SeqCst);
Ok(true)
@@ -557,6 +612,7 @@ where
async fn close(&self) -> Result<(), TargetError> {
self.invalidate_cached_publisher().await;
self.tls_state.lock().reset();
self.connected.store(false, Ordering::SeqCst);
info!(target_id = %self.id, "Redis target closed");
Ok(())
@@ -696,9 +752,20 @@ fn read_root_cert(tls: &RedisTlsConfig) -> Result<Option<Vec<u8>>, TargetError>
return Ok(None);
}
std::fs::read(&tls.ca_path)
.map(Some)
.map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))
let pem =
std::fs::read(&tls.ca_path).map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))?;
let mut reader = BufReader::new(pem.as_slice());
let certs_der = CertificateDer::pem_reader_iter(&mut reader)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| TargetError::Configuration(format!("Failed to parse Redis root CA cert: {e}")))?;
if certs_der.is_empty() {
return Err(TargetError::Configuration(
"Redis root CA cert did not contain any parsable certificates".to_string(),
));
}
Ok(Some(pem))
}
fn map_redis_error(err: RedisError) -> TargetError {
@@ -722,6 +789,46 @@ fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration)
min_delay.saturating_mul(factor).min(max_delay)
}
/// Coordinated TLS hot-reload implementation for Redis targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the Redis client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for RedisTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Client;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.tls.ca_path.clone(),
client_cert_path: self.args.tls.client_cert_path.clone(),
client_key_path: self.args.tls.client_key_path.clone(),
target_label: format!("redis:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
build_redis_client(&self.args)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
*self.publisher_client.lock() = (*material).clone();
self.invalidate_cached_publisher().await;
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
}
}
#[cfg(test)]
mod tests {
use super::*;
+105 -10
View File
@@ -16,14 +16,21 @@ use crate::{
StoreError, Target,
arn::TargetID,
error::TargetError,
runtime::tls::{
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
validate::validate_tls_material,
},
store::{Key, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store,
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, open_target_queue_store,
persist_queued_payload_to_store,
},
};
use async_trait::async_trait;
use parking_lot::Mutex;
use reqwest::{Client, StatusCode, Url};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{
@@ -104,7 +111,11 @@ where
id: TargetID,
args: WebhookArgs,
health_check_url: Option<Url>,
http_client: Arc<Client>,
http_client: Arc<Mutex<Client>>,
tls_state: Arc<Mutex<TargetTlsState>>,
/// When present, the adapter provides coordinator-managed TLS material;
/// otherwise the inline fingerprint path is used as a fallback.
tls_adapter: Option<TlsReloadAdapter<Client>>,
// Add Send + Sync constraints to ensure thread safety
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
initialized: AtomicBool,
@@ -124,6 +135,8 @@ where
args: self.args.clone(),
health_check_url: self.health_check_url.clone(),
http_client: Arc::clone(&self.http_client),
tls_state: Arc::clone(&self.tls_state),
tls_adapter: self.tls_adapter.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
cancel_sender: self.cancel_sender.clone(),
@@ -146,7 +159,7 @@ where
};
// Build HTTP client using the helper function
let http_client = Arc::new(Self::build_http_client(&args)?);
let http_client = Arc::new(Mutex::new(Self::build_http_client(&args)?));
let queue_store = open_target_queue_store(
&args.queue_dir,
@@ -165,6 +178,8 @@ where
args,
health_check_url,
http_client,
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
tls_adapter: None,
store: queue_store,
initialized: AtomicBool::new(false),
cancel_sender,
@@ -188,11 +203,18 @@ where
);
} else if !args.client_ca.is_empty() {
// Use user-provided custom CA certificate
let ca_cert_pem = std::fs::read(&args.client_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to read root CA cert: {e}")))?;
let ca_cert = reqwest::Certificate::from_pem(&ca_cert_pem)
let certs_der = load_cert_bundle_der_bytes(&args.client_ca)
.map_err(|e| TargetError::Configuration(format!("Failed to parse root CA cert: {e}")))?;
client_builder = client_builder.add_root_certificate(ca_cert);
if certs_der.is_empty() {
return Err(TargetError::Configuration(
"Webhook client_ca did not contain any parsable certificates".to_string(),
));
}
for cert_der in certs_der {
let ca_cert = reqwest::Certificate::from_der(&cert_der)
.map_err(|e| TargetError::Configuration(format!("Failed to load root CA cert: {e}")))?;
client_builder = client_builder.add_root_certificate(ca_cert);
}
}
// If neither is set, use the system's default trust store
@@ -213,6 +235,29 @@ where
.map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}")))
}
async fn refresh_tls(&self) -> Result<(), TargetError> {
let next_fingerprint =
build_target_tls_fingerprint(&self.args.client_ca, &self.args.client_cert, &self.args.client_key).await?;
let tls_changed = {
let tls_state_guard = self.tls_state.lock();
tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
};
if !tls_changed {
return Ok(());
}
let new_client = Self::build_http_client(&self.args)?;
{
let mut tls_state_guard = self.tls_state.lock();
if tls_state_guard.fingerprint.as_ref() == Some(&next_fingerprint) {
return Ok(());
}
*self.http_client.lock() = new_client;
tls_state_guard.refresh(next_fingerprint);
}
Ok(())
}
fn health_check_url(endpoint: &Url) -> Result<Url, TargetError> {
endpoint
.host()
@@ -230,7 +275,8 @@ where
return Ok(false);
};
match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(health_check_url.as_str()).send()).await {
let client = self.http_client.lock().clone();
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
Ok(Ok(resp)) => {
debug!(
target = %self.id,
@@ -299,8 +345,14 @@ where
"Sending webhook payload"
);
let mut req_builder = self
.http_client
// When a TLS reload adapter is attached, it drives client rebuilds in
// the background. The inline per-send fingerprint check is skipped.
if self.tls_adapter.is_none() {
self.refresh_tls().await?;
}
let client = self.http_client.lock().clone();
let mut req_builder = client
.post(self.args.endpoint.as_str())
.header("Content-Type", meta.content_type.as_str());
@@ -425,6 +477,7 @@ where
async fn close(&self) -> Result<(), TargetError> {
// Send cancel signal to background tasks
let _ = self.cancel_sender.try_send(());
// Adapter cleanup is done by the coordinator; no local state to reset.
info!("Webhook target closed: {}", self.id);
Ok(())
}
@@ -460,6 +513,48 @@ where
}
}
/// Coordinated TLS hot-reload implementation for Webhook targets.
///
/// The coordinator calls these methods on a background poll loop to detect
/// TLS file changes and rebuild the HTTP client without restarting.
#[async_trait]
impl<E> ReloadableTargetTls for WebhookTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
type Material = Client;
fn tls_input_set(&self) -> TargetTlsInputSet {
TargetTlsInputSet {
ca_path: self.args.client_ca.clone(),
client_cert_path: self.args.client_cert.clone(),
client_key_path: self.args.client_key.clone(),
target_label: format!("webhook:{}", self.id.id),
}
}
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
// build_http_client is synchronous (reads files + configures reqwest).
// The coordinator already runs this in a background task, so the
// synchronous file I/O does not block the send path.
Self::build_http_client(&self.args)
}
async fn apply_tls_material(
&self,
_generation: TargetTlsGeneration,
material: Arc<Self::Material>,
_mode: ReloadApplyMode,
) -> Result<(), TargetError> {
*self.http_client.lock() = (*material).clone();
Ok(())
}
async fn validate_tls_files(&self) -> Result<(), TargetError> {
validate_tls_material(&self.args.client_ca, &self.args.client_cert, &self.args.client_key)
}
}
#[cfg(test)]
mod tests {
use super::{WebhookArgs, WebhookTarget};