mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor(tls): centralize runtime foundation (#3065)
* refactor(targets): move notify net helpers from utils * refactor(tls): centralize runtime foundation * refactor(targets): move notify net helpers from utils * refactor(tls): centralize runtime foundation * feat(tls-runtime): add TLS debug state and admin handler * refactor(tls-runtime): unify TLS debug consumer status view * fix(tls): address PR3065 review feedback * refactor(tls): align debug status payload types * refactor(targets): harden TLS hot reload paths * fix(targets): resolve review-4348251652 findings * fix(targets): finalize tls runtime review follow-ups * fix(targets): harden tls reload and review follow-ups * fix(targets): align tls reload handling across targets * fix(targets): finalize tls reload state and metrics updates * chore(deps): trim unused TLS deps * style(targets): normalize TLS reload formatting * refactor(targets): introduce tls runtime adapter path * chore: update workspace manifests for tls refactor * fix(tls): stabilize material reload and audit workflow * fix(targets): refresh tls fingerprint flow across sinks * fix(tls): align runtime coordinator and http reader updates * fix(sftp): simplify protocol error mapping * fix(tls): harmonize material loading behavior * fix(server): finalize tls material wiring in startup flow * fix(protos): tighten tls generation cache and deps
This commit is contained in:
@@ -14,20 +14,23 @@ documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["notify", "constants", "audit"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["notify", "tls"] }
|
||||
rustfs-tls-runtime = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
async-nats = { workspace = true }
|
||||
deadpool-postgres = { workspace = true }
|
||||
hyper = { workspace = true }
|
||||
hyper-rustls = { workspace = true }
|
||||
lapin = { workspace = true }
|
||||
libc = { workspace = true }
|
||||
pulsar = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
rumqttc = { workspace = true }
|
||||
redis = { workspace = true }
|
||||
rustls = { workspace = true }
|
||||
rustls-native-certs = { workspace = true }
|
||||
rustls-pki-types = { workspace = true }
|
||||
s3s = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
snap = { workspace = true }
|
||||
@@ -45,6 +48,8 @@ mysql_async = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
hashbrown = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
|
||||
@@ -64,8 +64,8 @@ pub async fn check_mqtt_broker_available_with_tls(
|
||||
use crate::target::mqtt::build_mqtt_options;
|
||||
use rumqttc::{AsyncClient, QoS};
|
||||
|
||||
let url = rustfs_utils::parse_url(broker_url)
|
||||
.map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
|
||||
let url =
|
||||
crate::parse_url(broker_url).map_err(|e| crate::TargetError::Configuration(format!("Broker URL parsing failed: {e}")))?;
|
||||
let url = url.url();
|
||||
|
||||
// build_mqtt_options returns TargetError directly; Configuration variants propagate as-is.
|
||||
|
||||
@@ -20,6 +20,7 @@ pub mod control_plane;
|
||||
pub mod domain;
|
||||
pub mod error;
|
||||
pub mod manifest;
|
||||
mod net;
|
||||
pub mod plugin;
|
||||
pub mod runtime;
|
||||
pub mod store;
|
||||
@@ -48,6 +49,7 @@ pub use manifest::{
|
||||
TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginPackaging,
|
||||
TargetPluginRuntimeTransport, builtin_target_marketplace_manifest, installable_target_marketplace_manifest,
|
||||
};
|
||||
pub use net::*;
|
||||
pub use plugin::{
|
||||
BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetPluginRegistry,
|
||||
TargetRequestValidator, boxed_target,
|
||||
|
||||
@@ -0,0 +1,642 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use hashbrown::HashMap;
|
||||
use hyper::HeaderMap;
|
||||
use regex::Regex;
|
||||
use s3s::{S3Request, S3Response};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::net::IpAddr;
|
||||
use std::path::Path;
|
||||
use std::sync::LazyLock;
|
||||
use thiserror::Error;
|
||||
use url::Url;
|
||||
|
||||
static HOST_LABEL_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?$").unwrap());
|
||||
|
||||
/// NetError represents errors that can occur in network operations.
|
||||
#[derive(Error, Debug)]
|
||||
pub enum NetError {
|
||||
#[error("invalid argument")]
|
||||
InvalidArgument,
|
||||
#[error("invalid hostname")]
|
||||
InvalidHost,
|
||||
#[error("missing '[' in host")]
|
||||
MissingBracket,
|
||||
#[error("parse error: {0}")]
|
||||
ParseError(String),
|
||||
#[error("unexpected scheme: {0}")]
|
||||
UnexpectedScheme(String),
|
||||
#[error("scheme appears with empty host")]
|
||||
SchemeWithEmptyHost,
|
||||
}
|
||||
|
||||
/// Host represents a network host with IP/name and port.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Host {
|
||||
pub name: String,
|
||||
pub port: Option<u16>,
|
||||
}
|
||||
|
||||
impl Host {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.name.is_empty()
|
||||
}
|
||||
|
||||
pub fn equal(&self, other: &Host) -> bool {
|
||||
self.to_string() == other.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Host {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self.port {
|
||||
Some(p) => write!(f, "{}:{}", self.name, p),
|
||||
None => write!(f, "{}", self.name),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract request parameters from S3Request, mainly header information.
|
||||
pub fn extract_req_params<T>(req: &S3Request<T>) -> HashMap<String, String> {
|
||||
extract_params_header(&req.headers)
|
||||
}
|
||||
|
||||
/// Extract request parameters from hyper::HeaderMap, mainly header information.
|
||||
/// This function is useful when you have a raw HTTP request and need to extract parameters.
|
||||
#[deprecated(since = "0.1.0", note = "Use extract_params_header instead")]
|
||||
pub fn extract_req_params_header(head: &HeaderMap) -> HashMap<String, String> {
|
||||
extract_params_header(head)
|
||||
}
|
||||
|
||||
/// Extract parameters from hyper::HeaderMap, mainly header information.
|
||||
pub fn extract_params_header(head: &HeaderMap) -> HashMap<String, String> {
|
||||
let mut params = HashMap::new();
|
||||
for (key, value) in head.iter() {
|
||||
if let Ok(val_str) = value.to_str() {
|
||||
params.insert(key.as_str().to_string(), val_str.to_string());
|
||||
}
|
||||
}
|
||||
params
|
||||
}
|
||||
|
||||
/// Extract response elements from S3Response, mainly header information.
|
||||
pub fn extract_resp_elements<T>(resp: &S3Response<T>) -> HashMap<String, String> {
|
||||
extract_params_header(&resp.headers)
|
||||
}
|
||||
|
||||
/// Get host from header information.
|
||||
pub fn get_request_host(headers: &HeaderMap) -> String {
|
||||
headers
|
||||
.get("host")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Get Port from header information.
|
||||
/// Priority:
|
||||
/// 1. x-forwarded-port
|
||||
/// 2. host header (parse port)
|
||||
/// 3. x-forwarded-proto inferred default (http=80, https=443) when host has no explicit port
|
||||
/// 4. port header
|
||||
pub fn get_request_port(headers: &HeaderMap) -> u16 {
|
||||
if let Some(port) = headers
|
||||
.get("x-forwarded-port")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.parse::<u16>().ok())
|
||||
{
|
||||
return port;
|
||||
}
|
||||
|
||||
if let Some(host) = headers.get("host").and_then(|v| v.to_str().ok()) {
|
||||
if let Some(idx) = host.rfind(':') {
|
||||
let valid_colon = match host.rfind(']') {
|
||||
Some(close_bracket_idx) => idx > close_bracket_idx,
|
||||
None => true,
|
||||
};
|
||||
|
||||
if valid_colon
|
||||
&& let Ok(port) = host[idx + 1..].parse::<u16>()
|
||||
&& port > 0
|
||||
{
|
||||
return port;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(proto) = headers.get("x-forwarded-proto").and_then(|v| v.to_str().ok()) {
|
||||
match proto {
|
||||
"http" => return 80,
|
||||
"https" => return 443,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
headers
|
||||
.get("port")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.parse::<u16>().ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Get content-length from header information.
|
||||
pub fn get_request_content_length(headers: &HeaderMap) -> u64 {
|
||||
headers
|
||||
.get("content-length")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.parse::<u64>().ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Get referer from header information.
|
||||
pub fn get_request_referer(headers: &HeaderMap) -> String {
|
||||
headers
|
||||
.get("referer")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Get user-agent from header information.
|
||||
pub fn get_request_user_agent(headers: &HeaderMap) -> String {
|
||||
headers
|
||||
.get("user-agent")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// parse_host parses a string into a Host, with validation similar to Go's ParseHost.
|
||||
pub fn parse_host(s: &str) -> Result<Host, NetError> {
|
||||
if s.is_empty() {
|
||||
return Err(NetError::InvalidArgument);
|
||||
}
|
||||
|
||||
let is_valid_host = |host: &str| -> bool {
|
||||
if host.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if host.parse::<IpAddr>().is_ok() {
|
||||
return true;
|
||||
}
|
||||
if !(1..=253).contains(&host.len()) {
|
||||
return false;
|
||||
}
|
||||
for (i, label) in host.split('.').enumerate() {
|
||||
if i + 1 == host.split('.').count() && label.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !(1..=63).contains(&label.len()) || !HOST_LABEL_REGEX.is_match(label) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
};
|
||||
|
||||
let (host, port) = if let Some(rest) = s.strip_prefix('[') {
|
||||
let Some(end) = rest.find(']') else {
|
||||
return Err(NetError::MissingBracket);
|
||||
};
|
||||
let host = rest[..end].to_string();
|
||||
let port_str = &rest[end + 1..];
|
||||
let port = if let Some(port_str) = port_str.strip_prefix(':') {
|
||||
if port_str.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(port_str.parse().map_err(|_| NetError::ParseError(port_str.to_string()))?)
|
||||
}
|
||||
} else if port_str.is_empty() {
|
||||
None
|
||||
} else {
|
||||
return Err(NetError::InvalidHost);
|
||||
};
|
||||
|
||||
(host, port)
|
||||
} else {
|
||||
if s.contains(']') {
|
||||
return Err(NetError::MissingBracket);
|
||||
}
|
||||
|
||||
let (host_str, port_str) = if s.matches(':').count() > 1 {
|
||||
(s, "")
|
||||
} else {
|
||||
s.rsplit_once(':').map_or((s, ""), |(h, p)| (h, p))
|
||||
};
|
||||
let port = if !port_str.is_empty() {
|
||||
Some(port_str.parse().map_err(|_| NetError::ParseError(port_str.to_string()))?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
(trim_ipv6(host_str)?, port)
|
||||
};
|
||||
|
||||
let trimmed_host = host.split('%').next().unwrap_or(&host);
|
||||
|
||||
if !is_valid_host(trimmed_host) {
|
||||
return Err(NetError::InvalidHost);
|
||||
}
|
||||
|
||||
Ok(Host { name: host, port })
|
||||
}
|
||||
|
||||
fn trim_ipv6(host: &str) -> Result<String, NetError> {
|
||||
if host.ends_with(']') {
|
||||
if !host.starts_with('[') {
|
||||
return Err(NetError::MissingBracket);
|
||||
}
|
||||
Ok(host[1..host.len() - 1].to_string())
|
||||
} else {
|
||||
Ok(host.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// URL is a wrapper around url::Url for custom handling.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ParsedURL(pub Url);
|
||||
|
||||
impl ParsedURL {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.as_str() == "" || (self.0.scheme() == "about" && self.0.path() == "blank")
|
||||
}
|
||||
|
||||
pub fn hostname(&self) -> String {
|
||||
self.0.host_str().unwrap_or("").to_string()
|
||||
}
|
||||
|
||||
pub fn port(&self) -> String {
|
||||
match self.0.port() {
|
||||
Some(p) => p.to_string(),
|
||||
None => match self.0.scheme() {
|
||||
"http" => "80".to_string(),
|
||||
"https" => "443".to_string(),
|
||||
_ => "".to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scheme(&self) -> &str {
|
||||
self.0.scheme()
|
||||
}
|
||||
|
||||
pub fn url(&self) -> &Url {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ParsedURL {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut url = self.0.clone();
|
||||
if let Some(host) = url.host_str().map(|h| h.to_string())
|
||||
&& let Some(port) = url.port()
|
||||
&& ((url.scheme() == "http" && port == 80) || (url.scheme() == "https" && port == 443))
|
||||
{
|
||||
let _ = url.set_host(Some(&host));
|
||||
let _ = url.set_port(None);
|
||||
}
|
||||
let mut s = url.to_string();
|
||||
|
||||
if s.ends_with('/') && url.path() == "/" {
|
||||
s.pop();
|
||||
}
|
||||
|
||||
write!(f, "{s}")
|
||||
}
|
||||
}
|
||||
|
||||
impl serde::Serialize for ParsedURL {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
serializer.serialize_str(&self.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for ParsedURL {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s: String = serde::Deserialize::deserialize(deserializer)?;
|
||||
if s.is_empty() {
|
||||
Ok(ParsedURL(Url::parse("about:blank").unwrap()))
|
||||
} else {
|
||||
parse_url(&s).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// parse_url parses a string into a ParsedURL, with host validation and path cleaning.
|
||||
pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
|
||||
if let Some(scheme_end) = s.find("://")
|
||||
&& s[scheme_end + 3..].starts_with('/')
|
||||
{
|
||||
let scheme = &s[..scheme_end];
|
||||
if !scheme.is_empty() {
|
||||
return Err(NetError::SchemeWithEmptyHost);
|
||||
}
|
||||
}
|
||||
|
||||
let mut uu = Url::parse(s).map_err(|e| NetError::ParseError(e.to_string()))?;
|
||||
if uu.host_str().is_none_or(|h| h.is_empty()) {
|
||||
if uu.scheme() != "" {
|
||||
return Err(NetError::SchemeWithEmptyHost);
|
||||
}
|
||||
} else {
|
||||
let port_str = uu.port().map(|p| p.to_string()).unwrap_or_else(|| match uu.scheme() {
|
||||
"http" => "80".to_string(),
|
||||
"https" => "443".to_string(),
|
||||
_ => "".to_string(),
|
||||
});
|
||||
|
||||
if !port_str.is_empty() {
|
||||
let host_port = format!("{}:{}", uu.host_str().unwrap(), port_str);
|
||||
parse_host(&host_port)?;
|
||||
}
|
||||
}
|
||||
|
||||
if !uu.path().is_empty() {
|
||||
let mut cleaned_path = String::new();
|
||||
for comp in Path::new(uu.path()).components() {
|
||||
use std::path::Component;
|
||||
match comp {
|
||||
Component::RootDir => cleaned_path.push('/'),
|
||||
Component::Normal(s) => {
|
||||
if !cleaned_path.ends_with('/') {
|
||||
cleaned_path.push('/');
|
||||
}
|
||||
cleaned_path.push_str(&s.to_string_lossy());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if s.ends_with('/') && !cleaned_path.ends_with('/') {
|
||||
cleaned_path.push('/');
|
||||
}
|
||||
if cleaned_path.is_empty() {
|
||||
cleaned_path.push('/');
|
||||
}
|
||||
uu.set_path(&cleaned_path);
|
||||
}
|
||||
|
||||
Ok(ParsedURL(uu))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn parse_http_url(s: &str) -> Result<ParsedURL, NetError> {
|
||||
let u = parse_url(s)?;
|
||||
match u.0.scheme() {
|
||||
"http" | "https" => Ok(u),
|
||||
_ => Err(NetError::UnexpectedScheme(u.0.scheme().to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn is_network_or_host_down(err: &std::io::Error, expect_timeouts: bool) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return !expect_timeouts;
|
||||
}
|
||||
let err_str = err.to_string().to_lowercase();
|
||||
err_str.contains("connection reset by peer")
|
||||
|| err_str.contains("connection timed out")
|
||||
|| err_str.contains("broken pipe")
|
||||
|| err_str.contains("use of closed network connection")
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn is_conn_reset_err(err: &std::io::Error) -> bool {
|
||||
err.to_string().contains("connection reset by peer") || matches!(err.raw_os_error(), Some(libc::ECONNRESET))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn is_conn_refused_err(err: &std::io::Error) -> bool {
|
||||
err.to_string().contains("connection refused") || matches!(err.raw_os_error(), Some(libc::ECONNREFUSED))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use hyper::header::HeaderValue;
|
||||
|
||||
#[test]
|
||||
fn test_get_request_port() {
|
||||
let mut headers = HeaderMap::new();
|
||||
|
||||
assert_eq!(get_request_port(&headers), 0);
|
||||
|
||||
headers.insert("port", HeaderValue::from_static("8080"));
|
||||
assert_eq!(get_request_port(&headers), 8080);
|
||||
|
||||
headers.remove("port");
|
||||
headers.insert("host", HeaderValue::from_static("example.com:9000"));
|
||||
assert_eq!(get_request_port(&headers), 9000);
|
||||
|
||||
headers.insert("host", HeaderValue::from_static("example.com"));
|
||||
assert_eq!(get_request_port(&headers), 0);
|
||||
|
||||
headers.insert("host", HeaderValue::from_static("[::1]:9001"));
|
||||
assert_eq!(get_request_port(&headers), 9001);
|
||||
|
||||
headers.insert("host", HeaderValue::from_static("[::1]"));
|
||||
assert_eq!(get_request_port(&headers), 0);
|
||||
|
||||
headers.insert("x-forwarded-port", HeaderValue::from_static("7000"));
|
||||
assert_eq!(get_request_port(&headers), 7000);
|
||||
|
||||
headers.remove("x-forwarded-port");
|
||||
headers.insert("host", HeaderValue::from_static("example.com"));
|
||||
headers.insert("x-forwarded-proto", HeaderValue::from_static("http"));
|
||||
assert_eq!(get_request_port(&headers), 80);
|
||||
|
||||
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
|
||||
assert_eq!(get_request_port(&headers), 443);
|
||||
|
||||
headers.insert("x-forwarded-proto", HeaderValue::from_static("ftp"));
|
||||
assert_eq!(get_request_port(&headers), 0);
|
||||
|
||||
headers.insert("host", HeaderValue::from_static("example.com:0"));
|
||||
headers.insert("x-forwarded-proto", HeaderValue::from_static("https"));
|
||||
assert_eq!(get_request_port(&headers), 443);
|
||||
|
||||
headers.remove("x-forwarded-proto");
|
||||
assert_eq!(get_request_port(&headers), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_empty_string_returns_error() {
|
||||
let result = parse_host("");
|
||||
assert!(matches!(result, Err(NetError::InvalidArgument)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_valid_ipv4() {
|
||||
let result = parse_host("192.168.1.1:8080");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "192.168.1.1");
|
||||
assert_eq!(host.port, Some(8080));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_valid_hostname() {
|
||||
let result = parse_host("example.com:443");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "example.com");
|
||||
assert_eq!(host.port, Some(443));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_ipv6_brackets() {
|
||||
let result = parse_host("[::1]:8080");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "::1");
|
||||
assert_eq!(host.port, Some(8080));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_bare_ipv6_without_port() {
|
||||
let result = parse_host("::1");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "::1");
|
||||
assert_eq!(host.port, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_ipv6_zone_without_port() {
|
||||
let result = parse_host("fe80::1%eth0");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "fe80::1%eth0");
|
||||
assert_eq!(host.port, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_bracketed_ipv6_zone_and_port() {
|
||||
let result = parse_host("[fe80::1%eth0]:9000");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "fe80::1%eth0");
|
||||
assert_eq!(host.port, Some(9000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_bracketed_ipv6_without_port() {
|
||||
let result = parse_host("[::1]");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "::1");
|
||||
assert_eq!(host.port, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_invalid_ipv6_missing_bracket() {
|
||||
let result = parse_host("::1]:8080");
|
||||
assert!(matches!(result, Err(NetError::MissingBracket)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_with_invalid_hostname() {
|
||||
let result = parse_host("invalid..host:80");
|
||||
assert!(matches!(result, Err(NetError::InvalidHost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_host_without_port() {
|
||||
let result = parse_host("example.com");
|
||||
assert!(result.is_ok());
|
||||
let host = result.unwrap();
|
||||
assert_eq!(host.name, "example.com");
|
||||
assert_eq!(host.port, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_is_empty_when_name_is_empty() {
|
||||
let host = Host {
|
||||
name: "".to_string(),
|
||||
port: None,
|
||||
};
|
||||
assert!(host.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_is_not_empty_when_name_present() {
|
||||
let host = Host {
|
||||
name: "example.com".to_string(),
|
||||
port: Some(80),
|
||||
};
|
||||
assert!(!host.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_to_string_with_port() {
|
||||
let host = Host {
|
||||
name: "example.com".to_string(),
|
||||
port: Some(80),
|
||||
};
|
||||
assert_eq!(host.to_string(), "example.com:80");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_to_string_without_port() {
|
||||
let host = Host {
|
||||
name: "example.com".to_string(),
|
||||
port: None,
|
||||
};
|
||||
assert_eq!(host.to_string(), "example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_url_with_valid_http_url() {
|
||||
let result = parse_url("http://example.com/path");
|
||||
assert!(result.is_ok());
|
||||
let parsed = result.unwrap();
|
||||
assert_eq!(parsed.hostname(), "example.com");
|
||||
assert_eq!(parsed.port(), "80");
|
||||
assert_eq!(parsed.scheme(), "http");
|
||||
assert_eq!(parsed.to_string(), "http://example.com/path");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_url_with_explicit_default_https_port() {
|
||||
let result = parse_url("https://example.com:443/path");
|
||||
assert!(result.is_ok());
|
||||
let parsed = result.unwrap();
|
||||
assert_eq!(parsed.to_string(), "https://example.com/path");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_url_with_empty_host_returns_error() {
|
||||
let result = parse_url("http:///path");
|
||||
assert!(matches!(result, Err(NetError::SchemeWithEmptyHost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_url_with_invalid_host_returns_error() {
|
||||
let result = parse_url("http://invalid..host/path");
|
||||
assert!(matches!(result, Err(NetError::InvalidHost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_url_normalizes_path() {
|
||||
let result = parse_url("http://example.com//path/../path/");
|
||||
assert!(result.is_ok());
|
||||
let parsed = result.unwrap();
|
||||
assert_eq!(parsed.to_string(), "http://example.com/path/");
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
pub mod adapter;
|
||||
pub mod sidecar;
|
||||
pub mod sidecar_protocol;
|
||||
pub mod tls;
|
||||
|
||||
use crate::Target;
|
||||
use crate::arn::TargetID;
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! `TlsReloadAdapter<M>` — the single entry-point that connects a target to
|
||||
//! the TLS reload coordinator. Each target holds an `Option<TlsReloadAdapter<M>>`
|
||||
//! and calls `current_material()` on the hot path. When `None`, the target
|
||||
//! falls back to its legacy inline fingerprint logic.
|
||||
|
||||
use super::config::TlsReloadOptions;
|
||||
use super::coordinator::TargetTlsReloadCoordinator;
|
||||
use super::state::{TargetTlsRuntimeState, TargetTlsStatusSnapshot};
|
||||
use super::r#trait::ReloadableTargetTls;
|
||||
use std::sync::Arc;
|
||||
use tracing::warn;
|
||||
|
||||
/// Bridges a `ReloadableTargetTls` implementor and the reload coordinator.
|
||||
///
|
||||
/// Created via [`TlsReloadAdapter::try_register`]. Holds the coordinator-
|
||||
/// managed runtime state and exposes a zero-cost `current_material()` accessor
|
||||
/// for the send hot-path.
|
||||
pub struct TlsReloadAdapter<M> {
|
||||
runtime_state: Arc<TargetTlsRuntimeState<M>>,
|
||||
options: TlsReloadOptions,
|
||||
}
|
||||
|
||||
impl<M> std::fmt::Debug for TlsReloadAdapter<M> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("TlsReloadAdapter")
|
||||
.field("target_label", &self.runtime_state.inputs.target_label)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl<M> Clone for TlsReloadAdapter<M> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
runtime_state: Arc::clone(&self.runtime_state),
|
||||
options: self.options.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<M: Send + Sync + 'static> TlsReloadAdapter<M> {
|
||||
/// Registers `target` with the coordinator and returns an adapter.
|
||||
///
|
||||
/// On success the coordinator has:
|
||||
/// - built initial TLS material
|
||||
/// - spawned a background poll loop
|
||||
///
|
||||
/// On failure returns `None` (the caller should keep its inline fallback
|
||||
/// path intact — the target continues to work, just without coordinator
|
||||
/// support).
|
||||
pub async fn try_register<T: ReloadableTargetTls<Material = M>>(
|
||||
target: Arc<T>,
|
||||
options: TlsReloadOptions,
|
||||
coordinator: &TargetTlsReloadCoordinator,
|
||||
) -> Option<Self> {
|
||||
let label = target.tls_input_set().target_label.clone();
|
||||
match coordinator.register(target, options.clone()).await {
|
||||
Ok(runtime_state) => {
|
||||
tracing::info!(target = %label, "TLS reload adapter registered");
|
||||
Some(Self { runtime_state, options })
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(target = %label, error = %err, "TLS reload adapter registration failed; target will use inline fallback");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hot-path accessor: returns the current TLS material managed by the
|
||||
/// coordinator. The returned `Arc<M>` is cheap to clone.
|
||||
#[inline]
|
||||
pub fn current_material(&self) -> Arc<M> {
|
||||
Arc::clone(&self.runtime_state.current.load().material)
|
||||
}
|
||||
|
||||
/// Returns the active generation counter.
|
||||
#[inline]
|
||||
pub fn generation(&self) -> u64 {
|
||||
self.runtime_state.current.load().generation.0
|
||||
}
|
||||
|
||||
/// Returns a read-only status snapshot for admin/observability.
|
||||
pub fn status_snapshot(&self) -> TargetTlsStatusSnapshot {
|
||||
TargetTlsReloadCoordinator::build_status_snapshot(&self.runtime_state, &self.options)
|
||||
}
|
||||
|
||||
/// Returns the underlying runtime state (for `close()` cleanup etc.).
|
||||
pub fn runtime_state(&self) -> &Arc<TargetTlsRuntimeState<M>> {
|
||||
&self.runtime_state
|
||||
}
|
||||
|
||||
/// Unregisters from the coordinator (stops the poll loop).
|
||||
pub async fn unregister(&self, coordinator: &TargetTlsReloadCoordinator) {
|
||||
let label = &self.runtime_state.inputs.target_label;
|
||||
if let Err(err) = coordinator.unregister(label).await {
|
||||
warn!(target = %label, error = %err, "Failed to unregister TLS reload adapter");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::error::TargetError;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
struct FakeTarget {
|
||||
label: String,
|
||||
build_calls: AtomicUsize,
|
||||
should_fail: AtomicBool,
|
||||
}
|
||||
|
||||
impl FakeTarget {
|
||||
fn new(label: &str) -> Self {
|
||||
Self {
|
||||
label: label.to_string(),
|
||||
build_calls: AtomicUsize::new(0),
|
||||
should_fail: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ReloadableTargetTls for FakeTarget {
|
||||
type Material = String;
|
||||
|
||||
fn tls_input_set(&self) -> super::super::state::TargetTlsInputSet {
|
||||
super::super::state::TargetTlsInputSet {
|
||||
ca_path: String::new(),
|
||||
client_cert_path: String::new(),
|
||||
client_key_path: String::new(),
|
||||
target_label: self.label.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
self.build_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.should_fail.load(Ordering::SeqCst) {
|
||||
return Err(TargetError::Configuration("fail".to_string()));
|
||||
}
|
||||
Ok("material".to_string())
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: super::super::fingerprint::TargetTlsGeneration,
|
||||
_material: Arc<Self::Material>,
|
||||
_mode: super::super::config::ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_register_returns_adapter_on_success() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(FakeTarget::new("test:fake"));
|
||||
let options = TlsReloadOptions::default();
|
||||
|
||||
let adapter = TlsReloadAdapter::try_register(target.clone(), options, &coordinator).await;
|
||||
assert!(adapter.is_some());
|
||||
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
|
||||
|
||||
let a = adapter.unwrap();
|
||||
assert_eq!(*a.current_material(), "material");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_register_returns_none_on_failure() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(FakeTarget::new("test:fail"));
|
||||
target.should_fail.store(true, Ordering::SeqCst);
|
||||
let options = TlsReloadOptions::default();
|
||||
|
||||
let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await;
|
||||
assert!(adapter.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn adapter_is_clone_and_shares_state() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(FakeTarget::new("test:clone"));
|
||||
let options = TlsReloadOptions::default();
|
||||
|
||||
let a = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
|
||||
let b = a.clone();
|
||||
|
||||
assert_eq!(*a.current_material(), *b.current_material());
|
||||
assert_eq!(a.generation(), b.generation());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn status_snapshot_contains_label() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(FakeTarget::new("test:snap"));
|
||||
let options = TlsReloadOptions::default();
|
||||
|
||||
let adapter = TlsReloadAdapter::try_register(target, options, &coordinator).await.unwrap();
|
||||
let snap = adapter.status_snapshot();
|
||||
assert_eq!(snap.target_label, "test:snap");
|
||||
assert!(snap.reload_enabled);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Target-level TLS reload configuration.
|
||||
//!
|
||||
//! Re-exports the shared types from `rustfs_tls_runtime::config` so that
|
||||
//! targets and their callers use a single source of truth.
|
||||
|
||||
pub use rustfs_tls_runtime::config::{ReloadApplyHint as ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};
|
||||
@@ -0,0 +1,656 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Target TLS reload coordinator. Manages per-target background poll loops
|
||||
//! that periodically check TLS material fingerprints and drive safe reload.
|
||||
|
||||
use super::config::{ReloadApplyMode, ReloadDetectMode, TlsReloadOptions};
|
||||
#[cfg(test)]
|
||||
use super::fingerprint::TargetTlsFingerprint;
|
||||
use super::fingerprint::{TargetTlsGeneration, build_target_tls_fingerprint};
|
||||
use super::metrics::{record_target_tls_publication_fail, record_target_tls_reload_result, record_target_tls_reload_skipped};
|
||||
#[cfg(test)]
|
||||
use super::state::TargetTlsInputSet;
|
||||
use super::state::{TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
|
||||
use super::r#trait::ReloadableTargetTls;
|
||||
use super::validate::validate_tls_material;
|
||||
use crate::error::TargetError;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
struct TargetReloadEntry {
|
||||
#[expect(dead_code)]
|
||||
target_label: String,
|
||||
cancel_tx: tokio::sync::mpsc::Sender<()>,
|
||||
poll_handle: JoinHandle<()>,
|
||||
}
|
||||
|
||||
/// The top-level coordinator that manages TLS reload for all registered targets.
|
||||
///
|
||||
/// Typically one instance per process, held alongside `TargetRuntimeManager`.
|
||||
/// Each registered target gets its own background poll loop that periodically
|
||||
/// checks TLS fingerprints and drives the build/apply cycle.
|
||||
pub struct TargetTlsReloadCoordinator {
|
||||
entries: RwLock<HashMap<String, TargetReloadEntry>>,
|
||||
}
|
||||
|
||||
impl Default for TargetTlsReloadCoordinator {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetTlsReloadCoordinator {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
entries: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a target for coordinated TLS reload. Spawns a background poll loop.
|
||||
///
|
||||
/// Returns the initial runtime state that the target should hold for
|
||||
/// accessing the current TLS material via `ArcSwap`.
|
||||
pub async fn register<T: ReloadableTargetTls>(
|
||||
&self,
|
||||
target: Arc<T>,
|
||||
options: TlsReloadOptions,
|
||||
) -> Result<Arc<TargetTlsRuntimeState<T::Material>>, TargetError> {
|
||||
if !options.enabled {
|
||||
return Err(TargetError::Configuration("TLS reload is disabled".to_string()));
|
||||
}
|
||||
|
||||
let inputs = target.tls_input_set();
|
||||
let target_label = inputs.target_label.clone();
|
||||
|
||||
// Build initial material
|
||||
let initial_material = Arc::new(target.build_tls_material().await?);
|
||||
let initial_fingerprint =
|
||||
build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
|
||||
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(1),
|
||||
fingerprint: initial_fingerprint,
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: unix_time_ms(),
|
||||
});
|
||||
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state.clone(), inputs));
|
||||
|
||||
if options.detect_mode == ReloadDetectMode::Poll || options.detect_mode == ReloadDetectMode::Hybrid {
|
||||
let (cancel_tx, cancel_rx) = tokio::sync::mpsc::channel(1);
|
||||
let poll_handle = tokio::spawn(spawn_target_poll_loop(target, Arc::clone(&runtime_state), options, cancel_rx));
|
||||
|
||||
let mut entries = self.entries.write().await;
|
||||
entries.insert(
|
||||
target_label.clone(),
|
||||
TargetReloadEntry {
|
||||
target_label: target_label.clone(),
|
||||
cancel_tx,
|
||||
poll_handle,
|
||||
},
|
||||
);
|
||||
|
||||
info!(target = %target_label, "Registered target for TLS reload coordinator");
|
||||
}
|
||||
|
||||
Ok(runtime_state)
|
||||
}
|
||||
|
||||
/// Unregister a target and stop its poll loop.
|
||||
pub async fn unregister(&self, target_label: &str) -> Result<(), TargetError> {
|
||||
let mut entries = self.entries.write().await;
|
||||
if let Some(entry) = entries.remove(target_label) {
|
||||
let _ = entry.cancel_tx.send(()).await;
|
||||
entry.poll_handle.abort();
|
||||
info!(target = %target_label, "Unregistered target from TLS reload coordinator");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Force an immediate reload check for a specific target.
|
||||
/// Used by admin endpoints and test harnesses.
|
||||
pub async fn force_reload<T: ReloadableTargetTls>(
|
||||
&self,
|
||||
target: &T,
|
||||
runtime_state: &TargetTlsRuntimeState<T::Material>,
|
||||
options: &TlsReloadOptions,
|
||||
) -> Result<TargetTlsGeneration, TargetError> {
|
||||
reload_target_once(target, runtime_state, options).await
|
||||
}
|
||||
|
||||
/// Stop all poll loops.
|
||||
pub async fn shutdown(&self) {
|
||||
let mut entries = self.entries.write().await;
|
||||
for (label, entry) in entries.drain() {
|
||||
let _ = entry.cancel_tx.send(()).await;
|
||||
entry.poll_handle.abort();
|
||||
debug!(target = %label, "Stopped TLS reload poll loop");
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect status snapshots from all registered targets.
|
||||
/// The caller must provide the runtime states separately since the
|
||||
/// coordinator does not hold type-erased references to them.
|
||||
pub fn build_status_snapshot<M>(
|
||||
runtime_state: &TargetTlsRuntimeState<M>,
|
||||
options: &TlsReloadOptions,
|
||||
) -> TargetTlsStatusSnapshot {
|
||||
let current = runtime_state.current.load();
|
||||
let last_attempt = runtime_state.last_attempt_unix_ms();
|
||||
let last_success = runtime_state.last_success_unix_ms();
|
||||
let last_error = runtime_state.last_error.read().clone();
|
||||
|
||||
TargetTlsStatusSnapshot {
|
||||
target_label: runtime_state.inputs.target_label.clone(),
|
||||
generation: current.generation.0,
|
||||
reload_enabled: options.enabled,
|
||||
detect_mode: match options.detect_mode {
|
||||
ReloadDetectMode::Poll => "poll",
|
||||
ReloadDetectMode::Watch => "watch",
|
||||
ReloadDetectMode::Hybrid => "hybrid",
|
||||
},
|
||||
apply_mode: match options.apply_hint {
|
||||
ReloadApplyMode::Lazy => "lazy",
|
||||
ReloadApplyMode::SoftReconnect => "soft_reconnect",
|
||||
},
|
||||
last_attempt_time: if last_attempt > 0 { Some(last_attempt) } else { None },
|
||||
last_success_time: if last_success > 0 { Some(last_success) } else { None },
|
||||
last_error,
|
||||
ca_path: runtime_state.inputs.ca_path.clone(),
|
||||
client_cert_path: runtime_state.inputs.client_cert_path.clone(),
|
||||
client_key_path: runtime_state.inputs.client_key_path.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Background poll loop for a single target.
|
||||
async fn spawn_target_poll_loop<T: ReloadableTargetTls>(
|
||||
target: Arc<T>,
|
||||
runtime_state: Arc<TargetTlsRuntimeState<T::Material>>,
|
||||
options: TlsReloadOptions,
|
||||
mut cancel_rx: tokio::sync::mpsc::Receiver<()>,
|
||||
) {
|
||||
let mut interval = tokio::time::interval(options.interval);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
interval.tick().await; // skip the immediate first tick
|
||||
|
||||
let label = &runtime_state.inputs.target_label;
|
||||
let debounce = options.debounce;
|
||||
debug!(target = %label, interval_secs = options.interval.as_secs(), "TLS reload poll loop started");
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel_rx.recv() => {
|
||||
info!(target = %label, "TLS reload poll loop stopped");
|
||||
return;
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
// Enforce minimum stable age: if the last attempt was too recent
|
||||
// (e.g. a rapid succession of ticks), wait one debounce period
|
||||
// before reading files again to avoid picking up half-written certs.
|
||||
let last_attempt = runtime_state.last_attempt_unix_ms();
|
||||
if last_attempt > 0 {
|
||||
let elapsed_since_last = unix_time_ms().saturating_sub(last_attempt);
|
||||
if elapsed_since_last < debounce.as_millis() as u64 {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = reload_target_once(target.as_ref(), runtime_state.as_ref(), &options).await {
|
||||
warn!(target = %label, error = %err, "TLS reload poll check failed (will retry)");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Single reload cycle: read → compare → validate → build → apply → publish.
|
||||
///
|
||||
/// Returns the new generation on success, or an error on failure.
|
||||
/// On failure the current generation and material are untouched.
|
||||
async fn reload_target_once<T: ReloadableTargetTls>(
|
||||
target: &T,
|
||||
runtime_state: &TargetTlsRuntimeState<T::Material>,
|
||||
options: &TlsReloadOptions,
|
||||
) -> Result<TargetTlsGeneration, TargetError> {
|
||||
let now = unix_time_ms();
|
||||
runtime_state.mark_attempt(now);
|
||||
let started_at = std::time::Instant::now();
|
||||
let label = &runtime_state.inputs.target_label;
|
||||
|
||||
// 1. Read TLS files and compute fingerprint
|
||||
let inputs = &runtime_state.inputs;
|
||||
let next_fingerprint =
|
||||
build_target_tls_fingerprint(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path).await?;
|
||||
|
||||
// 2. Compare with current — skip if unchanged
|
||||
let current = runtime_state.current.load();
|
||||
if current.fingerprint == next_fingerprint {
|
||||
record_target_tls_reload_skipped(label, "unchanged");
|
||||
return Ok(current.generation);
|
||||
}
|
||||
|
||||
// 3. Validate TLS files (cert/key pairing, CA parseable)
|
||||
if let Err(err) = validate_tls_material(&inputs.ca_path, &inputs.client_cert_path, &inputs.client_key_path) {
|
||||
*runtime_state.last_error.write() = Some(err.to_string());
|
||||
record_target_tls_publication_fail(label);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// Also call target-specific validation
|
||||
if let Err(err) = target.validate_tls_files().await {
|
||||
*runtime_state.last_error.write() = Some(err.to_string());
|
||||
record_target_tls_publication_fail(label);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// 4. Build new material (does not touch current state yet)
|
||||
let new_material = match target.build_tls_material().await {
|
||||
Ok(m) => Arc::new(m),
|
||||
Err(err) => {
|
||||
*runtime_state.last_error.write() = Some(err.to_string());
|
||||
record_target_tls_publication_fail(label);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
// 5. Bump generation and apply
|
||||
let new_generation = runtime_state.bump_generation();
|
||||
if let Err(err) = target
|
||||
.apply_tls_material(new_generation, Arc::clone(&new_material), options.apply_hint)
|
||||
.await
|
||||
{
|
||||
*runtime_state.last_error.write() = Some(err.to_string());
|
||||
record_target_tls_publication_fail(label);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// 6. Publish new state
|
||||
let published = Arc::new(TargetTlsPublishedState {
|
||||
generation: new_generation,
|
||||
fingerprint: next_fingerprint,
|
||||
material: new_material,
|
||||
loaded_at_unix_ms: now,
|
||||
});
|
||||
runtime_state.current.store(published.clone());
|
||||
runtime_state.last_good.store(published);
|
||||
runtime_state.mark_success(now);
|
||||
*runtime_state.last_error.write() = None;
|
||||
|
||||
record_target_tls_reload_result(label, "ok", started_at.elapsed().as_secs_f64(), new_generation.0);
|
||||
|
||||
debug!(target = %label, generation = new_generation.0, "TLS reload successful");
|
||||
Ok(new_generation)
|
||||
}
|
||||
|
||||
fn unix_time_ms() -> u64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
struct MockTarget {
|
||||
inputs: TargetTlsInputSet,
|
||||
build_calls: AtomicUsize,
|
||||
apply_calls: AtomicUsize,
|
||||
validate_calls: AtomicUsize,
|
||||
should_fail_build: AtomicBool,
|
||||
should_fail_apply: AtomicBool,
|
||||
should_fail_validate: AtomicBool,
|
||||
}
|
||||
|
||||
impl MockTarget {
|
||||
fn new(label: &str) -> Self {
|
||||
Self {
|
||||
inputs: TargetTlsInputSet {
|
||||
ca_path: String::new(),
|
||||
client_cert_path: String::new(),
|
||||
client_key_path: String::new(),
|
||||
target_label: label.to_string(),
|
||||
},
|
||||
build_calls: AtomicUsize::new(0),
|
||||
apply_calls: AtomicUsize::new(0),
|
||||
validate_calls: AtomicUsize::new(0),
|
||||
should_fail_build: AtomicBool::new(false),
|
||||
should_fail_apply: AtomicBool::new(false),
|
||||
should_fail_validate: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ReloadableTargetTls for MockTarget {
|
||||
type Material = String;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
self.inputs.clone()
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
self.build_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.should_fail_build.load(Ordering::SeqCst) {
|
||||
return Err(TargetError::Configuration("build failed".to_string()));
|
||||
}
|
||||
Ok("mock-material".to_string())
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
_material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
self.apply_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.should_fail_apply.load(Ordering::SeqCst) {
|
||||
return Err(TargetError::Configuration("apply failed".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
self.validate_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.should_fail_validate.load(Ordering::SeqCst) {
|
||||
return Err(TargetError::Configuration("validate failed".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn default_options() -> TlsReloadOptions {
|
||||
TlsReloadOptions {
|
||||
enabled: true,
|
||||
detect_mode: ReloadDetectMode::Poll,
|
||||
interval: std::time::Duration::from_secs(1),
|
||||
debounce: std::time::Duration::from_secs(1),
|
||||
min_stable_age: std::time::Duration::from_millis(100),
|
||||
apply_hint: ReloadApplyMode::Lazy,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn register_builds_initial_material() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(MockTarget::new("test:webhook"));
|
||||
|
||||
let options = TlsReloadOptions {
|
||||
detect_mode: ReloadDetectMode::Watch, // no poll loop for this test
|
||||
..default_options()
|
||||
};
|
||||
let state = coordinator.register(target.clone(), options).await.unwrap();
|
||||
|
||||
assert_eq!(state.current.load().generation, TargetTlsGeneration(1));
|
||||
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn register_disabled_returns_error() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(MockTarget::new("test:webhook"));
|
||||
|
||||
let options = TlsReloadOptions {
|
||||
enabled: false,
|
||||
..default_options()
|
||||
};
|
||||
let result = coordinator.register(target, options).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_stops_all_loops() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(MockTarget::new("test:webhook"));
|
||||
|
||||
let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
|
||||
assert_eq!(coordinator.entries.read().await.len(), 1);
|
||||
|
||||
coordinator.shutdown().await;
|
||||
assert!(coordinator.entries.read().await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_reload_calls_build_and_apply() {
|
||||
let target = MockTarget::new("test:webhook");
|
||||
let initial_material = Arc::new("initial".to_string());
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(1),
|
||||
fingerprint: TargetTlsFingerprint::default(),
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: 0,
|
||||
});
|
||||
let inputs = target.tls_input_set();
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, inputs));
|
||||
let options = default_options();
|
||||
|
||||
// Force reload should succeed since MockTarget uses empty paths
|
||||
// and the fingerprint won't change from default
|
||||
let result = reload_target_once(&target, &runtime_state, &options).await.unwrap();
|
||||
// Since fingerprint is unchanged (empty paths), generation stays at 1
|
||||
assert_eq!(result, TargetTlsGeneration(1));
|
||||
// Build should NOT be called because fingerprint unchanged
|
||||
assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_failure_preserves_old_generation() {
|
||||
let target = MockTarget::new("test:webhook");
|
||||
target.should_fail_build.store(true, Ordering::SeqCst);
|
||||
|
||||
// Use a non-default fingerprint so the reload will detect a change
|
||||
// (empty paths → default fingerprint ≠ initial fingerprint)
|
||||
let initial_material = Arc::new("initial".to_string());
|
||||
let initial_fingerprint = TargetTlsFingerprint {
|
||||
ca_sha256: Some([1; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(1),
|
||||
fingerprint: initial_fingerprint,
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: 0,
|
||||
});
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
|
||||
let options = default_options();
|
||||
|
||||
// Empty paths produce default fingerprint which differs from initial →
|
||||
// validate passes (empty paths), then build is called and fails.
|
||||
let result = reload_target_once(&target, &runtime_state, &options).await;
|
||||
assert!(result.is_err());
|
||||
// Generation should remain at 1
|
||||
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
|
||||
assert_eq!(target.build_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn status_snapshot_reflects_state() {
|
||||
let target = MockTarget::new("test:webhook");
|
||||
let initial_material = Arc::new("initial".to_string());
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(1),
|
||||
fingerprint: TargetTlsFingerprint::default(),
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: 0,
|
||||
});
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
|
||||
let options = default_options();
|
||||
|
||||
let snapshot = TargetTlsReloadCoordinator::build_status_snapshot(runtime_state.as_ref(), &options);
|
||||
assert_eq!(snapshot.target_label, "test:webhook");
|
||||
assert_eq!(snapshot.generation, 1);
|
||||
assert!(snapshot.reload_enabled);
|
||||
assert_eq!(snapshot.detect_mode, "poll");
|
||||
assert_eq!(snapshot.apply_mode, "lazy");
|
||||
assert!(snapshot.last_attempt_time.is_none());
|
||||
assert!(snapshot.last_error.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn apply_failure_preserves_old_generation() {
|
||||
let target = MockTarget::new("test:webhook");
|
||||
target.should_fail_apply.store(true, Ordering::SeqCst);
|
||||
|
||||
// Use a non-default fingerprint so reload detects a change
|
||||
let initial_material = Arc::new("initial".to_string());
|
||||
let initial_fingerprint = TargetTlsFingerprint {
|
||||
ca_sha256: Some([42; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(3),
|
||||
fingerprint: initial_fingerprint,
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: 0,
|
||||
});
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
|
||||
let options = default_options();
|
||||
|
||||
let result = reload_target_once(&target, &runtime_state, &options).await;
|
||||
assert!(result.is_err());
|
||||
// Generation should remain at 3
|
||||
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(3));
|
||||
assert!(runtime_state.last_error.read().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_failure_prevents_build() {
|
||||
let target = MockTarget::new("test:kafka");
|
||||
target.should_fail_validate.store(true, Ordering::SeqCst);
|
||||
|
||||
// Non-default fingerprint to trigger reload
|
||||
let initial_material = Arc::new("initial".to_string());
|
||||
let initial_fingerprint = TargetTlsFingerprint {
|
||||
ca_sha256: Some([99; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(1),
|
||||
fingerprint: initial_fingerprint,
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: 0,
|
||||
});
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
|
||||
let options = default_options();
|
||||
|
||||
let result = reload_target_once(&target, &runtime_state, &options).await;
|
||||
assert!(result.is_err());
|
||||
// Build should NOT have been called
|
||||
assert_eq!(target.build_calls.load(Ordering::SeqCst), 0);
|
||||
// Validate was called
|
||||
assert!(target.validate_calls.load(Ordering::SeqCst) > 0);
|
||||
assert_eq!(runtime_state.current_generation(), TargetTlsGeneration(1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn error_is_cleared_on_successful_reload() {
|
||||
let target = MockTarget::new("test:nats");
|
||||
|
||||
let initial_material = Arc::new("initial".to_string());
|
||||
let initial_fingerprint = TargetTlsFingerprint {
|
||||
ca_sha256: Some([1; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(1),
|
||||
fingerprint: initial_fingerprint,
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: 0,
|
||||
});
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
|
||||
let options = default_options();
|
||||
|
||||
// First: fail the reload
|
||||
target.should_fail_build.store(true, Ordering::SeqCst);
|
||||
let _ = reload_target_once(&target, &runtime_state, &options).await;
|
||||
assert!(runtime_state.last_error.read().is_some());
|
||||
|
||||
// Now succeed (fingerprint still different from default)
|
||||
target.should_fail_build.store(false, Ordering::SeqCst);
|
||||
let result = reload_target_once(&target, &runtime_state, &options).await;
|
||||
assert!(result.is_ok());
|
||||
assert!(runtime_state.last_error.read().is_none());
|
||||
assert!(runtime_state.last_success_unix_ms() > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn last_good_is_never_overwritten_by_failed_reload() {
|
||||
let target = MockTarget::new("test:amqp");
|
||||
|
||||
let initial_material = Arc::new("good".to_string());
|
||||
let initial_fingerprint = TargetTlsFingerprint {
|
||||
ca_sha256: Some([5; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let initial_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(2),
|
||||
fingerprint: initial_fingerprint.clone(),
|
||||
material: initial_material.clone(),
|
||||
loaded_at_unix_ms: 100,
|
||||
});
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(initial_state, target.tls_input_set()));
|
||||
|
||||
// Verify last_good matches initial
|
||||
let good = runtime_state.last_good.load();
|
||||
assert_eq!(good.generation, TargetTlsGeneration(2));
|
||||
|
||||
// Fail a reload
|
||||
target.should_fail_build.store(true, Ordering::SeqCst);
|
||||
let _ = reload_target_once(&target, &runtime_state, &default_options()).await;
|
||||
|
||||
// last_good should still be the initial state
|
||||
let good_after = runtime_state.last_good.load();
|
||||
assert_eq!(good_after.generation, TargetTlsGeneration(2));
|
||||
assert_eq!(good_after.fingerprint, initial_fingerprint);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unregister_stops_target_poll_loop() {
|
||||
let coordinator = TargetTlsReloadCoordinator::new();
|
||||
let target = Arc::new(MockTarget::new("test:pulsar"));
|
||||
|
||||
let _state = coordinator.register(target.clone(), default_options()).await.unwrap();
|
||||
assert_eq!(coordinator.entries.read().await.len(), 1);
|
||||
|
||||
coordinator.unregister("test:pulsar").await.unwrap();
|
||||
assert!(coordinator.entries.read().await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bump_generation_saturates_at_max() {
|
||||
let target = MockTarget::new("test:saturation");
|
||||
let initial_material = Arc::new("initial".to_string());
|
||||
let max_gen_state = Arc::new(TargetTlsPublishedState {
|
||||
generation: TargetTlsGeneration(u64::MAX),
|
||||
fingerprint: TargetTlsFingerprint::default(),
|
||||
material: initial_material,
|
||||
loaded_at_unix_ms: 0,
|
||||
});
|
||||
let runtime_state = Arc::new(TargetTlsRuntimeState::new(max_gen_state, target.tls_input_set()));
|
||||
|
||||
let bumped = runtime_state.bump_generation();
|
||||
assert_eq!(bumped, TargetTlsGeneration(u64::MAX)); // saturating add
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! TLS fingerprint types for per-target certificate hot-reload detection.
|
||||
|
||||
use crate::error::TargetError;
|
||||
|
||||
/// SHA256 digest per TLS file component used to detect certificate changes.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct TargetTlsFingerprint {
|
||||
pub ca_sha256: Option<[u8; 32]>,
|
||||
pub client_cert_sha256: Option<[u8; 32]>,
|
||||
pub client_key_sha256: Option<[u8; 32]>,
|
||||
}
|
||||
|
||||
/// Monotonically increasing generation counter bumped on each successful reload.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct TargetTlsGeneration(pub u64);
|
||||
|
||||
/// Combined TLS state held per-target for tracking reload progress.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct TargetTlsState {
|
||||
pub generation: TargetTlsGeneration,
|
||||
pub fingerprint: Option<TargetTlsFingerprint>,
|
||||
}
|
||||
|
||||
impl TargetTlsState {
|
||||
/// Compares `next_fingerprint` with the current one. If different, bumps
|
||||
/// generation and stores the new fingerprint. Returns `true` when changed.
|
||||
pub fn refresh(&mut self, next_fingerprint: TargetTlsFingerprint) -> bool {
|
||||
if self.fingerprint.as_ref() == Some(&next_fingerprint) {
|
||||
return false;
|
||||
}
|
||||
|
||||
self.generation = TargetTlsGeneration(self.generation.0.saturating_add(1));
|
||||
self.fingerprint = Some(next_fingerprint);
|
||||
true
|
||||
}
|
||||
|
||||
/// Checks whether `candidate` differs from the stored fingerprint without
|
||||
/// mutating state. Use this to gate a rebuild, then call `refresh` only
|
||||
/// after the rebuild succeeds.
|
||||
pub fn needs_update(&self, candidate: &TargetTlsFingerprint) -> bool {
|
||||
self.fingerprint.as_ref() != Some(candidate)
|
||||
}
|
||||
|
||||
/// Resets state to default (generation 0, no fingerprint).
|
||||
pub fn reset(&mut self) {
|
||||
*self = Self::default();
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the three TLS material files from disk and returns a fingerprint
|
||||
/// computed from their SHA256 digests. Empty paths produce `None` digests.
|
||||
pub async fn build_target_tls_fingerprint(
|
||||
ca_path: &str,
|
||||
client_cert_path: &str,
|
||||
client_key_path: &str,
|
||||
) -> Result<TargetTlsFingerprint, TargetError> {
|
||||
async fn load_optional_digest(path: &str) -> Result<Option<[u8; 32]>, TargetError> {
|
||||
if path.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let bytes = tokio::fs::read(path)
|
||||
.await
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to read TLS material '{path}': {e}")))?;
|
||||
let digest = rustfs_tls_runtime::TlsFingerprint::from_optional_bytes(Some(&bytes), None, None, None, None).server_sha256;
|
||||
Ok(digest)
|
||||
}
|
||||
|
||||
Ok(TargetTlsFingerprint {
|
||||
ca_sha256: load_optional_digest(ca_path).await?,
|
||||
client_cert_sha256: load_optional_digest(client_cert_path).await?,
|
||||
client_key_sha256: load_optional_digest(client_key_path).await?,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState};
|
||||
|
||||
#[test]
|
||||
fn refresh_increments_generation_only_when_fingerprint_changes() {
|
||||
let mut state = TargetTlsState::default();
|
||||
let first = TargetTlsFingerprint {
|
||||
ca_sha256: Some([1; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let second = TargetTlsFingerprint {
|
||||
ca_sha256: Some([2; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
|
||||
assert!(state.refresh(first.clone()));
|
||||
assert_eq!(state.generation, TargetTlsGeneration(1));
|
||||
assert!(!state.refresh(first));
|
||||
assert_eq!(state.generation, TargetTlsGeneration(1));
|
||||
assert!(state.refresh(second));
|
||||
assert_eq!(state.generation, TargetTlsGeneration(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_generation_and_fingerprint() {
|
||||
let mut state = TargetTlsState {
|
||||
generation: TargetTlsGeneration(5),
|
||||
fingerprint: Some(TargetTlsFingerprint {
|
||||
ca_sha256: Some([9; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
}),
|
||||
};
|
||||
|
||||
state.reset();
|
||||
assert_eq!(state, TargetTlsState::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_eq_when_all_fields_match() {
|
||||
let a = TargetTlsFingerprint {
|
||||
ca_sha256: Some([42; 32]),
|
||||
client_cert_sha256: Some([1; 32]),
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let b = TargetTlsFingerprint {
|
||||
ca_sha256: Some([42; 32]),
|
||||
client_cert_sha256: Some([1; 32]),
|
||||
client_key_sha256: None,
|
||||
};
|
||||
assert_eq!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fingerprint_ne_when_ca_differs() {
|
||||
let a = TargetTlsFingerprint {
|
||||
ca_sha256: Some([1; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let b = TargetTlsFingerprint {
|
||||
ca_sha256: Some([2; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Target-level TLS reload metrics.
|
||||
|
||||
use ::metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram};
|
||||
|
||||
const TARGET_TLS_RELOAD_TOTAL: &str = "rustfs_target_tls_reload_total";
|
||||
const TARGET_TLS_RELOAD_SKIPPED_TOTAL: &str = "rustfs_target_tls_reload_skipped_total";
|
||||
const TARGET_TLS_GENERATION: &str = "rustfs_target_tls_generation";
|
||||
const TARGET_TLS_RELOAD_DURATION_SECONDS: &str = "rustfs_target_tls_reload_duration_seconds";
|
||||
const TARGET_TLS_PUBLICATION_FAIL_TOTAL: &str = "rustfs_target_tls_publication_fail_total";
|
||||
const TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL: &str = "rustfs_target_tls_active_generation_mismatch_total";
|
||||
|
||||
/// Describes all target TLS metrics. Call once during initialization.
|
||||
pub fn init_target_tls_metrics() {
|
||||
describe_counter!(TARGET_TLS_RELOAD_TOTAL, "Total number of TLS reload attempts per target");
|
||||
describe_counter!(
|
||||
TARGET_TLS_RELOAD_SKIPPED_TOTAL,
|
||||
"Number of TLS reloads skipped per target (unchanged, etc.)"
|
||||
);
|
||||
describe_gauge!(TARGET_TLS_GENERATION, "Current TLS generation per target");
|
||||
describe_histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "Duration of TLS reload attempts per target");
|
||||
describe_counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "Number of TLS reload publication failures per target");
|
||||
describe_counter!(
|
||||
TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL,
|
||||
"Number of times a target's active connection used a stale TLS generation"
|
||||
);
|
||||
}
|
||||
|
||||
/// Records a TLS reload result (success or failure).
|
||||
pub fn record_target_tls_reload_result(target: &str, result: &str, duration_secs: f64, generation: u64) {
|
||||
counter!(TARGET_TLS_RELOAD_TOTAL, "target_id" => target.to_string(), "result" => result.to_string()).increment(1);
|
||||
histogram!(TARGET_TLS_RELOAD_DURATION_SECONDS, "target_id" => target.to_string(), "result" => result.to_string())
|
||||
.record(duration_secs);
|
||||
gauge!(TARGET_TLS_GENERATION, "target_id" => target.to_string()).set(generation as f64);
|
||||
}
|
||||
|
||||
/// Records a skipped reload (typically because fingerprint was unchanged).
|
||||
pub fn record_target_tls_reload_skipped(target: &str, reason: &str) {
|
||||
counter!(TARGET_TLS_RELOAD_SKIPPED_TOTAL, "target_id" => target.to_string(), "reason" => reason.to_string()).increment(1);
|
||||
}
|
||||
|
||||
/// Records a TLS reload publication failure.
|
||||
pub fn record_target_tls_publication_fail(target: &str) {
|
||||
counter!(TARGET_TLS_PUBLICATION_FAIL_TOTAL, "target_id" => target.to_string()).increment(1);
|
||||
}
|
||||
|
||||
/// Records that a target used a stale TLS generation (active ≠ latest published).
|
||||
pub fn record_target_tls_stale_generation(target: &str) {
|
||||
counter!(TARGET_TLS_ACTIVE_GENERATION_MISMATCH_TOTAL, "target_id" => target.to_string()).increment(1);
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Unified TLS hot-reload infrastructure for notification targets.
|
||||
//!
|
||||
//! This module provides:
|
||||
//! - Fingerprint-based change detection (`fingerprint`)
|
||||
//! - Per-target reload configuration (`config`)
|
||||
//! - Runtime state tracking with atomic timestamps (`state`)
|
||||
//! - The `ReloadableTargetTls` trait protocol (`trait`)
|
||||
//! - TLS material validation helpers (`validate`)
|
||||
//! - The reload coordinator with background poll loops (`coordinator`)
|
||||
//! - Target-level reload metrics (`metrics`)
|
||||
|
||||
pub mod adapter;
|
||||
pub mod config;
|
||||
pub mod coordinator;
|
||||
pub mod fingerprint;
|
||||
pub mod metrics;
|
||||
pub mod state;
|
||||
pub mod r#trait;
|
||||
pub mod validate;
|
||||
|
||||
pub use adapter::TlsReloadAdapter;
|
||||
pub use coordinator::TargetTlsReloadCoordinator;
|
||||
pub use fingerprint::{TargetTlsFingerprint, TargetTlsGeneration, TargetTlsState, build_target_tls_fingerprint};
|
||||
pub use metrics::init_target_tls_metrics;
|
||||
pub use state::{TargetTlsInputSet, TargetTlsPublishedState, TargetTlsRuntimeState, TargetTlsStatusSnapshot};
|
||||
pub use r#trait::ReloadableTargetTls;
|
||||
pub use validate::validate_tls_material;
|
||||
@@ -0,0 +1,127 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Per-target TLS reload runtime state with atomic timestamps and error tracking.
|
||||
|
||||
use super::fingerprint::{TargetTlsFingerprint, TargetTlsGeneration};
|
||||
use ::arc_swap::ArcSwap;
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// Describes which TLS files a target reads.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TargetTlsInputSet {
|
||||
pub ca_path: String,
|
||||
pub client_cert_path: String,
|
||||
pub client_key_path: String,
|
||||
/// Human-readable label for logging and metrics (e.g. "webhook:primary").
|
||||
pub target_label: String,
|
||||
}
|
||||
|
||||
impl TargetTlsInputSet {
|
||||
/// Returns `true` when no TLS paths are configured (no CA, cert, or key).
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.ca_path.is_empty() && self.client_cert_path.is_empty() && self.client_key_path.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Immutable snapshot of a successfully published TLS material generation.
|
||||
pub struct TargetTlsPublishedState<M> {
|
||||
pub generation: TargetTlsGeneration,
|
||||
pub fingerprint: TargetTlsFingerprint,
|
||||
pub material: Arc<M>,
|
||||
pub loaded_at_unix_ms: u64,
|
||||
}
|
||||
|
||||
/// Per-target TLS reload runtime state. Owns the current published material
|
||||
/// and tracks timestamps and the last error for observability.
|
||||
pub struct TargetTlsRuntimeState<M> {
|
||||
/// The currently active TLS material generation.
|
||||
pub current: ArcSwap<TargetTlsPublishedState<M>>,
|
||||
/// The last known-good generation (never overwritten by a failed reload).
|
||||
pub last_good: ArcSwap<TargetTlsPublishedState<M>>,
|
||||
/// Unix-millis timestamp of the last reload *attempt* (success or failure).
|
||||
pub last_attempt_unix_ms: AtomicU64,
|
||||
/// Unix-millis timestamp of the last *successful* reload.
|
||||
pub last_success_unix_ms: AtomicU64,
|
||||
/// Last reload error message, if any.
|
||||
pub last_error: parking_lot::RwLock<Option<String>>,
|
||||
/// The TLS file paths this state watches.
|
||||
pub inputs: TargetTlsInputSet,
|
||||
}
|
||||
|
||||
impl<M> TargetTlsRuntimeState<M> {
|
||||
/// Creates a new runtime state with the given initial published state.
|
||||
pub fn new(initial: Arc<TargetTlsPublishedState<M>>, inputs: TargetTlsInputSet) -> Self {
|
||||
Self {
|
||||
current: ArcSwap::from(initial.clone()),
|
||||
last_good: ArcSwap::from(initial),
|
||||
last_attempt_unix_ms: AtomicU64::new(0),
|
||||
last_success_unix_ms: AtomicU64::new(0),
|
||||
last_error: parking_lot::RwLock::new(None),
|
||||
inputs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the generation of the currently active material.
|
||||
pub fn current_generation(&self) -> TargetTlsGeneration {
|
||||
self.current.load().generation
|
||||
}
|
||||
|
||||
/// Atomically bumps and returns the next generation.
|
||||
pub fn bump_generation(&self) -> TargetTlsGeneration {
|
||||
// Load the current generation from the arc-swap, compute next,
|
||||
// and return it. The caller is responsible for publishing the new state.
|
||||
let current = self.current.load();
|
||||
TargetTlsGeneration(current.generation.0.saturating_add(1))
|
||||
}
|
||||
|
||||
/// Records the timestamp of a reload attempt.
|
||||
pub fn mark_attempt(&self, unix_ms: u64) {
|
||||
self.last_attempt_unix_ms.store(unix_ms, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Records the timestamp of a successful reload.
|
||||
pub fn mark_success(&self, unix_ms: u64) {
|
||||
self.last_success_unix_ms.store(unix_ms, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Returns the last attempt timestamp.
|
||||
pub fn last_attempt_unix_ms(&self) -> u64 {
|
||||
self.last_attempt_unix_ms.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Returns the last success timestamp.
|
||||
pub fn last_success_unix_ms(&self) -> u64 {
|
||||
self.last_success_unix_ms.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only status snapshot for admin/debug visibility.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct TargetTlsStatusSnapshot {
|
||||
pub target_label: String,
|
||||
pub generation: u64,
|
||||
pub reload_enabled: bool,
|
||||
pub detect_mode: &'static str,
|
||||
pub apply_mode: &'static str,
|
||||
pub last_attempt_time: Option<u64>,
|
||||
pub last_success_time: Option<u64>,
|
||||
pub last_error: Option<String>,
|
||||
/// TLS file paths this target watches (for admin diagnostics).
|
||||
pub ca_path: String,
|
||||
pub client_cert_path: String,
|
||||
pub client_key_path: String,
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The `ReloadableTargetTls` trait — the public protocol each TLS-capable
|
||||
//! target implements to participate in coordinated hot-reload.
|
||||
|
||||
use crate::error::TargetError;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::config::ReloadApplyMode;
|
||||
use super::fingerprint::TargetTlsGeneration;
|
||||
use super::state::TargetTlsInputSet;
|
||||
|
||||
/// Protocol that each TLS-capable target implements so the reload coordinator
|
||||
/// can drive certificate hot-reload without knowing the target's internals.
|
||||
///
|
||||
/// The target is responsible for:
|
||||
/// - Declaring which TLS files it reads (`tls_input_set`)
|
||||
/// - Building a new client/pool/connector from current files (`build_tls_material`)
|
||||
/// - Atomically swapping the active connection state (`apply_tls_material`)
|
||||
///
|
||||
/// The coordinator is responsible for:
|
||||
/// - Deciding *when* to check
|
||||
/// - Detecting *whether* material changed
|
||||
/// - Ensuring *safety* (validate, build-then-apply, fallback on failure)
|
||||
#[async_trait]
|
||||
pub trait ReloadableTargetTls: Send + Sync + 'static {
|
||||
/// The rebuilt connection/client/pool object this target uses.
|
||||
type Material: Send + Sync + 'static;
|
||||
|
||||
/// Returns the TLS file paths this target reads.
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet;
|
||||
|
||||
/// Build a fresh TLS material object from current files on disk.
|
||||
///
|
||||
/// Called by the coordinator on the reload path only — never on the send hot path.
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError>;
|
||||
|
||||
/// Atomically apply new TLS material, replacing the current active connection state.
|
||||
///
|
||||
/// On success, the target's internal state must point to the new material.
|
||||
/// On failure, the target must keep its current state unchanged.
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError>;
|
||||
|
||||
/// Optional pre-check: validate that TLS files on disk are self-consistent
|
||||
/// (cert/key pair parseable, CA loadable) before attempting `build_tls_material`.
|
||||
/// Default implementation returns `Ok(())`.
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! TLS material validation helpers used by the reload coordinator before
|
||||
//! attempting to build new client/pool objects.
|
||||
|
||||
use crate::error::TargetError;
|
||||
use rustfs_tls_runtime::{load_certs, load_private_key};
|
||||
|
||||
/// Validates that a client certificate and private key file can be loaded
|
||||
/// and paired together. Returns `Ok(())` if both files parse successfully,
|
||||
/// or `Ok(())` if both paths are empty (no mTLS configured).
|
||||
pub fn validate_cert_key_pairing(cert_path: &str, key_path: &str) -> Result<(), TargetError> {
|
||||
if cert_path.is_empty() && key_path.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if cert_path.is_empty() || key_path.is_empty() {
|
||||
return Err(TargetError::Configuration(
|
||||
"Client certificate and key must both be specified or both be empty".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
load_certs(cert_path).map_err(|e| TargetError::Configuration(format!("Invalid client certificate '{cert_path}': {e}")))?;
|
||||
|
||||
load_private_key(key_path).map_err(|e| TargetError::Configuration(format!("Invalid client key '{key_path}': {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates that a CA certificate file can be loaded. Returns `Ok(())`
|
||||
/// if the path is empty (no custom CA) or if the file parses successfully.
|
||||
pub fn validate_ca_file(ca_path: &str) -> Result<(), TargetError> {
|
||||
if ca_path.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Invalid CA certificate '{ca_path}': {e}")))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates all three TLS material files in one call.
|
||||
pub fn validate_tls_material(ca_path: &str, cert_path: &str, key_path: &str) -> Result<(), TargetError> {
|
||||
validate_ca_file(ca_path)?;
|
||||
validate_cert_key_pairing(cert_path, key_path)
|
||||
}
|
||||
@@ -22,11 +22,15 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
|
||||
validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload_with_records, is_connectivity_error, open_target_queue_store,
|
||||
persist_queued_payload_to_store,
|
||||
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
|
||||
open_target_queue_store, persist_queued_payload_to_store,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -37,6 +41,7 @@ use lapin::{
|
||||
};
|
||||
use parking_lot::Mutex;
|
||||
use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY};
|
||||
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::fmt;
|
||||
@@ -196,16 +201,24 @@ async fn build_tls_config(args: &AMQPArgs) -> Result<OwnedTLSConfig, TargetError
|
||||
let cert_chain = if args.tls_ca.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
tokio::fs::read_to_string(&args.tls_ca)
|
||||
.await
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?,
|
||||
)
|
||||
let certs_der = load_cert_bundle_der_bytes(&args.tls_ca)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CA}: {e}")))?;
|
||||
if certs_der.is_empty() {
|
||||
return Err(TargetError::Configuration(format!(
|
||||
"{AMQP_TLS_CA} did not contain any parsable certificates"
|
||||
)));
|
||||
}
|
||||
let pem = tokio::fs::read_to_string(&args.tls_ca)
|
||||
.await
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?;
|
||||
Some(pem)
|
||||
};
|
||||
|
||||
let identity = if args.tls_client_cert.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let _ = load_cert_bundle_der_bytes(&args.tls_client_cert)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to parse {AMQP_TLS_CLIENT_CERT}: {e}")))?;
|
||||
let pem = tokio::fs::read(&args.tls_client_cert)
|
||||
.await
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_CERT}: {e}")))?;
|
||||
@@ -290,6 +303,9 @@ where
|
||||
id: TargetID,
|
||||
args: AMQPArgs,
|
||||
connection: Arc<Mutex<Option<Arc<AMQPConnection>>>>,
|
||||
tls_state: Arc<Mutex<TargetTlsState>>,
|
||||
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
|
||||
tls_adapter: Option<TlsReloadAdapter<AMQPConnection>>,
|
||||
connect_lock: Arc<AsyncMutex<()>>,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
delivery_counters: Arc<TargetDeliveryCounters>,
|
||||
@@ -305,6 +321,8 @@ where
|
||||
id: self.id.clone(),
|
||||
args: self.args.clone(),
|
||||
connection: Arc::clone(&self.connection),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
connect_lock: Arc::clone(&self.connect_lock),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
delivery_counters: Arc::clone(&self.delivery_counters),
|
||||
@@ -329,6 +347,8 @@ where
|
||||
id: target_id,
|
||||
args,
|
||||
connection: Arc::new(Mutex::new(None)),
|
||||
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
connect_lock: Arc::new(AsyncMutex::new(())),
|
||||
store: queue_store,
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
@@ -341,6 +361,27 @@ where
|
||||
}
|
||||
|
||||
async fn get_or_connect(&self) -> Result<Arc<AMQPConnection>, TargetError> {
|
||||
// When a TLS reload adapter is attached, it drives connection rebuilds
|
||||
// in the background. The inline per-send fingerprint check is skipped.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
let material = adapter.current_material();
|
||||
if material.connection.status().connected() && material.channel.status().connected() {
|
||||
return Ok(material);
|
||||
}
|
||||
self.clear_connection_handle();
|
||||
} else {
|
||||
let next_fingerprint =
|
||||
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock();
|
||||
tls_state_guard.needs_update(&next_fingerprint)
|
||||
};
|
||||
if tls_changed {
|
||||
self.clear_connection_handle();
|
||||
self.tls_state.lock().refresh(next_fingerprint);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(connection) = self.connection.lock().clone()
|
||||
&& connection.connection.status().connected()
|
||||
&& connection.channel.status().connected()
|
||||
@@ -362,10 +403,19 @@ where
|
||||
Ok(connection)
|
||||
}
|
||||
|
||||
fn clear_connection(&self) {
|
||||
fn clear_connection_handle(&self) {
|
||||
*self.connection.lock() = None;
|
||||
}
|
||||
|
||||
fn clear_connection_cache(&self) {
|
||||
self.clear_connection_handle();
|
||||
self.tls_state.lock().reset();
|
||||
}
|
||||
|
||||
fn clear_connection(&self) {
|
||||
self.clear_connection_cache();
|
||||
}
|
||||
|
||||
async fn send_body(&self, body: &[u8]) -> Result<(), TargetError> {
|
||||
let connection = self.get_or_connect().await?;
|
||||
let publish = connection
|
||||
@@ -407,6 +457,46 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for AMQP targets.
|
||||
///
|
||||
/// The coordinator calls these methods on a background poll loop to detect
|
||||
/// TLS file changes and rebuild the connection without restarting.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for AMQPTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = AMQPConnection;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls_ca.clone(),
|
||||
client_cert_path: self.args.tls_client_cert.clone(),
|
||||
client_key_path: self.args.tls_client_key.clone(),
|
||||
target_label: format!("amqp:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
connect_amqp(&self.args).await
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
let mut guard = self.connection.lock();
|
||||
*guard = Some(material);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<E> Target<E> for AMQPTarget<E>
|
||||
where
|
||||
@@ -458,6 +548,12 @@ where
|
||||
.await
|
||||
.map_err(|e| map_lapin_error(e, "Failed to close AMQP connection"))?;
|
||||
}
|
||||
self.tls_state.lock().reset();
|
||||
// If a TLS reload adapter is attached, reset its error tracking
|
||||
// so that a future re-init does not inherit stale failure state.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
*adapter.runtime_state().last_error.write() = None;
|
||||
}
|
||||
info!(target_id = %self.id, "AMQP target closed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -16,16 +16,21 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
|
||||
validate::validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, open_target_queue_store,
|
||||
persist_queued_payload_to_store,
|
||||
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
|
||||
open_target_queue_store, persist_queued_payload_to_store,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
|
||||
use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
|
||||
use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::{marker::PhantomData, sync::Arc, time::Duration};
|
||||
@@ -104,6 +109,11 @@ where
|
||||
args: KafkaArgs,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
|
||||
tls_state: Arc<Mutex<TargetTlsState>>,
|
||||
/// Adapter that bridges this target to the TLS reload coordinator.
|
||||
/// When `Some`, the target uses coordinator-managed material; when `None`,
|
||||
/// it falls back to inline fingerprint-based change detection.
|
||||
tls_adapter: Option<TlsReloadAdapter<Arc<AsyncProducer>>>,
|
||||
delivery_counters: Arc<TargetDeliveryCounters>,
|
||||
_phantom: PhantomData<E>,
|
||||
}
|
||||
@@ -144,6 +154,8 @@ where
|
||||
args,
|
||||
store: queue_store,
|
||||
producer: Arc::new(Mutex::new(None)),
|
||||
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
_phantom: PhantomData,
|
||||
})
|
||||
@@ -164,9 +176,27 @@ where
|
||||
if self.args.tls_enable {
|
||||
let mut security = SecurityConfig::new();
|
||||
if !self.args.tls_ca.is_empty() {
|
||||
let certs = load_cert_bundle_der_bytes(&self.args.tls_ca)
|
||||
.map_err(|e| Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_ca"))?;
|
||||
if certs.is_empty() {
|
||||
return Err(TargetError::Configuration(
|
||||
"Kafka tls_ca did not contain any parsable certificates".to_string(),
|
||||
));
|
||||
}
|
||||
security = security.with_ca_cert(self.args.tls_ca.clone());
|
||||
}
|
||||
if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
|
||||
let certs = load_cert_bundle_der_bytes(&self.args.tls_client_cert).map_err(|e| {
|
||||
Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_cert")
|
||||
})?;
|
||||
if certs.is_empty() {
|
||||
return Err(TargetError::Configuration(
|
||||
"Kafka tls_client_cert did not contain any parsable certificates".to_string(),
|
||||
));
|
||||
}
|
||||
let _ = load_private_key(&self.args.tls_client_key).map_err(|e| {
|
||||
Self::map_kafka_error(KafkaError::Config(e.to_string()), "Failed to parse Kafka tls_client_key")
|
||||
})?;
|
||||
security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone());
|
||||
}
|
||||
config = config.with_security(security);
|
||||
@@ -178,6 +208,31 @@ where
|
||||
}
|
||||
|
||||
async fn get_or_build_producer(&self) -> Result<Arc<AsyncProducer>, TargetError> {
|
||||
// Adapter-managed path: use the material directly from the TLS reload adapter.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
let producer: Arc<AsyncProducer> = (*adapter.current_material()).clone();
|
||||
|
||||
// Ensure the producer is also stored locally so that close() can drain it.
|
||||
{
|
||||
let mut guard = self.producer.lock().await;
|
||||
*guard = Some(Arc::clone(&producer));
|
||||
}
|
||||
return Ok(producer);
|
||||
}
|
||||
|
||||
// Inline fingerprint fallback path (no coordinator).
|
||||
let next_fingerprint =
|
||||
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock().await;
|
||||
tls_state_guard.needs_update(&next_fingerprint)
|
||||
};
|
||||
if tls_changed {
|
||||
let mut cached = self.producer.lock().await;
|
||||
*cached = None;
|
||||
self.tls_state.lock().await.refresh(next_fingerprint);
|
||||
}
|
||||
|
||||
let mut cached = self.producer.lock().await;
|
||||
if let Some(producer) = cached.as_ref() {
|
||||
return Ok(Arc::clone(producer));
|
||||
@@ -191,6 +246,7 @@ where
|
||||
async fn invalidate_cached_producer(&self) {
|
||||
let mut cached = self.producer.lock().await;
|
||||
*cached = None;
|
||||
self.tls_state.lock().await.reset();
|
||||
}
|
||||
|
||||
/// Serializes the event and builds a QueuedPayload
|
||||
@@ -230,6 +286,8 @@ where
|
||||
args: self.args.clone(),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
producer: Arc::clone(&self.producer),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
delivery_counters: Arc::clone(&self.delivery_counters),
|
||||
_phantom: PhantomData,
|
||||
})
|
||||
@@ -292,6 +350,13 @@ where
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
{
|
||||
let mut guard = self.producer.lock().await;
|
||||
*guard = None;
|
||||
}
|
||||
|
||||
self.tls_state.lock().await.reset();
|
||||
|
||||
info!("Kafka target closed: {}", self.id);
|
||||
Ok(())
|
||||
}
|
||||
@@ -318,6 +383,47 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for Kafka targets.
|
||||
///
|
||||
/// The coordinator calls these methods on a background poll loop to detect
|
||||
/// TLS file changes and rebuild the producer without restarting.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for KafkaTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = Arc<AsyncProducer>;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls_ca.clone(),
|
||||
client_cert_path: self.args.tls_client_cert.clone(),
|
||||
client_key_path: self.args.tls_client_key.clone(),
|
||||
target_label: format!("kafka:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
let producer = self.build_producer().await?;
|
||||
Ok(Arc::new(producer))
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
let mut guard = self.producer.lock().await;
|
||||
*guard = Some((*material).clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -37,6 +37,13 @@ pub mod pulsar;
|
||||
pub mod redis;
|
||||
pub mod webhook;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsFingerprint as TargetTlsFingerprintState;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsGeneration;
|
||||
pub(crate) use crate::runtime::tls::fingerprint::TargetTlsState;
|
||||
pub(crate) use crate::runtime::tls::fingerprint::build_target_tls_fingerprint;
|
||||
|
||||
/// A read-only snapshot of delivery counters for a target.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct TargetDeliverySnapshot {
|
||||
@@ -531,6 +538,48 @@ pub(crate) fn ensure_rustls_provider_installed() {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tls_state_tests {
|
||||
use super::{TargetTlsFingerprintState, TargetTlsGeneration, TargetTlsState};
|
||||
|
||||
#[test]
|
||||
fn refresh_increments_generation_only_when_fingerprint_changes() {
|
||||
let mut state = TargetTlsState::default();
|
||||
let first = TargetTlsFingerprintState {
|
||||
ca_sha256: Some([1; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
let second = TargetTlsFingerprintState {
|
||||
ca_sha256: Some([2; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
};
|
||||
|
||||
assert!(state.refresh(first.clone()));
|
||||
assert_eq!(state.generation, TargetTlsGeneration(1));
|
||||
assert!(!state.refresh(first));
|
||||
assert_eq!(state.generation, TargetTlsGeneration(1));
|
||||
assert!(state.refresh(second));
|
||||
assert_eq!(state.generation, TargetTlsGeneration(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_generation_and_fingerprint() {
|
||||
let mut state = TargetTlsState {
|
||||
generation: TargetTlsGeneration(5),
|
||||
fingerprint: Some(TargetTlsFingerprintState {
|
||||
ca_sha256: Some([9; 32]),
|
||||
client_cert_sha256: None,
|
||||
client_key_sha256: None,
|
||||
}),
|
||||
};
|
||||
|
||||
state.reset();
|
||||
assert_eq!(state, TargetTlsState::default());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -16,6 +16,10 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TargetTlsState, TlsReloadAdapter, config::ReloadApplyMode,
|
||||
validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
@@ -23,6 +27,7 @@ use crate::{
|
||||
persist_queued_payload_to_store,
|
||||
},
|
||||
};
|
||||
use arc_swap::ArcSwap;
|
||||
use async_trait::async_trait;
|
||||
use hyper_rustls::ConfigBuilderExt;
|
||||
use rumqttc::{
|
||||
@@ -32,6 +37,7 @@ use rumqttc::{
|
||||
use rustfs_config::{
|
||||
EnableState, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_WS_PATH_ALLOWLIST,
|
||||
};
|
||||
use rustfs_tls_runtime::{load_certs, load_private_key};
|
||||
use rustls::ClientConfig;
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
@@ -185,8 +191,7 @@ fn validate_path_is_absolute(path: &str, field: &str) -> Result<(), TargetError>
|
||||
}
|
||||
|
||||
fn build_root_store(ca_path: &str, trust_leaf_as_ca: bool) -> Result<rustls::RootCertStore, TargetError> {
|
||||
let certs =
|
||||
rustfs_utils::load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
|
||||
let certs = load_certs(ca_path).map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_ca: {e}")))?;
|
||||
let mut store = rustls::RootCertStore::empty();
|
||||
|
||||
if trust_leaf_as_ca {
|
||||
@@ -222,9 +227,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
|
||||
if tls.client_cert_path.is_empty() {
|
||||
builder.with_no_client_auth()
|
||||
} else {
|
||||
let certs = rustfs_utils::load_certs(&tls.client_cert_path)
|
||||
let certs = load_certs(&tls.client_cert_path)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
|
||||
let key = rustfs_utils::load_private_key(&tls.client_key_path)
|
||||
let key = load_private_key(&tls.client_key_path)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
|
||||
builder
|
||||
.with_client_auth_cert(certs, key)
|
||||
@@ -237,9 +242,9 @@ fn build_mqtt_tls_transport(broker: &Url, tls: &MQTTTlsConfig) -> Result<Transpo
|
||||
if tls.client_cert_path.is_empty() {
|
||||
builder.with_no_client_auth()
|
||||
} else {
|
||||
let certs = rustfs_utils::load_certs(&tls.client_cert_path)
|
||||
let certs = load_certs(&tls.client_cert_path)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_cert: {e}")))?;
|
||||
let key = rustfs_utils::load_private_key(&tls.client_key_path)
|
||||
let key = load_private_key(&tls.client_key_path)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load MQTT tls_client_key: {e}")))?;
|
||||
builder
|
||||
.with_client_auth_cert(certs, key)
|
||||
@@ -490,6 +495,12 @@ where
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
connected: Arc<AtomicBool>,
|
||||
bg_task_manager: Arc<BgTaskManager>,
|
||||
/// TLS fingerprint tracking for inline fallback path.
|
||||
tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
|
||||
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
|
||||
tls_adapter: Option<TlsReloadAdapter<MqttOptions>>,
|
||||
/// Updated MqttOptions from coordinator for use on next reconnection.
|
||||
pending_mqtt_options: Arc<ArcSwap<MqttOptions>>,
|
||||
delivery_counters: Arc<TargetDeliveryCounters>,
|
||||
_phantom: PhantomData<E>,
|
||||
}
|
||||
@@ -519,6 +530,17 @@ where
|
||||
initial_cancel_rx: Mutex::new(Some(cancel_rx)),
|
||||
});
|
||||
|
||||
// Build the initial MqttOptions for TLS reload support.
|
||||
let initial_mqtt_options = build_mqtt_options(
|
||||
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
|
||||
&args.broker,
|
||||
Some(args.username.as_str()),
|
||||
Some(args.password.as_str()),
|
||||
&args.tls,
|
||||
args.keep_alive,
|
||||
Some(MAX_MQTT_PACKET_SIZE_BYTES),
|
||||
)?;
|
||||
|
||||
info!(target_id = %target_id, "MQTT target created");
|
||||
Ok(MQTTTarget::<E> {
|
||||
id: target_id,
|
||||
@@ -527,6 +549,9 @@ where
|
||||
store: queue_store,
|
||||
connected: Arc::new(AtomicBool::new(false)),
|
||||
bg_task_manager,
|
||||
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
pending_mqtt_options: Arc::new(ArcSwap::from(Arc::new(initial_mqtt_options))),
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
_phantom: PhantomData,
|
||||
})
|
||||
@@ -544,20 +569,15 @@ where
|
||||
let connected_arc = Arc::clone(&self.connected);
|
||||
let target_id_clone = self.id.clone();
|
||||
let args_clone = self.args.clone();
|
||||
let pending_mqtt_options = Arc::clone(&self.pending_mqtt_options);
|
||||
|
||||
let _ = bg_task_manager
|
||||
.init_cell
|
||||
.get_or_try_init(|| async {
|
||||
debug!(target_id = %target_id_clone, "Initializing MQTT background task.");
|
||||
let mqtt_options = build_mqtt_options(
|
||||
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
|
||||
&args_clone.broker,
|
||||
Some(args_clone.username.as_str()),
|
||||
Some(args_clone.password.as_str()),
|
||||
&args_clone.tls,
|
||||
args_clone.keep_alive,
|
||||
Some(MAX_MQTT_PACKET_SIZE_BYTES),
|
||||
)?;
|
||||
|
||||
// Use the latest MqttOptions (may have been updated by TLS reload coordinator).
|
||||
let mqtt_options: MqttOptions = (**pending_mqtt_options.load()).clone();
|
||||
|
||||
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
|
||||
|
||||
@@ -662,12 +682,66 @@ where
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
connected: self.connected.clone(),
|
||||
bg_task_manager: self.bg_task_manager.clone(),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
pending_mqtt_options: Arc::clone(&self.pending_mqtt_options),
|
||||
delivery_counters: self.delivery_counters.clone(),
|
||||
_phantom: PhantomData,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for MQTT targets.
|
||||
///
|
||||
/// MQTT uses `MqttOptions` as the material type. The coordinator rebuilds
|
||||
/// `MqttOptions` on TLS file changes, and `apply_tls_material` stores it in
|
||||
/// an `ArcSwap` for use on the next reconnection. The running event loop is
|
||||
/// not interrupted; rumqttc handles reconnection internally.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for MQTTTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = MqttOptions;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls.ca_path.clone(),
|
||||
client_cert_path: self.args.tls.client_cert_path.clone(),
|
||||
client_key_path: self.args.tls.client_key_path.clone(),
|
||||
target_label: format!("mqtt:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
build_mqtt_options(
|
||||
format!("rustfs_notify_{}", uuid::Uuid::new_v4()),
|
||||
&self.args.broker,
|
||||
Some(self.args.username.as_str()),
|
||||
Some(self.args.password.as_str()),
|
||||
&self.args.tls,
|
||||
self.args.keep_alive,
|
||||
Some(MAX_MQTT_PACKET_SIZE_BYTES),
|
||||
)
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
// Store the new MqttOptions for use on next reconnection.
|
||||
// The running event loop is not interrupted; rumqttc handles reconnection.
|
||||
self.pending_mqtt_options.store(material);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_mqtt_event_loop(
|
||||
mut eventloop: EventLoop,
|
||||
connected_status: Arc<AtomicBool>,
|
||||
@@ -968,6 +1042,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
self.tls_state.lock().reset();
|
||||
// If a TLS reload adapter is attached, reset its error tracking
|
||||
// so that a future re-init does not inherit stale failure state.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
*adapter.runtime_state().last_error.write() = None;
|
||||
}
|
||||
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
info!(target_id = %self.id, "MQTT target close method finished.");
|
||||
Ok(())
|
||||
|
||||
@@ -16,6 +16,10 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
|
||||
validate::validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
@@ -26,6 +30,7 @@ use crate::{
|
||||
use async_trait::async_trait;
|
||||
use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable};
|
||||
use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY};
|
||||
use rustfs_tls_runtime::{load_certs, load_private_key};
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::marker::PhantomData;
|
||||
@@ -478,6 +483,11 @@ where
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
/// Lazily-initialized MySQL connection pool
|
||||
pool: Arc<Mutex<Option<Pool>>>,
|
||||
/// TLS fingerprint tracking for hot reload (inline fallback path)
|
||||
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
|
||||
/// When present, the adapter provides coordinator-managed TLS material;
|
||||
/// otherwise the inline fingerprint path is used as a fallback.
|
||||
tls_adapter: Option<TlsReloadAdapter<Pool>>,
|
||||
/// Success/failure counters exposed via `delivery_snapshot`
|
||||
delivery_counters: Arc<TargetDeliveryCounters>,
|
||||
/// Zero-sized marker for the event type `E`
|
||||
@@ -489,6 +499,9 @@ where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
/// Creates a new MySqlTarget.
|
||||
///
|
||||
/// The target starts without a TLS reload coordinator. Use
|
||||
/// `TlsReloadAdapter::try_register` to opt into coordinated TLS hot-reload.
|
||||
pub fn new(id: String, args: MySqlArgs) -> Result<Self, TargetError> {
|
||||
args.validate()?;
|
||||
|
||||
@@ -511,6 +524,8 @@ where
|
||||
store: queue_store,
|
||||
// Pool is lazily initialized on first use to avoid unnecessary connections at startup and allow for better error handling
|
||||
pool: Arc::new(Mutex::new(None)),
|
||||
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
_phantom: PhantomData,
|
||||
})
|
||||
@@ -518,6 +533,10 @@ where
|
||||
|
||||
/// Returns or lazily initializes the MySQL connection pool.
|
||||
///
|
||||
/// When `tls_adapter` is present (coordinator-managed), the pool
|
||||
/// is sourced from the coordinator's published material.
|
||||
/// Otherwise, the inline fingerprint-based path is used as a fallback.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// | Scenario | Error variant |
|
||||
@@ -528,6 +547,31 @@ where
|
||||
/// | Existing table has incompatible schema | `Initialization` |
|
||||
/// | DSN parse failure / invalid config | `Configuration` |
|
||||
async fn get_or_init_pool(&self) -> Result<Pool, TargetError> {
|
||||
// Adapter-managed path: use the material directly from the coordinator.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
let pool: Pool = (*adapter.current_material()).clone();
|
||||
|
||||
// Ensure the pool is also stored locally so that close() can drain it.
|
||||
{
|
||||
let mut guard = self.pool.lock().await;
|
||||
*guard = Some(pool.clone());
|
||||
}
|
||||
return Ok(pool);
|
||||
}
|
||||
|
||||
// Inline fingerprint fallback path (no coordinator).
|
||||
let next_fingerprint =
|
||||
super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock();
|
||||
tls_state_guard.needs_update(&next_fingerprint)
|
||||
};
|
||||
if tls_changed {
|
||||
let mut guard = self.pool.lock().await;
|
||||
*guard = None;
|
||||
self.tls_state.lock().refresh(next_fingerprint);
|
||||
}
|
||||
|
||||
{
|
||||
let guard = self.pool.lock().await;
|
||||
if let Some(pool) = guard.as_ref() {
|
||||
@@ -535,68 +579,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
let dsn = MySqlDsn::parse(&self.args.dsn_string)?;
|
||||
|
||||
let mut builder = OptsBuilder::default()
|
||||
.user(Some(dsn.user.clone()))
|
||||
.pass(Some(dsn.password.clone()))
|
||||
.ip_or_hostname(dsn.host.clone())
|
||||
.tcp_port(dsn.port)
|
||||
.db_name(Some(dsn.database.clone()));
|
||||
|
||||
if dsn.tls {
|
||||
super::ensure_rustls_provider_installed();
|
||||
let mut ssl_opts = SslOpts::default();
|
||||
if !self.args.tls_ca.is_empty() {
|
||||
ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(self.args.tls_ca.clone()).into()]);
|
||||
}
|
||||
if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
|
||||
let identity = mysql_async::ClientIdentity::new(
|
||||
PathBuf::from(self.args.tls_client_cert.clone()).into(),
|
||||
PathBuf::from(self.args.tls_client_key.clone()).into(),
|
||||
);
|
||||
ssl_opts = ssl_opts.with_client_identity(Some(identity));
|
||||
}
|
||||
builder = builder.ssl_opts(Some(ssl_opts));
|
||||
} else {
|
||||
warn!(
|
||||
"MySQL target '{}' is configured without TLS. This is insecure and should not be used in production.",
|
||||
self.id
|
||||
);
|
||||
}
|
||||
|
||||
// When max_open_connections is 0, no explicit upper bound is set —
|
||||
// mysql_async uses its default pool constraints (10–100).
|
||||
if self.args.max_open_connections > 0 {
|
||||
let constraints = PoolConstraints::new(1, self.args.max_open_connections).ok_or_else(|| {
|
||||
TargetError::Configuration(format!(
|
||||
"MySQL max_open_connections must be >= 1, got {}",
|
||||
self.args.max_open_connections
|
||||
))
|
||||
})?;
|
||||
builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
|
||||
}
|
||||
|
||||
let opts = Opts::from(builder);
|
||||
let pool = Pool::new(opts);
|
||||
|
||||
// Uses a double-check pattern: the mutex guard is only held for
|
||||
// short reads/writes to the pool cache. All I/O (connecting,
|
||||
// DDL, schema validation) happens outside the lock so that
|
||||
// concurrent callers are not blocked by a slow MySQL server.
|
||||
let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
|
||||
|
||||
conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
|
||||
|
||||
let ddl = format!(
|
||||
"CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
|
||||
quote_table_name(&self.args.table)?
|
||||
);
|
||||
conn.query_drop(ddl)
|
||||
.await
|
||||
.map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
|
||||
|
||||
validate_existing_schema(&mut conn, &self.args.table).await?;
|
||||
let pool = build_mysql_pool_from_args(&self.args).await?;
|
||||
|
||||
// Double-check: another caller may have initialized the pool
|
||||
// while we were doing I/O.
|
||||
@@ -653,12 +636,87 @@ where
|
||||
args: self.args.clone(),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
pool: Arc::clone(&self.pool),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
delivery_counters: Arc::clone(&self.delivery_counters),
|
||||
_phantom: PhantomData,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a MySQL connection pool from the given args, including TLS setup,
|
||||
/// DDL table creation, and schema validation.
|
||||
///
|
||||
/// This is a standalone function so it can be called both from
|
||||
/// `get_or_init_pool` (inline fallback) and from `build_tls_material`
|
||||
/// (coordinator path).
|
||||
async fn build_mysql_pool_from_args(args: &MySqlArgs) -> Result<Pool, TargetError> {
|
||||
let dsn = MySqlDsn::parse(&args.dsn_string)?;
|
||||
|
||||
let mut builder = OptsBuilder::default()
|
||||
.user(Some(dsn.user.clone()))
|
||||
.pass(Some(dsn.password.clone()))
|
||||
.ip_or_hostname(dsn.host.clone())
|
||||
.tcp_port(dsn.port)
|
||||
.db_name(Some(dsn.database.clone()));
|
||||
|
||||
if dsn.tls {
|
||||
super::ensure_rustls_provider_installed();
|
||||
let mut ssl_opts = SslOpts::default();
|
||||
if !args.tls_ca.is_empty() {
|
||||
let _ =
|
||||
load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_ca: {e}")))?;
|
||||
ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(args.tls_ca.clone()).into()]);
|
||||
}
|
||||
if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
|
||||
let _ = load_certs(&args.tls_client_cert)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_cert: {e}")))?;
|
||||
let _ = load_private_key(&args.tls_client_key)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load MySQL tls_client_key: {e}")))?;
|
||||
let identity = mysql_async::ClientIdentity::new(
|
||||
PathBuf::from(args.tls_client_cert.clone()).into(),
|
||||
PathBuf::from(args.tls_client_key.clone()).into(),
|
||||
);
|
||||
ssl_opts = ssl_opts.with_client_identity(Some(identity));
|
||||
}
|
||||
builder = builder.ssl_opts(Some(ssl_opts));
|
||||
} else {
|
||||
warn!("MySQL target is configured without TLS. This is insecure and should not be used in production.");
|
||||
}
|
||||
|
||||
// When max_open_connections is 0, no explicit upper bound is set —
|
||||
// mysql_async uses its default pool constraints (10–100).
|
||||
if args.max_open_connections > 0 {
|
||||
let constraints = PoolConstraints::new(1, args.max_open_connections).ok_or_else(|| {
|
||||
TargetError::Configuration(format!("MySQL max_open_connections must be >= 1, got {}", args.max_open_connections))
|
||||
})?;
|
||||
builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints));
|
||||
}
|
||||
|
||||
let opts = Opts::from(builder);
|
||||
let pool = Pool::new(opts);
|
||||
|
||||
// Uses a double-check pattern: the mutex guard is only held for
|
||||
// short reads/writes to the pool cache. All I/O (connecting,
|
||||
// DDL, schema validation) happens outside the lock so that
|
||||
// concurrent callers are not blocked by a slow MySQL server.
|
||||
let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?;
|
||||
|
||||
conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?;
|
||||
|
||||
let ddl = format!(
|
||||
"CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)",
|
||||
quote_table_name(&args.table)?
|
||||
);
|
||||
conn.query_drop(ddl)
|
||||
.await
|
||||
.map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?;
|
||||
|
||||
validate_existing_schema(&mut conn, &args.table).await?;
|
||||
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
/// Maps a mysql_async error to `TargetError`:
|
||||
/// - `Io`/`Driver` → `NotConnected` (connection lost, fixed-delay retry)
|
||||
/// - `Server(1213|1205|1040)` → `Timeout` (deadlock/lock timeout/too
|
||||
@@ -793,6 +851,8 @@ where
|
||||
.map_err(|err| TargetError::Network(format!("Failed to disconnect MySQL pool: {err}")))?;
|
||||
}
|
||||
|
||||
// Adapter cleanup is done by the coordinator; no local state to reset.
|
||||
|
||||
info!("MySQL target closed: {}", self.id);
|
||||
Ok(())
|
||||
}
|
||||
@@ -828,6 +888,46 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for MySQL targets.
|
||||
///
|
||||
/// The coordinator calls these methods on a background poll loop to detect
|
||||
/// TLS file changes and rebuild the connection pool without restarting.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for MySqlTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = Pool;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls_ca.clone(),
|
||||
client_cert_path: self.args.tls_client_cert.clone(),
|
||||
client_key_path: self.args.tls_client_key.clone(),
|
||||
target_label: format!("mysql:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
build_mysql_pool_from_args(&self.args).await
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
let mut guard = self.pool.lock().await;
|
||||
*guard = Some((*material).clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -16,10 +16,15 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
|
||||
validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
|
||||
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
|
||||
persist_queued_payload_to_store,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -28,9 +33,10 @@ use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing::{info, instrument};
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{info, instrument, warn};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NATSArgs {
|
||||
@@ -168,7 +174,12 @@ where
|
||||
{
|
||||
id: TargetID,
|
||||
args: NATSArgs,
|
||||
client: Mutex<Option<async_nats::Client>>,
|
||||
client: Arc<Mutex<Option<async_nats::Client>>>,
|
||||
tls_state: Arc<parking_lot::Mutex<TargetTlsState>>,
|
||||
/// Adapter that bridges this target to the TLS reload coordinator.
|
||||
/// When `Some`, the target uses coordinator-managed material; when `None`,
|
||||
/// it falls back to inline fingerprint-based change detection.
|
||||
tls_adapter: Option<TlsReloadAdapter<async_nats::Client>>,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
connected: AtomicBool,
|
||||
delivery_counters: Arc<TargetDeliveryCounters>,
|
||||
@@ -183,7 +194,9 @@ where
|
||||
Box::new(NATSTarget::<E> {
|
||||
id: self.id.clone(),
|
||||
args: self.args.clone(),
|
||||
client: Mutex::new(self.client.lock().unwrap().clone()),
|
||||
client: Arc::clone(&self.client),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
|
||||
delivery_counters: Arc::clone(&self.delivery_counters),
|
||||
@@ -207,7 +220,9 @@ where
|
||||
Ok(Self {
|
||||
id: target_id,
|
||||
args,
|
||||
client: Mutex::new(None),
|
||||
client: Arc::new(Mutex::new(None)),
|
||||
tls_state: Arc::new(parking_lot::Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
store: queue_store,
|
||||
connected: AtomicBool::new(false),
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
@@ -215,11 +230,42 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
async fn invalidate_cached_client_connection(&self) {
|
||||
*self.client.lock().await = None;
|
||||
}
|
||||
|
||||
async fn get_or_connect(&self) -> Result<async_nats::Client, TargetError> {
|
||||
if let Some(client) = self.client.lock().unwrap().clone() {
|
||||
// Adapter-managed path: use the material directly from the TLS reload adapter.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
let client: async_nats::Client = (*adapter.current_material()).clone();
|
||||
|
||||
// Ensure the client is also stored locally so that close() can drain it.
|
||||
{
|
||||
let mut guard = self.client.lock().await;
|
||||
*guard = Some(client.clone());
|
||||
}
|
||||
return Ok(client);
|
||||
}
|
||||
|
||||
// Inline fingerprint fallback path (no coordinator).
|
||||
let next_fingerprint =
|
||||
build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key).await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock();
|
||||
tls_state_guard.needs_update(&next_fingerprint)
|
||||
};
|
||||
if tls_changed {
|
||||
self.invalidate_cached_client_connection().await;
|
||||
self.tls_state.lock().refresh(next_fingerprint);
|
||||
}
|
||||
|
||||
{
|
||||
let guard = self.client.lock().await;
|
||||
if let Some(client) = guard.as_ref() {
|
||||
return Ok(client.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let client = connect_nats(&self.args).await?;
|
||||
client
|
||||
.flush()
|
||||
@@ -227,7 +273,7 @@ where
|
||||
.map_err(|e| TargetError::Network(format!("Failed to flush NATS connection: {e}")))?;
|
||||
self.connected.store(true, Ordering::SeqCst);
|
||||
|
||||
let mut guard = self.client.lock().unwrap();
|
||||
let mut guard = self.client.lock().await;
|
||||
let shared = guard.get_or_insert_with(|| client.clone()).clone();
|
||||
Ok(shared)
|
||||
}
|
||||
@@ -294,7 +340,11 @@ where
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
let client = self.client.lock().unwrap().take();
|
||||
let client = {
|
||||
let mut guard = self.client.lock().await;
|
||||
guard.take()
|
||||
};
|
||||
self.tls_state.lock().reset();
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
if let Some(client) = client {
|
||||
client
|
||||
@@ -335,3 +385,87 @@ where
|
||||
self.delivery_counters.record_final_failure();
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for NATS targets.
|
||||
///
|
||||
/// The coordinator calls these methods on a background poll loop to detect
|
||||
/// TLS file changes and rebuild the NATS client without restarting.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for NATSTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = async_nats::Client;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls_ca.clone(),
|
||||
client_cert_path: self.args.tls_client_cert.clone(),
|
||||
client_key_path: self.args.tls_client_key.clone(),
|
||||
target_label: format!("nats:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
connect_nats(&self.args).await
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
let mut guard = self.client.lock().await;
|
||||
*guard = Some((*material).clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn base_args() -> NATSArgs {
|
||||
NATSArgs {
|
||||
enable: true,
|
||||
address: "nats://127.0.0.1:4222".to_string(),
|
||||
subject: "rustfs.events".to_string(),
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
token: String::new(),
|
||||
credentials_file: String::new(),
|
||||
tls_ca: String::new(),
|
||||
tls_client_cert: String::new(),
|
||||
tls_client_key: String::new(),
|
||||
tls_required: false,
|
||||
queue_dir: String::new(),
|
||||
queue_limit: 0,
|
||||
target_type: TargetType::NotifyEvent,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_rejects_multiple_auth_methods() {
|
||||
let args = NATSArgs {
|
||||
token: "abc".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
..base_args()
|
||||
};
|
||||
assert!(args.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_nats_rejects_relative_queue_dir() {
|
||||
let args = NATSArgs {
|
||||
queue_dir: "relative/path".to_string(),
|
||||
..base_args()
|
||||
};
|
||||
assert!(args.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,10 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
|
||||
validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
@@ -38,12 +42,10 @@ use crate::{
|
||||
use async_trait::async_trait;
|
||||
use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod};
|
||||
use rustfs_config::{POSTGRES_DSN_STRING, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY};
|
||||
use rustls_pki_types::pem::PemObject;
|
||||
use rustls_pki_types::{CertificateDer, PrivateKeyDer};
|
||||
use rustfs_tls_runtime::{load_certs, load_private_key};
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::fmt;
|
||||
use std::io::BufReader;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tokio_postgres::Config;
|
||||
@@ -425,11 +427,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
|
||||
let _ = root_store.add(cert);
|
||||
}
|
||||
} else {
|
||||
let pem = std::fs::read(&args.tls_ca)
|
||||
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CA}: {e}")))?;
|
||||
let mut reader = BufReader::new(pem.as_slice());
|
||||
for cert in CertificateDer::pem_reader_iter(&mut reader) {
|
||||
let cert = cert.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
|
||||
let certs =
|
||||
load_certs(&args.tls_ca).map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?;
|
||||
for cert in certs {
|
||||
root_store
|
||||
.add(cert)
|
||||
.map_err(|e| TargetError::Configuration(format!("failed to add CA cert: {e}")))?;
|
||||
@@ -439,16 +439,9 @@ pub fn build_tls_config(args: &PostgresArgs) -> Result<rustls::ClientConfig, Tar
|
||||
let builder = rustls::ClientConfig::builder().with_root_certificates(root_store);
|
||||
|
||||
let client_config = if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
|
||||
let cert_pem = std::fs::read(&args.tls_client_cert)
|
||||
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
|
||||
let key_pem = std::fs::read(&args.tls_client_key)
|
||||
.map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
|
||||
|
||||
let certs: Vec<_> = CertificateDer::pem_reader_iter(&mut BufReader::new(cert_pem.as_slice()))
|
||||
.collect::<Result<_, _>>()
|
||||
let certs = load_certs(&args.tls_client_cert)
|
||||
.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_CERT}: {e}")))?;
|
||||
|
||||
let key = PrivateKeyDer::from_pem_reader(&mut BufReader::new(key_pem.as_slice()))
|
||||
let key = load_private_key(&args.tls_client_key)
|
||||
.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_KEY}: {e}")))?;
|
||||
|
||||
builder
|
||||
@@ -548,13 +541,22 @@ fn resolve_payload_key(payload: &serde_json::Value, meta: &QueuedPayloadMeta) ->
|
||||
/// so that `clone_box` does not duplicate connection state. The optional
|
||||
/// `QueueStore` provides at-least-once delivery semantics consistent with the
|
||||
/// other built-in targets.
|
||||
///
|
||||
/// When `tls_adapter` is `Some`, the target participates in the
|
||||
/// coordinated TLS hot-reload system driven by `TlsReloadAdapter`,
|
||||
/// and the inline fingerprint check in `send_body` is skipped. When `None`,
|
||||
/// the legacy inline fingerprint check is used as a fallback.
|
||||
pub struct PostgresTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
id: TargetID,
|
||||
args: PostgresArgs,
|
||||
pool: Pool,
|
||||
pool: Arc<parking_lot::Mutex<Pool>>,
|
||||
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
|
||||
/// When present, the adapter provides coordinator-managed TLS material;
|
||||
/// otherwise the inline fingerprint path is used as a fallback.
|
||||
tls_adapter: Option<TlsReloadAdapter<Pool>>,
|
||||
namespace_sql: String,
|
||||
access_sql: String,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
@@ -570,7 +572,9 @@ where
|
||||
Box::new(PostgresTarget::<E> {
|
||||
id: self.id.clone(),
|
||||
args: self.args.clone(),
|
||||
pool: self.pool.clone(),
|
||||
pool: Arc::clone(&self.pool),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
namespace_sql: self.namespace_sql.clone(),
|
||||
access_sql: self.access_sql.clone(),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
@@ -599,7 +603,9 @@ where
|
||||
namespace_sql: namespace_upsert_sql(&args.schema, &args.table),
|
||||
access_sql: access_insert_sql(&args.schema, &args.table),
|
||||
args,
|
||||
pool,
|
||||
pool: Arc::new(parking_lot::Mutex::new(pool)),
|
||||
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
store: queue_store,
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
_phantom: std::marker::PhantomData,
|
||||
@@ -611,8 +617,25 @@ where
|
||||
/// Identifier validation has already happened in `PostgresArgs::validate()`,
|
||||
/// so `qualified_table` cannot produce a malformed SQL string here.
|
||||
async fn send_body(&self, body: &[u8], event_id: &str, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
|
||||
let client = self
|
||||
.pool
|
||||
// When a TLS reload adapter is attached, it drives pool rebuilds in
|
||||
// the background. The inline per-send fingerprint check is skipped.
|
||||
if self.tls_adapter.is_none() {
|
||||
let next_fingerprint =
|
||||
super::build_target_tls_fingerprint(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
|
||||
.await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock();
|
||||
tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
|
||||
};
|
||||
if tls_changed {
|
||||
let new_pool = build_pool(&self.args)?;
|
||||
*self.pool.lock() = new_pool;
|
||||
self.tls_state.lock().refresh(next_fingerprint);
|
||||
}
|
||||
}
|
||||
|
||||
let pool = self.pool.lock().clone();
|
||||
let client = pool
|
||||
.get()
|
||||
.await
|
||||
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
|
||||
@@ -645,8 +668,8 @@ where
|
||||
/// Probes the table from `init()`. Failure is non-fatal when a queue is
|
||||
/// configured: events buffer in the store until the schema is fixed.
|
||||
async fn probe_table(&self) -> Result<(), TargetError> {
|
||||
let client = self
|
||||
.pool
|
||||
let pool = self.pool.lock().clone();
|
||||
let client = pool
|
||||
.get()
|
||||
.await
|
||||
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed during init probe"))?;
|
||||
@@ -659,6 +682,41 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for PostgresTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = Pool;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls_ca.clone(),
|
||||
client_cert_path: self.args.tls_client_cert.clone(),
|
||||
client_key_path: self.args.tls_client_key.clone(),
|
||||
target_label: format!("postgres:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
build_pool(&self.args)
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
*self.pool.lock() = (*material).clone();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.tls_ca, &self.args.tls_client_cert, &self.args.tls_client_key)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<E> Target<E> for PostgresTarget<E>
|
||||
where
|
||||
@@ -674,8 +732,8 @@ where
|
||||
}
|
||||
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
let client = self
|
||||
.pool
|
||||
let pool = self.pool.lock().clone();
|
||||
let client = pool
|
||||
.get()
|
||||
.await
|
||||
.map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?;
|
||||
@@ -728,7 +786,8 @@ where
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
self.pool.close();
|
||||
self.pool.lock().close();
|
||||
// Adapter cleanup is done by the coordinator; no local state to reset.
|
||||
info!(target_id = %self.id, "PostgreSQL target closed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -16,14 +16,20 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
|
||||
validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
|
||||
TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
|
||||
persist_queued_payload_to_store,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use pulsar::{Authentication, Producer, Pulsar, TokioExecutor};
|
||||
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::path::Path;
|
||||
@@ -136,6 +142,13 @@ pub async fn connect_pulsar(args: &PulsarArgs) -> Result<Pulsar<TokioExecutor>,
|
||||
}
|
||||
|
||||
if !args.tls_ca.is_empty() {
|
||||
let certs = load_cert_bundle_der_bytes(&args.tls_ca)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to parse Pulsar tls_ca: {e}")))?;
|
||||
if certs.is_empty() {
|
||||
return Err(TargetError::Configuration(
|
||||
"Pulsar tls_ca did not contain any parsable certificates".to_string(),
|
||||
));
|
||||
}
|
||||
builder = builder
|
||||
.with_certificate_chain_file(&args.tls_ca)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load Pulsar tls_ca: {e}")))?;
|
||||
@@ -158,6 +171,9 @@ where
|
||||
id: TargetID,
|
||||
args: PulsarArgs,
|
||||
client: Mutex<Option<Pulsar<TokioExecutor>>>,
|
||||
tls_state: Mutex<TargetTlsState>,
|
||||
/// When set, the coordinator drives TLS reload; inline fingerprint check is skipped.
|
||||
tls_adapter: Option<TlsReloadAdapter<Pulsar<TokioExecutor>>>,
|
||||
producer: AsyncMutex<Option<Producer<TokioExecutor>>>,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
connected: AtomicBool,
|
||||
@@ -174,6 +190,8 @@ where
|
||||
id: self.id.clone(),
|
||||
args: self.args.clone(),
|
||||
client: Mutex::new(self.client.lock().unwrap().clone()),
|
||||
tls_state: Mutex::new(self.tls_state.lock().unwrap().clone()),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
producer: AsyncMutex::new(None),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
connected: AtomicBool::new(self.connected.load(Ordering::SeqCst)),
|
||||
@@ -199,6 +217,8 @@ where
|
||||
id: target_id,
|
||||
args,
|
||||
client: Mutex::new(None),
|
||||
tls_state: Mutex::new(TargetTlsState::default()),
|
||||
tls_adapter: None,
|
||||
producer: AsyncMutex::new(None),
|
||||
store: queue_store,
|
||||
connected: AtomicBool::new(false),
|
||||
@@ -207,7 +227,36 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
fn clear_cached_client_connection(&self) {
|
||||
self.client.lock().unwrap().take();
|
||||
}
|
||||
|
||||
fn clear_cached_client(&self) {
|
||||
self.clear_cached_client_connection();
|
||||
self.tls_state.lock().unwrap().reset();
|
||||
}
|
||||
|
||||
async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
|
||||
// When a TLS reload adapter is attached, it drives client rebuilds
|
||||
// in the background. The inline per-send fingerprint check is skipped.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
let material = adapter.current_material();
|
||||
{
|
||||
let mut guard = self.client.lock().unwrap();
|
||||
*guard = Some((*material).clone());
|
||||
}
|
||||
} else {
|
||||
let next_fingerprint = build_target_tls_fingerprint(&self.args.tls_ca, "", "").await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock().unwrap();
|
||||
tls_state_guard.needs_update(&next_fingerprint)
|
||||
};
|
||||
if tls_changed {
|
||||
self.clear_cached_client_connection();
|
||||
self.tls_state.lock().unwrap().refresh(next_fingerprint);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(client) = self.client.lock().unwrap().clone() {
|
||||
return Ok(client);
|
||||
}
|
||||
@@ -262,6 +311,56 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for Pulsar targets.
|
||||
///
|
||||
/// Pulsar only uses a CA certificate (no client cert/key).
|
||||
/// The coordinator calls these methods on a background poll loop to detect
|
||||
/// TLS file changes and rebuild the client without restarting.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for PulsarTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = Pulsar<TokioExecutor>;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls_ca.clone(),
|
||||
client_cert_path: String::new(),
|
||||
client_key_path: String::new(),
|
||||
target_label: format!("pulsar:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
connect_pulsar(&self.args).await
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
// Pulsar client is Clone, so we clone from the Arc and store it.
|
||||
{
|
||||
let mut guard = self.client.lock().unwrap();
|
||||
*guard = Some((*material).clone());
|
||||
}
|
||||
// Producer is bound to the old client; clear it so next send rebuilds.
|
||||
{
|
||||
let mut producer = self.producer.lock().await;
|
||||
*producer = None;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
// Pulsar only uses CA, no client cert/key.
|
||||
validate_tls_material(&self.args.tls_ca, "", "")
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<E> Target<E> for PulsarTarget<E>
|
||||
where
|
||||
@@ -321,8 +420,13 @@ where
|
||||
.map_err(|e| TargetError::Network(format!("Failed to close Pulsar producer: {e}")))?;
|
||||
}
|
||||
*producer = None;
|
||||
self.client.lock().unwrap().take();
|
||||
self.clear_cached_client();
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
// If a TLS reload adapter is attached, reset its error tracking
|
||||
// so that a future re-init does not inherit stale failure state.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
*adapter.runtime_state().last_error.write() = None;
|
||||
}
|
||||
info!(target_id = %self.id, "Pulsar target closed");
|
||||
Ok(())
|
||||
}
|
||||
@@ -355,3 +459,45 @@ where
|
||||
self.delivery_counters.record_final_failure();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn base_args() -> PulsarArgs {
|
||||
PulsarArgs {
|
||||
enable: true,
|
||||
broker: "pulsar://127.0.0.1:6650".to_string(),
|
||||
topic: "persistent://public/default/rustfs-events".to_string(),
|
||||
auth_token: String::new(),
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
tls_ca: String::new(),
|
||||
tls_allow_insecure: false,
|
||||
tls_hostname_verification: true,
|
||||
queue_dir: String::new(),
|
||||
queue_limit: 0,
|
||||
target_type: TargetType::NotifyEvent,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_pulsar_rejects_mixed_auth_methods() {
|
||||
let args = PulsarArgs {
|
||||
auth_token: "token".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
..base_args()
|
||||
};
|
||||
assert!(args.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_pulsar_rejects_relative_queue_dir() {
|
||||
let args = PulsarArgs {
|
||||
queue_dir: "relative/path".to_string(),
|
||||
..base_args()
|
||||
};
|
||||
assert!(args.validate().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,10 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsGeneration, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode,
|
||||
validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
@@ -31,9 +35,12 @@ use redis::{
|
||||
io::tcp::{TcpSettings, socket2},
|
||||
};
|
||||
use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY};
|
||||
use rustls::pki_types::CertificateDer;
|
||||
use rustls::pki_types::pem::PemObject;
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::fmt;
|
||||
use std::io::BufReader;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
@@ -297,7 +304,8 @@ where
|
||||
{
|
||||
id: TargetID,
|
||||
args: RedisArgs,
|
||||
publisher_client: Client,
|
||||
/// Redis client, wrapped in a lock so TLS hot-reload can atomically replace it.
|
||||
publisher_client: Arc<parking_lot::Mutex<Client>>,
|
||||
publisher: Arc<Mutex<Option<ConnectionManager>>>,
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
/// Business-level liveness flag.
|
||||
@@ -306,6 +314,12 @@ where
|
||||
/// publish exhausted retries, or the target was explicitly closed). Temporary reconnectable
|
||||
/// errors only invalidate the cached publisher so that a later request can lazily rebuild it.
|
||||
connected: Arc<AtomicBool>,
|
||||
/// TLS fingerprint tracking for hot reload (inline fallback path).
|
||||
tls_state: Arc<parking_lot::Mutex<super::TargetTlsState>>,
|
||||
/// Adapter that bridges this target to the TLS reload coordinator.
|
||||
/// When `Some`, the target uses coordinator-managed material; when `None`,
|
||||
/// it falls back to inline fingerprint-based change detection.
|
||||
tls_adapter: Option<TlsReloadAdapter<Client>>,
|
||||
delivery_counters: Arc<TargetDeliveryCounters>,
|
||||
_phantom: std::marker::PhantomData<E>,
|
||||
}
|
||||
@@ -334,10 +348,12 @@ where
|
||||
Ok(Self {
|
||||
id: target_id,
|
||||
args,
|
||||
publisher_client,
|
||||
publisher_client: Arc::new(parking_lot::Mutex::new(publisher_client)),
|
||||
publisher: Arc::new(Mutex::new(None)),
|
||||
store: queue_store,
|
||||
connected: Arc::new(AtomicBool::new(false)),
|
||||
tls_state: Arc::new(parking_lot::Mutex::new(super::TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
|
||||
_phantom: std::marker::PhantomData,
|
||||
})
|
||||
@@ -347,23 +363,61 @@ where
|
||||
Box::new(Self {
|
||||
id: self.id.clone(),
|
||||
args: self.args.clone(),
|
||||
publisher_client: self.publisher_client.clone(),
|
||||
publisher_client: Arc::clone(&self.publisher_client),
|
||||
publisher: Arc::clone(&self.publisher),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
connected: Arc::clone(&self.connected),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
delivery_counters: Arc::clone(&self.delivery_counters),
|
||||
_phantom: std::marker::PhantomData,
|
||||
})
|
||||
}
|
||||
|
||||
async fn get_or_create_publisher(&self) -> Result<ConnectionManager, TargetError> {
|
||||
// Adapter-managed path: use the material directly from the TLS reload adapter.
|
||||
if let Some(adapter) = &self.tls_adapter {
|
||||
let client: Client = (*adapter.current_material()).clone();
|
||||
|
||||
// Ensure the client is also stored locally so close() can drain it.
|
||||
*self.publisher_client.lock() = client.clone();
|
||||
|
||||
let manager = client
|
||||
.get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
|
||||
.map_err(map_redis_error)?;
|
||||
|
||||
*self.publisher.lock().await = Some(manager.clone());
|
||||
return Ok(manager);
|
||||
}
|
||||
|
||||
// Inline fingerprint fallback path (no coordinator).
|
||||
let secure_scheme = matches!(self.args.url.scheme(), "rediss" | "valkeys");
|
||||
if secure_scheme {
|
||||
let next_fingerprint = super::build_target_tls_fingerprint(
|
||||
&self.args.tls.ca_path,
|
||||
&self.args.tls.client_cert_path,
|
||||
&self.args.tls.client_key_path,
|
||||
)
|
||||
.await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock();
|
||||
tls_state_guard.needs_update(&next_fingerprint)
|
||||
};
|
||||
if tls_changed {
|
||||
let new_client = build_redis_client(&self.args)?;
|
||||
*self.publisher_client.lock() = new_client;
|
||||
self.invalidate_cached_publisher().await;
|
||||
self.tls_state.lock().refresh(next_fingerprint);
|
||||
}
|
||||
}
|
||||
|
||||
let mut guard = self.publisher.lock().await;
|
||||
if let Some(manager) = guard.clone() {
|
||||
return Ok(manager);
|
||||
}
|
||||
|
||||
let manager = self
|
||||
.publisher_client
|
||||
let client = self.publisher_client.lock().clone();
|
||||
let manager = client
|
||||
.get_connection_manager_lazy(build_redis_connection_manager_config(&self.args))
|
||||
.map_err(map_redis_error)?;
|
||||
|
||||
@@ -475,7 +529,8 @@ where
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&self.publisher_client, &self.args)).await {
|
||||
let client = self.publisher_client.lock().clone();
|
||||
match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&client, &self.args)).await {
|
||||
Ok(Ok(())) => {
|
||||
self.connected.store(true, Ordering::SeqCst);
|
||||
Ok(true)
|
||||
@@ -557,6 +612,7 @@ where
|
||||
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
self.invalidate_cached_publisher().await;
|
||||
self.tls_state.lock().reset();
|
||||
self.connected.store(false, Ordering::SeqCst);
|
||||
info!(target_id = %self.id, "Redis target closed");
|
||||
Ok(())
|
||||
@@ -696,9 +752,20 @@ fn read_root_cert(tls: &RedisTlsConfig) -> Result<Option<Vec<u8>>, TargetError>
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
std::fs::read(&tls.ca_path)
|
||||
.map(Some)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))
|
||||
let pem =
|
||||
std::fs::read(&tls.ca_path).map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}")))?;
|
||||
let mut reader = BufReader::new(pem.as_slice());
|
||||
let certs_der = CertificateDer::pem_reader_iter(&mut reader)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to parse Redis root CA cert: {e}")))?;
|
||||
|
||||
if certs_der.is_empty() {
|
||||
return Err(TargetError::Configuration(
|
||||
"Redis root CA cert did not contain any parsable certificates".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(Some(pem))
|
||||
}
|
||||
|
||||
fn map_redis_error(err: RedisError) -> TargetError {
|
||||
@@ -722,6 +789,46 @@ fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration)
|
||||
min_delay.saturating_mul(factor).min(max_delay)
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for Redis targets.
|
||||
///
|
||||
/// The coordinator calls these methods on a background poll loop to detect
|
||||
/// TLS file changes and rebuild the Redis client without restarting.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for RedisTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = Client;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.tls.ca_path.clone(),
|
||||
client_cert_path: self.args.tls.client_cert_path.clone(),
|
||||
client_key_path: self.args.tls.client_key_path.clone(),
|
||||
target_label: format!("redis:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
build_redis_client(&self.args)
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
*self.publisher_client.lock() = (*material).clone();
|
||||
self.invalidate_cached_publisher().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.tls.ca_path, &self.args.tls.client_cert_path, &self.args.tls.client_key_path)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -16,14 +16,21 @@ use crate::{
|
||||
StoreError, Target,
|
||||
arn::TargetID,
|
||||
error::TargetError,
|
||||
runtime::tls::{
|
||||
ReloadableTargetTls, TargetTlsInputSet, TlsReloadAdapter, config::ReloadApplyMode, fingerprint::TargetTlsGeneration,
|
||||
validate::validate_tls_material,
|
||||
},
|
||||
store::{Key, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store,
|
||||
TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, open_target_queue_store,
|
||||
persist_queued_payload_to_store,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use parking_lot::Mutex;
|
||||
use reqwest::{Client, StatusCode, Url};
|
||||
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
|
||||
use serde::Serialize;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::{
|
||||
@@ -104,7 +111,11 @@ where
|
||||
id: TargetID,
|
||||
args: WebhookArgs,
|
||||
health_check_url: Option<Url>,
|
||||
http_client: Arc<Client>,
|
||||
http_client: Arc<Mutex<Client>>,
|
||||
tls_state: Arc<Mutex<TargetTlsState>>,
|
||||
/// When present, the adapter provides coordinator-managed TLS material;
|
||||
/// otherwise the inline fingerprint path is used as a fallback.
|
||||
tls_adapter: Option<TlsReloadAdapter<Client>>,
|
||||
// Add Send + Sync constraints to ensure thread safety
|
||||
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
|
||||
initialized: AtomicBool,
|
||||
@@ -124,6 +135,8 @@ where
|
||||
args: self.args.clone(),
|
||||
health_check_url: self.health_check_url.clone(),
|
||||
http_client: Arc::clone(&self.http_client),
|
||||
tls_state: Arc::clone(&self.tls_state),
|
||||
tls_adapter: self.tls_adapter.clone(),
|
||||
store: self.store.as_ref().map(|s| s.boxed_clone()),
|
||||
initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
|
||||
cancel_sender: self.cancel_sender.clone(),
|
||||
@@ -146,7 +159,7 @@ where
|
||||
};
|
||||
|
||||
// Build HTTP client using the helper function
|
||||
let http_client = Arc::new(Self::build_http_client(&args)?);
|
||||
let http_client = Arc::new(Mutex::new(Self::build_http_client(&args)?));
|
||||
|
||||
let queue_store = open_target_queue_store(
|
||||
&args.queue_dir,
|
||||
@@ -165,6 +178,8 @@ where
|
||||
args,
|
||||
health_check_url,
|
||||
http_client,
|
||||
tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
|
||||
tls_adapter: None,
|
||||
store: queue_store,
|
||||
initialized: AtomicBool::new(false),
|
||||
cancel_sender,
|
||||
@@ -188,11 +203,18 @@ where
|
||||
);
|
||||
} else if !args.client_ca.is_empty() {
|
||||
// Use user-provided custom CA certificate
|
||||
let ca_cert_pem = std::fs::read(&args.client_ca)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to read root CA cert: {e}")))?;
|
||||
let ca_cert = reqwest::Certificate::from_pem(&ca_cert_pem)
|
||||
let certs_der = load_cert_bundle_der_bytes(&args.client_ca)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to parse root CA cert: {e}")))?;
|
||||
client_builder = client_builder.add_root_certificate(ca_cert);
|
||||
if certs_der.is_empty() {
|
||||
return Err(TargetError::Configuration(
|
||||
"Webhook client_ca did not contain any parsable certificates".to_string(),
|
||||
));
|
||||
}
|
||||
for cert_der in certs_der {
|
||||
let ca_cert = reqwest::Certificate::from_der(&cert_der)
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to load root CA cert: {e}")))?;
|
||||
client_builder = client_builder.add_root_certificate(ca_cert);
|
||||
}
|
||||
}
|
||||
// If neither is set, use the system's default trust store
|
||||
|
||||
@@ -213,6 +235,29 @@ where
|
||||
.map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}")))
|
||||
}
|
||||
|
||||
async fn refresh_tls(&self) -> Result<(), TargetError> {
|
||||
let next_fingerprint =
|
||||
build_target_tls_fingerprint(&self.args.client_ca, &self.args.client_cert, &self.args.client_key).await?;
|
||||
let tls_changed = {
|
||||
let tls_state_guard = self.tls_state.lock();
|
||||
tls_state_guard.fingerprint.as_ref() != Some(&next_fingerprint)
|
||||
};
|
||||
if !tls_changed {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let new_client = Self::build_http_client(&self.args)?;
|
||||
{
|
||||
let mut tls_state_guard = self.tls_state.lock();
|
||||
if tls_state_guard.fingerprint.as_ref() == Some(&next_fingerprint) {
|
||||
return Ok(());
|
||||
}
|
||||
*self.http_client.lock() = new_client;
|
||||
tls_state_guard.refresh(next_fingerprint);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn health_check_url(endpoint: &Url) -> Result<Url, TargetError> {
|
||||
endpoint
|
||||
.host()
|
||||
@@ -230,7 +275,8 @@ where
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(health_check_url.as_str()).send()).await {
|
||||
let client = self.http_client.lock().clone();
|
||||
match tokio::time::timeout(Duration::from_secs(5), client.head(health_check_url.as_str()).send()).await {
|
||||
Ok(Ok(resp)) => {
|
||||
debug!(
|
||||
target = %self.id,
|
||||
@@ -299,8 +345,14 @@ where
|
||||
"Sending webhook payload"
|
||||
);
|
||||
|
||||
let mut req_builder = self
|
||||
.http_client
|
||||
// When a TLS reload adapter is attached, it drives client rebuilds in
|
||||
// the background. The inline per-send fingerprint check is skipped.
|
||||
if self.tls_adapter.is_none() {
|
||||
self.refresh_tls().await?;
|
||||
}
|
||||
|
||||
let client = self.http_client.lock().clone();
|
||||
let mut req_builder = client
|
||||
.post(self.args.endpoint.as_str())
|
||||
.header("Content-Type", meta.content_type.as_str());
|
||||
|
||||
@@ -425,6 +477,7 @@ where
|
||||
async fn close(&self) -> Result<(), TargetError> {
|
||||
// Send cancel signal to background tasks
|
||||
let _ = self.cancel_sender.try_send(());
|
||||
// Adapter cleanup is done by the coordinator; no local state to reset.
|
||||
info!("Webhook target closed: {}", self.id);
|
||||
Ok(())
|
||||
}
|
||||
@@ -460,6 +513,48 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinated TLS hot-reload implementation for Webhook targets.
|
||||
///
|
||||
/// The coordinator calls these methods on a background poll loop to detect
|
||||
/// TLS file changes and rebuild the HTTP client without restarting.
|
||||
#[async_trait]
|
||||
impl<E> ReloadableTargetTls for WebhookTarget<E>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
type Material = Client;
|
||||
|
||||
fn tls_input_set(&self) -> TargetTlsInputSet {
|
||||
TargetTlsInputSet {
|
||||
ca_path: self.args.client_ca.clone(),
|
||||
client_cert_path: self.args.client_cert.clone(),
|
||||
client_key_path: self.args.client_key.clone(),
|
||||
target_label: format!("webhook:{}", self.id.id),
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_tls_material(&self) -> Result<Self::Material, TargetError> {
|
||||
// build_http_client is synchronous (reads files + configures reqwest).
|
||||
// The coordinator already runs this in a background task, so the
|
||||
// synchronous file I/O does not block the send path.
|
||||
Self::build_http_client(&self.args)
|
||||
}
|
||||
|
||||
async fn apply_tls_material(
|
||||
&self,
|
||||
_generation: TargetTlsGeneration,
|
||||
material: Arc<Self::Material>,
|
||||
_mode: ReloadApplyMode,
|
||||
) -> Result<(), TargetError> {
|
||||
*self.http_client.lock() = (*material).clone();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn validate_tls_files(&self) -> Result<(), TargetError> {
|
||||
validate_tls_material(&self.args.client_ca, &self.args.client_cert, &self.args.client_key)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{WebhookArgs, WebhookTarget};
|
||||
|
||||
Reference in New Issue
Block a user