diff --git a/Cargo.lock b/Cargo.lock index 498688f76..ad2aa33d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9493,6 +9493,7 @@ dependencies = [ "moka", "openidconnect", "pollster", + "rcgen", "reqwest", "rustfs-config", "rustfs-credentials", @@ -9504,6 +9505,8 @@ dependencies = [ "rustfs-storage-api", "rustfs-test-utils", "rustfs-utils", + "rustls", + "rustls-pki-types", "serde", "serde_json", "serial_test", diff --git a/crates/config/src/constants/tls.rs b/crates/config/src/constants/tls.rs index c675be83e..6efbe0621 100644 --- a/crates/config/src/constants/tls.rs +++ b/crates/config/src/constants/tls.rs @@ -36,6 +36,11 @@ pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA"; /// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1 pub const DEFAULT_TRUST_SYSTEM_CA: bool = false; +/// Environment variable for an extra outbound root CA certificate bundle. +/// Use this to trust an internal CA for outbound HTTPS clients without replacing +/// the default operating-system/web PKI roots via SSL_CERT_FILE. +pub const ENV_RUSTFS_EXTRA_CA_CERT: &str = "RUSTFS_EXTRA_CA_CERT"; + /// Environment variable to trust leaf certificates as CA /// When set to "1", RustFS will treat leaf certificates as CA certificates for trust validation. /// By default, this is disabled. diff --git a/crates/iam/Cargo.toml b/crates/iam/Cargo.toml index 0ef8ef1e9..ae89dd299 100644 --- a/crates/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -107,7 +107,10 @@ url = { workspace = true } [dev-dependencies] pollster.workspace = true +rcgen.workspace = true rustfs-test-utils = { workspace = true } +rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } +rustls-pki-types.workspace = true serial_test = { workspace = true } temp-env = { workspace = true, features = ["async_closure"] } tempfile = { workspace = true } diff --git a/crates/iam/src/lib.rs b/crates/iam/src/lib.rs index a487942d0..7588a3495 100644 --- a/crates/iam/src/lib.rs +++ b/crates/iam/src/lib.rs @@ -14,7 +14,7 @@ use crate::error::{Error, Result}; use manager::IamCache; -use oidc::OidcSys; +use oidc::{OidcExtraRootCaProvider, OidcSys}; use std::sync::{Arc, OnceLock}; use store::object::ObjectStore; use sys::IamSys; @@ -284,6 +284,23 @@ pub fn get_global_iam_sys() -> Option>> { /// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured. pub async fn init_oidc_sys() -> Result<()> { + init_oidc_sys_with_extra_root_ca(None).await +} + +/// Initialize the global OIDC system with an additional outbound root CA bundle. +pub async fn init_oidc_sys_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result<()> { + init_oidc_sys_with_extra_root_ca_provider_inner(None, root_ca_pem).await +} + +/// Initialize the global OIDC system with a reload-aware outbound root CA provider. +pub async fn init_oidc_sys_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<()> { + init_oidc_sys_with_extra_root_ca_provider_inner(Some(extra_root_ca_provider), None).await +} + +async fn init_oidc_sys_with_extra_root_ca_provider_inner( + extra_root_ca_provider: Option, + root_ca_pem: Option<&[u8]>, +) -> Result<()> { if OIDC_SYS.get().is_some() { debug!( event = EVENT_OIDC_STATE, @@ -303,7 +320,11 @@ pub async fn init_oidc_sys() -> Result<()> { "OIDC runtime starting" ); - let oidc_sys = match OidcSys::new().await { + let oidc_sys_result = match extra_root_ca_provider { + Some(provider) => OidcSys::new_with_extra_root_ca_provider(provider).await, + None => OidcSys::new_with_extra_root_ca(root_ca_pem).await, + }; + let oidc_sys = match oidc_sys_result { Ok(sys) => { if sys.has_providers() { debug!( diff --git a/crates/iam/src/oidc.rs b/crates/iam/src/oidc.rs index 727fd981c..05e8f33f2 100644 --- a/crates/iam/src/oidc.rs +++ b/crates/iam/src/oidc.rs @@ -25,7 +25,7 @@ use openidconnect::{ JsonWebKeySetUrl, LogoutRequest, Nonce, PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl, ProviderMetadataWithLogout, RedirectUrl, RequestTokenError, Scope, }; -use reqwest::Client; +use reqwest::{Certificate, Client}; use rustfs_config::oidc::*; use rustfs_config::server_config::{Config as ServerConfig, KVS}; use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_OIDC_RESPONSE_SIZE}; @@ -38,7 +38,6 @@ use std::fmt; use std::future::Future; use std::net::IpAddr; use std::pin::Pin; -#[cfg(test)] use std::sync::Arc; use std::sync::{LazyLock, Mutex, MutexGuard, RwLock}; use std::time::{Duration as StdDuration, Instant}; @@ -258,6 +257,7 @@ fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String) } OidcHttpError::Reqwest(_) => ("request", String::new()), OidcHttpError::Http(_) => ("http_build", String::new()), + OidcHttpError::ExtraRootCa(_) => ("extra_root_ca", String::new()), OidcHttpError::ForbiddenOutbound(_) => ("forbidden_outbound", String::new()), OidcHttpError::ResponseTooLarge(limit) => ("response_too_large", limit.to_string()), } @@ -270,6 +270,7 @@ fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String) pub enum OidcHttpError { Reqwest(reqwest::Error), Http(http::Error), + ExtraRootCa(String), /// The outbound destination was rejected by the shared egress policy before any /// connection was attempted (invalid URL, loopback/link-local/metadata/private IP, /// or a malformed allow-origins configuration). @@ -284,6 +285,7 @@ impl std::fmt::Display for OidcHttpError { match self { Self::Reqwest(e) => write!(f, "{e}"), Self::Http(e) => write!(f, "{e}"), + Self::ExtraRootCa(reason) => write!(f, "failed to load OIDC extra root CA bundle: {reason}"), Self::ForbiddenOutbound(reason) => write!(f, "outbound request rejected: {reason}"), Self::ResponseTooLarge(limit) => write!(f, "oidc response body exceeds {limit} bytes"), } @@ -295,11 +297,48 @@ impl std::error::Error for OidcHttpError { match self { Self::Reqwest(e) => Some(e), Self::Http(e) => Some(e), - Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None, + Self::ExtraRootCa(_) | Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None, } } } +#[derive(Clone, Debug, Default)] +pub struct OidcExtraRootCaMaterial { + pub generation: u64, + pub root_ca_pem: Option>, +} + +type OidcExtraRootCaFuture = Pin> + Send>>; +type OidcExtraRootCaLoader = dyn Fn() -> OidcExtraRootCaFuture + Send + Sync; + +#[derive(Clone)] +pub struct OidcExtraRootCaProvider { + loader: Arc, +} + +impl OidcExtraRootCaProvider { + pub fn new(loader: F) -> Self + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + Self { + loader: Arc::new(move || Box::pin(loader())), + } + } + + async fn load(&self) -> Result { + (self.loader)().await + } +} + +#[derive(Clone, Default)] +struct CachedOidcExtraRootCerts { + generation: u64, + initialized: bool, + certs: Vec, +} + /// HTTP client adapter bridging reqwest 0.13 to the `openidconnect` `AsyncHttpClient` trait. /// /// A fresh client is built for every request so the destination is re-validated and the @@ -311,10 +350,26 @@ pub(crate) struct ReqwestHttpClient { /// `None` in production: the process-cached outbound policy from the environment is used. /// `Some(..)` only in tests, to explicitly allow a loopback mock endpoint. policy_override: Option, + extra_root_certs: Arc>, + extra_root_ca_provider: Option, #[cfg(test)] dns_resolver_override: Option>, } +fn parse_oidc_extra_root_certs(source: &str, pem: &[u8]) -> Result, String> { + if pem.iter().all(|byte| byte.is_ascii_whitespace()) { + return Ok(Vec::new()); + } + Certificate::from_pem_bundle(pem).map_err(|err| format!("failed to parse OIDC extra root CA bundle from {source}: {err}")) +} + +fn oidc_extra_root_certs(root_ca_pem: Option<&[u8]>) -> Result, String> { + match root_ca_pem { + Some(pem) => parse_oidc_extra_root_certs("RustFS outbound TLS material", pem), + None => Ok(Vec::new()), + } +} + /// Build a reqwest client pinned to the shared outbound egress policy for a single request. /// /// [`OutboundPolicy::resolver_for`] validates the URL shape and rejects loopback, @@ -326,6 +381,7 @@ pub(crate) struct ReqwestHttpClient { fn build_oidc_http_client( uri: &str, policy_override: Option<&OutboundPolicy>, + extra_root_certs: &[Certificate], #[cfg(test)] dns_resolver_override: Option>, ) -> Result<(Client, Url), OidcHttpError> { let url = Url::parse(uri).map_err(|_| OidcHttpError::ForbiddenOutbound("invalid outbound OIDC URL".to_string()))?; @@ -356,6 +412,9 @@ fn build_oidc_http_client( if bypass_proxy { builder = builder.no_proxy(); } + if !extra_root_certs.is_empty() { + builder = builder.tls_certs_merge(extra_root_certs.iter().cloned()); + } builder.build().map(|client| (client, url)).map_err(OidcHttpError::Reqwest) } @@ -410,19 +469,93 @@ fn should_bypass_proxy_for_oidc_uri(uri: &str) -> bool { impl ReqwestHttpClient { fn new() -> Result { + Self::new_with_extra_root_certs(Vec::new()) + } + + fn extra_root_cert_cache(certs: Vec) -> Arc> { + Arc::new(RwLock::new(CachedOidcExtraRootCerts { + generation: 0, + initialized: true, + certs, + })) + } + + fn new_with_extra_root_certs(extra_root_certs: Vec) -> Result { Ok(Self { policy_override: None, + extra_root_certs: Self::extra_root_cert_cache(extra_root_certs), + extra_root_ca_provider: None, #[cfg(test)] dns_resolver_override: None, }) } + fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result { + Ok(Self { + policy_override: None, + extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())), + extra_root_ca_provider: Some(extra_root_ca_provider), + #[cfg(test)] + dns_resolver_override: None, + }) + } + + async fn current_extra_root_certs(&self) -> Result, OidcHttpError> { + let Some(provider) = self.extra_root_ca_provider.as_ref() else { + return self + .extra_root_certs + .read() + .map(|cache| cache.certs.clone()) + .map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}"))); + }; + + let material = provider.load().await.map_err(OidcHttpError::ExtraRootCa)?; + if let Ok(cache) = self.extra_root_certs.read() + && cache.initialized + && cache.generation == material.generation + { + return Ok(cache.certs.clone()); + } + + let certs = oidc_extra_root_certs(material.root_ca_pem.as_deref()).map_err(OidcHttpError::ExtraRootCa)?; + let mut cache = self + .extra_root_certs + .write() + .map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}")))?; + cache.generation = material.generation; + cache.initialized = true; + cache.certs = certs.clone(); + Ok(certs) + } + /// Test-only constructor that pins outbound requests to an explicit policy, so a /// loopback mock server can be reached without depending on process-wide environment. #[cfg(test)] fn with_policy(policy: OutboundPolicy) -> Self { Self { policy_override: Some(policy), + extra_root_certs: Self::extra_root_cert_cache(Vec::new()), + extra_root_ca_provider: None, + dns_resolver_override: None, + } + } + + #[cfg(test)] + fn with_policy_and_extra_root_certs(policy: OutboundPolicy, extra_root_certs: Vec) -> Self { + Self { + policy_override: Some(policy), + extra_root_certs: Self::extra_root_cert_cache(extra_root_certs), + extra_root_ca_provider: None, + dns_resolver_override: None, + } + } + + #[cfg(test)] + fn with_policy_and_extra_root_ca_provider(policy: OutboundPolicy, extra_root_ca_provider: OidcExtraRootCaProvider) -> Self { + Self { + policy_override: Some(policy), + extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())), + extra_root_ca_provider: Some(extra_root_ca_provider), dns_resolver_override: None, } } @@ -431,6 +564,8 @@ impl ReqwestHttpClient { fn with_policy_and_dns_resolver(policy: OutboundPolicy, resolver: Arc) -> Self { Self { policy_override: Some(policy), + extra_root_certs: Self::extra_root_cert_cache(Vec::new()), + extra_root_ca_provider: None, dns_resolver_override: Some(resolver), } } @@ -461,9 +596,11 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient { ); } + let extra_root_certs = self.current_extra_root_certs().await?; let (client, url) = build_oidc_http_client( &uri, self.policy_override.as_ref(), + &extra_root_certs, #[cfg(test)] self.dns_resolver_override.clone(), )?; @@ -678,7 +815,22 @@ fn trusted_aud(other_audiences: &[String], audience: &Audience) -> bool { impl OidcSys { /// Parse environment variables and discover all configured OIDC providers. pub async fn new() -> Result { - let http_client = ReqwestHttpClient::new()?; + Self::new_with_extra_root_ca(None).await + } + + /// Parse environment variables and discover providers with an additional outbound root CA bundle. + pub(crate) async fn new_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result { + let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?; + Self::new_with_http_client(http_client).await + } + + pub(crate) async fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result { + let http_client = ReqwestHttpClient::new_with_extra_root_ca_provider(extra_root_ca_provider)?; + http_client.current_extra_root_certs().await.map_err(|err| err.to_string())?; + Self::new_with_http_client(http_client).await + } + + async fn new_with_http_client(http_client: ReqwestHttpClient) -> Result { let server_config = crate::server_config::current_server_config(); let parsed_configs = load_effective_oidc_provider_configs(server_config.as_ref()); let mut configs = HashMap::new(); @@ -1874,7 +2026,14 @@ pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig> } pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result { - let http_client = ReqwestHttpClient::new()?; + validate_oidc_provider_config_with_extra_root_ca(config, None).await +} + +pub async fn validate_oidc_provider_config_with_extra_root_ca( + config: &OidcProviderConfig, + root_ca_pem: Option<&[u8]>, +) -> Result { + let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?; let state = OidcSys::discover_provider(config, &http_client).await?; Ok(OidcProviderValidationResult { @@ -2438,6 +2597,50 @@ mod tests { } } + fn read_mock_oidc_request_path(stream: &mut impl std::io::Read) -> String { + let mut request_bytes = Vec::new(); + let mut buffer = [0u8; 4096]; + loop { + match stream.read(&mut buffer) { + Ok(0) => break, + Ok(n) => request_bytes.extend_from_slice(&buffer[..n]), + Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => break, + Err(_) => break, + } + if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") { + break; + } + if request_bytes.len() >= 8192 { + break; + } + } + let request = String::from_utf8_lossy(&request_bytes); + request + .lines() + .next() + .unwrap_or("") + .split_whitespace() + .nth(1) + .unwrap_or("") + .to_string() + } + + fn mock_oidc_response(path: &str, discovery_body: &str, expected_jwks_path: &str, jwks_body: &str) -> String { + let (status, body) = if path.contains("/.well-known/openid-configuration") { + (200, discovery_body) + } else if path == expected_jwks_path { + (200, jwks_body) + } else { + (404, r#"{"error":"not found"}"#) + }; + + format!( + "HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + if status == 200 { "OK" } else { "Not Found" }, + body.len() + ) + } + fn start_mock_oidc_discovery_server( build_discovery_issuer: F, max_requests: usize, @@ -2445,7 +2648,6 @@ mod tests { where F: Fn(&str) -> (String, String, String) + Send + 'static, { - use std::io::Read; use std::io::Write; use std::net::{Shutdown, TcpListener}; use std::sync::mpsc; @@ -2516,41 +2718,8 @@ mod tests { .set_read_timeout(Some(Duration::from_secs(1))) .expect("failed to set discovery mock read timeout"); - let mut request_bytes = Vec::new(); - let mut buffer = [0u8; 4096]; - loop { - match stream.read(&mut buffer) { - Ok(0) => break, - Ok(n) => request_bytes.extend_from_slice(&buffer[..n]), - Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => { - break; - } - Err(_) => break, - } - if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") { - break; - } - if request_bytes.len() >= 8192 { - break; - } - } - let request = String::from_utf8_lossy(&request_bytes); - let path = request.lines().next().unwrap_or("").split_whitespace().nth(1).unwrap_or(""); - - let (status, body) = if path.contains("/.well-known/openid-configuration") { - (200, discovery_body.as_str()) - } else if path == expected_jwks_path { - (200, jwks_body) - } else { - (404, r#"{"error":"not found"}"#) - }; - - let response = format!( - "HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - if status == 200 { "OK" } else { "Not Found" }, - body.len() - ); - + let path = read_mock_oidc_request_path(&mut stream); + let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body); let _ = stream.write_all(response.as_bytes()); let _ = stream.flush(); let _ = stream.shutdown(Shutdown::Both); @@ -2568,6 +2737,114 @@ mod tests { Some((base, handle)) } + fn start_mock_oidc_tls_discovery_server( + build_discovery_issuer: F, + max_requests: usize, + ) -> Option<(String, String, std::thread::JoinHandle<()>)> + where + F: Fn(&str) -> (String, String, String) + Send + 'static, + { + use std::io::Write; + use std::net::{Shutdown, TcpListener}; + use std::sync::mpsc; + use std::time::{Duration, Instant}; + + const IDLE_SHUTDOWN: Duration = Duration::from_secs(1); + const ABSOLUTE_CAP: Duration = Duration::from_secs(5); + + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let certified = + rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).expect("generate OIDC TLS test certificate"); + let cert_pem = certified.cert.pem(); + let server_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![certified.cert.der().clone()], + rustls_pki_types::PrivateKeyDer::try_from(certified.signing_key.serialize_der()) + .expect("convert OIDC TLS test private key"), + ) + .expect("build OIDC TLS mock server config"); + + let listener = match TcpListener::bind("127.0.0.1:0") { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None, + Err(err) => panic!("test TLS listener should bind: {err}"), + }; + let base = format!("https://{}", listener.local_addr().expect("listener local address should be available")); + let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base); + let discovery_body = serde_json::json!({ + "issuer": discovery_issuer, + "authorization_endpoint": format!("{base}/authorize"), + "token_endpoint": format!("{base}/token"), + "jwks_uri": discovery_jwks_uri, + "response_types_supported": ["code"], + "response_modes_supported": ["query"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + }) + .to_string(); + let jwks_body = r#"{"keys":[]}"#; + let (ready_tx, ready_rx) = mpsc::channel(); + + let handle = std::thread::spawn(move || { + let server_config = Arc::new(server_config); + listener + .set_nonblocking(true) + .expect("failed to set TLS discovery mock listener non-blocking"); + let _ = ready_tx.send(()); + + let mut seen = 0usize; + let start = Instant::now(); + let mut last_completed = Instant::now(); + + loop { + if seen > 0 && last_completed.elapsed() >= IDLE_SHUTDOWN { + break; + } + if start.elapsed() >= ABSOLUTE_CAP { + break; + } + + let tcp_stream = match listener.accept() { + Ok((stream, _)) => stream, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(5)); + continue; + } + Err(_) => break, + }; + tcp_stream + .set_nonblocking(false) + .expect("failed to set TLS discovery mock stream blocking"); + tcp_stream + .set_read_timeout(Some(Duration::from_secs(1))) + .expect("failed to set TLS discovery mock read timeout"); + + seen += 1; + let connection = match rustls::ServerConnection::new(server_config.clone()) { + Ok(connection) => connection, + Err(_) => break, + }; + let mut stream = rustls::StreamOwned::new(connection, tcp_stream); + let path = read_mock_oidc_request_path(&mut stream); + let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + let _ = stream.sock.shutdown(Shutdown::Both); + last_completed = Instant::now(); + + if seen >= max_requests { + break; + } + } + }); + ready_rx + .recv_timeout(Duration::from_millis(100)) + .expect("mock TLS OIDC discovery server should become ready"); + + Some((base, cert_pem, handle)) + } + fn discovery_error_contains_all_variants(err: &str, base: &str) -> bool { err.contains(base) && err.contains(&format!("{base}/")) && err.contains("discovery failed for all issuer variants") } @@ -2590,6 +2867,100 @@ mod tests { }) } + #[tokio::test] + async fn oidc_discovery_accepts_extra_root_ca_for_https_provider() { + let Some((base, ca_pem, handle)) = start_mock_oidc_tls_discovery_server( + |base| (format!("{base}/application/o/rustfs"), format!("{base}/jwks"), "/jwks".to_string()), + 4, + ) else { + return; + }; + let config_url = format!("{base}/application/o/rustfs"); + let config = build_mocked_oidc_provider_config("default", &config_url); + let origin = Url::parse(&config.config_url) + .expect("mock config_url should parse") + .origin() + .ascii_serialization(); + let policy = OutboundPolicy::from_allowed_origins(&origin).expect("loopback TLS origin should be allowed"); + let extra_root_certs = + parse_oidc_extra_root_certs("test OIDC TLS CA", ca_pem.as_bytes()).expect("test CA bundle should parse"); + let http_client = ReqwestHttpClient::with_policy_and_extra_root_certs(policy, extra_root_certs); + + let state = OidcSys::discover_provider(&config, &http_client) + .await + .expect("OIDC discovery should trust the extra root CA"); + + assert_eq!(state.metadata.issuer().to_string(), format!("{base}/application/o/rustfs")); + assert!(handle.join().is_ok()); + } + + #[tokio::test] + async fn oidc_discovery_refreshes_extra_root_ca_when_generation_changes() { + let Some((base_a, ca_pem_a, handle_a)) = start_mock_oidc_tls_discovery_server( + |base| (format!("{base}/application/o/rustfs-a"), format!("{base}/jwks"), "/jwks".to_string()), + 4, + ) else { + return; + }; + let Some((base_b, ca_pem_b, handle_b)) = start_mock_oidc_tls_discovery_server( + |base| (format!("{base}/application/o/rustfs-b"), format!("{base}/jwks"), "/jwks".to_string()), + 4, + ) else { + assert!(handle_a.join().is_ok()); + return; + }; + + let origin_a = Url::parse(&base_a) + .expect("mock base A should parse") + .origin() + .ascii_serialization(); + let origin_b = Url::parse(&base_b) + .expect("mock base B should parse") + .origin() + .ascii_serialization(); + let allowed_origins = format!("{origin_a},{origin_b}"); + let policy = OutboundPolicy::from_allowed_origins(&allowed_origins).expect("loopback TLS origins should be allowed"); + let material = Arc::new(Mutex::new(OidcExtraRootCaMaterial { + generation: 1, + root_ca_pem: Some(ca_pem_a.into_bytes()), + })); + let provider = OidcExtraRootCaProvider::new({ + let material = material.clone(); + move || { + let material = material.clone(); + async move { + material + .lock() + .map(|material| material.clone()) + .map_err(|e| format!("test OIDC extra CA material lock poisoned: {e}")) + } + } + }); + let http_client = ReqwestHttpClient::with_policy_and_extra_root_ca_provider(policy, provider); + + let config_a = build_mocked_oidc_provider_config("a", &format!("{base_a}/application/o/rustfs-a")); + let state_a = OidcSys::discover_provider(&config_a, &http_client) + .await + .expect("OIDC discovery should trust initial extra root CA"); + assert_eq!(state_a.metadata.issuer().to_string(), format!("{base_a}/application/o/rustfs-a")); + + { + let mut material = material + .lock() + .expect("test OIDC extra CA material lock should not be poisoned"); + material.generation = 2; + material.root_ca_pem = Some(ca_pem_b.into_bytes()); + } + let config_b = build_mocked_oidc_provider_config("b", &format!("{base_b}/application/o/rustfs-b")); + let state_b = OidcSys::discover_provider(&config_b, &http_client) + .await + .expect("OIDC discovery should refresh extra root CA after generation change"); + + assert_eq!(state_b.metadata.issuer().to_string(), format!("{base_b}/application/o/rustfs-b")); + assert!(handle_a.join().is_ok()); + assert!(handle_b.join().is_ok()); + } + #[tokio::test] async fn test_validate_oidc_provider_config_retries_with_issuer_candidates() { // Discovery document must advertise the canonical issuer path. The first candidate has no @@ -2945,7 +3316,7 @@ mod tests { // Cloud metadata endpoint is never allowed. assert!( matches!( - build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, None), + build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, &[], None), Err(OidcHttpError::ForbiddenOutbound(_)) ), "metadata endpoint must be rejected" @@ -2953,7 +3324,7 @@ mod tests { // Loopback is rejected by default (no allow-origins configured). assert!( matches!( - build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, None), + build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, &[], None), Err(OidcHttpError::ForbiddenOutbound(_)) ), "loopback must be rejected by default" @@ -2961,7 +3332,7 @@ mod tests { // A public hostname passes the up-front shape/host check; the resolved IP is still // re-classified at connection time by the pinned resolver. assert!( - build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, None).is_ok(), + build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, &[], None).is_ok(), "public https endpoint should build" ); } @@ -2981,13 +3352,13 @@ mod tests { fn build_oidc_http_client_honors_explicit_allowlist_for_loopback() { let policy = OutboundPolicy::from_allowed_origins("http://127.0.0.1:8080").expect("origin should parse"); assert!( - build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), None).is_ok(), + build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), &[], None).is_ok(), "explicitly allow-listed loopback origin should build" ); // A metadata endpoint stays forbidden even when a loopback origin is allow-listed. assert!( matches!( - build_oidc_http_client("http://169.254.169.254/", Some(&policy), None), + build_oidc_http_client("http://169.254.169.254/", Some(&policy), &[], None), Err(OidcHttpError::ForbiddenOutbound(_)) ), "metadata endpoint stays forbidden despite an unrelated allow-list entry" @@ -3190,8 +3561,9 @@ mod tests { #[test] fn oidc_metadata_endpoint_rejection_does_not_offer_allowlist_bypass() { - let error = build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), None) - .expect_err("metadata endpoint must remain forbidden"); + let error = + build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), &[], None) + .expect_err("metadata endpoint must remain forbidden"); let message = error.to_string(); assert!(message.contains("metadata endpoint")); diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index 6b088ebc1..42ace4905 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -449,9 +449,15 @@ impl Operation for ValidateOidcConfigHandler { request.provider_id.trim().to_string() }; let provider_config = build_provider_config_from_validate(request, &provider_id)?; - let validation = rustfs_iam::oidc::validate_oidc_provider_config(&provider_config) + let oidc_extra_root_ca = crate::startup_auth::current_oidc_extra_root_ca_material() .await .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("validation failed: {e}")))?; + let validation = rustfs_iam::oidc::validate_oidc_provider_config_with_extra_root_ca( + &provider_config, + oidc_extra_root_ca.root_ca_pem.as_deref(), + ) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("validation failed: {e}")))?; json_response( StatusCode::OK, diff --git a/rustfs/src/server/tls_material.rs b/rustfs/src/server/tls_material.rs index a211f7392..01329ae38 100644 --- a/rustfs/src/server/tls_material.rs +++ b/rustfs/src/server/tls_material.rs @@ -24,9 +24,10 @@ use crate::startup_runtime_sources; use rustfs_common::MtlsIdentityPem; use rustfs_config::{ DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL, - DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_SERVER_MTLS_ENABLE, - ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_SYSTEM_CA, - RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_TLS_CERT, + DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_RUSTFS_EXTRA_CA_CERT, + ENV_SERVER_MTLS_ENABLE, ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, + ENV_TRUST_SYSTEM_CA, RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, + RUSTFS_TLS_CERT, }; use rustfs_tls_runtime::{ ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions, @@ -34,6 +35,7 @@ use rustfs_tls_runtime::{ }; use rustfs_utils::{get_env_bool, get_env_opt_str}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; +use std::io::Cursor; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::RwLock; @@ -267,6 +269,60 @@ fn map_runtime_tls_error(err: rustfs_tls_runtime::TlsRuntimeError) -> TlsMateria } } +pub(crate) async fn validate_configured_oidc_extra_ca_cert() -> Result<(), TlsMaterialError> { + if let Some(path) = configured_oidc_extra_ca_cert_path() { + let _ = load_configured_oidc_extra_ca_cert().await?; + info!( + component = LOG_COMPONENT_TLS, + subsystem = LOG_SUBSYSTEM_TLS, + event = "oidc_extra_ca_validated", + source = "oidc_extra_ca_bundle", + env_var = ENV_RUSTFS_EXTRA_CA_CERT, + path = ?path, + "OIDC extra root CA bundle validated" + ); + } + Ok(()) +} + +pub(crate) async fn load_configured_oidc_extra_ca_cert() -> Result>, TlsMaterialError> { + let Some(path) = configured_oidc_extra_ca_cert_path() else { + return Ok(None); + }; + + let data = tokio::fs::read(&path) + .await + .map_err(|e| TlsMaterialError::Io(format!("read extra CA bundle {path:?}: {e}")))?; + validate_cert_bundle(&data, &path)?; + Ok(Some(data)) +} + +fn configured_oidc_extra_ca_cert_path() -> Option { + let path = get_env_opt_str(ENV_RUSTFS_EXTRA_CA_CERT)?; + let path = path.trim(); + if path.is_empty() { + return None; + } + Some(PathBuf::from(path)) +} + +fn validate_cert_bundle(data: &[u8], path: &Path) -> Result<(), TlsMaterialError> { + let mut reader = Cursor::new(data); + let mut found = false; + let mut store = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_reader_iter(&mut reader) { + let cert = cert.map_err(|e| TlsMaterialError::Parse(format!("invalid extra CA bundle {path:?}: {e}")))?; + store + .add(cert) + .map_err(|e| TlsMaterialError::Parse(format!("invalid extra CA bundle {path:?}: {e}")))?; + found = true; + } + if !found { + return Err(TlsMaterialError::Parse(format!("no certificate found in extra CA bundle {path:?}"))); + } + Ok(()) +} + /// Load a single certificate file and append PEM data. /// Returns true if the file was successfully loaded. async fn load_cert_file(path: &Path, pem_data: &mut Vec, desc: &str) -> bool { @@ -681,6 +737,86 @@ mod tests { fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap(); } + #[tokio::test] + #[serial_test::serial] + async fn oidc_extra_ca_cert_loads_configured_bundle() { + let CertifiedKey { cert, .. } = + rcgen::generate_simple_self_signed(vec!["extra-ca.example".to_string()]).expect("generate extra CA cert"); + let temp_file = tempfile::NamedTempFile::new().expect("create extra CA file"); + fs::write(temp_file.path(), cert.pem()).expect("write extra CA file"); + let path = temp_file.path().to_string_lossy().to_string(); + + temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async { + let extra_ca = load_configured_oidc_extra_ca_cert() + .await + .expect("OIDC extra CA should load") + .expect("configured OIDC extra CA should be present"); + + assert!(extra_ca.starts_with(cert.pem().as_bytes())); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn oidc_extra_ca_cert_rejects_invalid_pem() { + let temp_file = tempfile::NamedTempFile::new().expect("create invalid extra CA file"); + fs::write(temp_file.path(), b"not a certificate").expect("write invalid extra CA file"); + let path = temp_file.path().to_string_lossy().to_string(); + + temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async { + let err = load_configured_oidc_extra_ca_cert() + .await + .expect_err("invalid extra CA should fail"); + + assert!(err.to_string().contains("no certificate found in extra CA bundle")); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn oidc_extra_ca_cert_rejects_malformed_der_certificate() { + let temp_file = tempfile::NamedTempFile::new().expect("create malformed extra CA file"); + fs::write( + temp_file.path(), + b"-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----\n", + ) + .expect("write malformed extra CA file"); + let path = temp_file.path().to_string_lossy().to_string(); + + temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async { + let err = load_configured_oidc_extra_ca_cert() + .await + .expect_err("malformed DER in PEM framing should fail"); + + assert!(err.to_string().contains("invalid extra CA bundle")); + }) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn load_tls_material_does_not_append_oidc_extra_ca_cert() { + let temp_dir = TempDir::new().expect("create TLS material dir"); + write_test_cert_pair(temp_dir.path(), "server.example"); + let CertifiedKey { cert, .. } = + rcgen::generate_simple_self_signed(vec!["extra-ca.example".to_string()]).expect("generate extra CA cert"); + let temp_file = tempfile::NamedTempFile::new().expect("create extra CA file"); + fs::write(temp_file.path(), cert.pem()).expect("write extra CA file"); + let path = temp_file.path().to_string_lossy().to_string(); + + temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async { + let snapshot = load_tls_material(temp_dir.path().to_str().expect("TLS material dir should be utf-8")) + .await + .expect("TLS material should load"); + + assert!(snapshot.outbound.root_ca_pem.is_empty()); + assert!(snapshot.server.is_some()); + }) + .await; + } + #[tokio::test] async fn build_acceptor_accepts_root_single_cert_with_trailing_slash() { ensure_rustls_crypto_provider(); diff --git a/rustfs/src/startup_auth.rs b/rustfs/src/startup_auth.rs index 02029e538..e21ff8dfb 100644 --- a/rustfs/src/startup_auth.rs +++ b/rustfs/src/startup_auth.rs @@ -14,9 +14,12 @@ use rustfs_iam::{ federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter}, - get_oidc, init_oidc_sys, + get_oidc, init_oidc_sys_with_extra_root_ca_provider, + oidc::{OidcExtraRootCaMaterial, OidcExtraRootCaProvider}, }; use std::{ + collections::hash_map::DefaultHasher, + hash::{Hash, Hasher}, io::{Error, Result}, sync::Arc, }; @@ -50,7 +53,7 @@ pub(crate) async fn init_auth_integrations() -> Result<()> { } } - match init_oidc_sys().await { + match init_oidc_sys_with_extra_root_ca_provider(oidc_extra_root_ca_provider()).await { Ok(()) => { if let Some(oidc) = get_oidc() { let adapter = Arc::new(StandardOidcAdapter::new(oidc)); @@ -72,3 +75,40 @@ pub(crate) async fn init_auth_integrations() -> Result<()> { Ok(()) } + +pub(crate) fn oidc_extra_root_ca_provider() -> OidcExtraRootCaProvider { + OidcExtraRootCaProvider::new(current_oidc_extra_root_ca_material) +} + +pub(crate) async fn current_oidc_extra_root_ca_material() -> std::result::Result { + let outbound_tls = crate::runtime_sources::current_outbound_tls_state().await; + let outbound_generation = outbound_tls.as_ref().map(|state| state.generation.0).unwrap_or_default(); + let mut root_ca_pem = outbound_tls.as_ref().and_then(|state| state.root_ca_pem.clone()); + + if let Some(extra_ca_pem) = crate::server::tls_material::load_configured_oidc_extra_ca_cert() + .await + .map_err(|err| err.to_string())? + { + match root_ca_pem.as_mut() { + Some(root_ca_pem) => { + if !root_ca_pem.is_empty() && !root_ca_pem.ends_with(b"\n") { + root_ca_pem.push(b'\n'); + } + root_ca_pem.extend_from_slice(&extra_ca_pem); + } + None => root_ca_pem = Some(extra_ca_pem), + } + } + + Ok(OidcExtraRootCaMaterial { + generation: oidc_extra_root_ca_generation(outbound_generation, root_ca_pem.as_deref()), + root_ca_pem, + }) +} + +fn oidc_extra_root_ca_generation(outbound_generation: u64, root_ca_pem: Option<&[u8]>) -> u64 { + let mut hasher = DefaultHasher::new(); + outbound_generation.hash(&mut hasher); + root_ca_pem.hash(&mut hasher); + hasher.finish() +} diff --git a/rustfs/src/startup_tls_material.rs b/rustfs/src/startup_tls_material.rs index edc053e3f..9d770293e 100644 --- a/rustfs/src/startup_tls_material.rs +++ b/rustfs/src/startup_tls_material.rs @@ -24,21 +24,14 @@ const EVENT_TLS_OUTBOUND_INITIALIZATION_FAILED: &str = "tls_outbound_initializat const TLS_STARTUP_GENERATION_CONSUMER: &str = "rustfs_server_startup"; pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { + crate::server::tls_material::validate_configured_oidc_extra_ca_cert() + .await + .map_err(|err| Error::other(err.to_string()))?; + if let Some(tls_path) = normalized_tls_path(config.tls_path.as_deref()) { match crate::server::tls_material::load_tls_material(tls_path).await { Ok(snapshot) => { - let generation = next_tls_generation(startup_runtime_sources::current_outbound_tls_generation()); - startup_runtime_sources::publish_outbound_tls_state(generation, &snapshot.outbound).await; - startup_runtime_sources::record_tls_generation(TLS_STARTUP_GENERATION_CONSUMER, generation.0); - info!( - target: "rustfs::main", - event = EVENT_TLS_OUTBOUND_INITIALIZED, - component = LOG_COMPONENT_MAIN, - subsystem = LOG_SUBSYSTEM_STARTUP, - tls_path, - generation = generation.0, - "Initialized TLS outbound material" - ); + publish_outbound_tls_material(&snapshot.outbound, Some(tls_path)).await; } Err(err) => { error!( @@ -61,6 +54,24 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> { Ok(()) } +async fn publish_outbound_tls_material(outbound: &rustfs_tls_runtime::OutboundTlsMaterial, tls_path: Option<&str>) { + let generation = next_tls_generation(startup_runtime_sources::current_outbound_tls_generation()); + startup_runtime_sources::publish_outbound_tls_state(generation, outbound).await; + startup_runtime_sources::record_tls_generation(TLS_STARTUP_GENERATION_CONSUMER, generation.0); + info!( + target: "rustfs::main", + event = EVENT_TLS_OUTBOUND_INITIALIZED, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "initialized", + tls_path = tls_path.unwrap_or(""), + generation = generation.0, + has_root_ca = !outbound.root_ca_pem.is_empty(), + has_mtls_identity = outbound.mtls_identity.is_some(), + "Initialized TLS outbound material" + ); +} + fn normalized_tls_path(path: Option<&str>) -> Option<&str> { path.map(str::trim).filter(|value| !value.is_empty()) }