fix(iam): merge OIDC extra root CAs (#5915)

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-10 15:24:52 +08:00
committed by GitHub
parent d900e11a09
commit d97e059c3c
9 changed files with 665 additions and 68 deletions
Generated
+3
View File
@@ -9493,6 +9493,7 @@ dependencies = [
"moka",
"openidconnect",
"pollster",
"rcgen",
"reqwest",
"rustfs-config",
"rustfs-credentials",
@@ -9504,6 +9505,8 @@ dependencies = [
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-utils",
"rustls",
"rustls-pki-types",
"serde",
"serde_json",
"serial_test",
+5
View File
@@ -36,6 +36,11 @@ pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA";
/// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1
pub const DEFAULT_TRUST_SYSTEM_CA: bool = false;
/// Environment variable for an extra outbound root CA certificate bundle.
/// Use this to trust an internal CA for outbound HTTPS clients without replacing
/// the default operating-system/web PKI roots via SSL_CERT_FILE.
pub const ENV_RUSTFS_EXTRA_CA_CERT: &str = "RUSTFS_EXTRA_CA_CERT";
/// Environment variable to trust leaf certificates as CA
/// When set to "1", RustFS will treat leaf certificates as CA certificates for trust validation.
/// By default, this is disabled.
+3
View File
@@ -107,7 +107,10 @@ url = { workspace = true }
[dev-dependencies]
pollster.workspace = true
rcgen.workspace = true
rustfs-test-utils = { workspace = true }
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
rustls-pki-types.workspace = true
serial_test = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tempfile = { workspace = true }
+23 -2
View File
@@ -14,7 +14,7 @@
use crate::error::{Error, Result};
use manager::IamCache;
use oidc::OidcSys;
use oidc::{OidcExtraRootCaProvider, OidcSys};
use std::sync::{Arc, OnceLock};
use store::object::ObjectStore;
use sys::IamSys;
@@ -284,6 +284,23 @@ pub fn get_global_iam_sys() -> Option<Arc<IamSys<ObjectStore>>> {
/// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured.
pub async fn init_oidc_sys() -> Result<()> {
init_oidc_sys_with_extra_root_ca(None).await
}
/// Initialize the global OIDC system with an additional outbound root CA bundle.
pub async fn init_oidc_sys_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result<()> {
init_oidc_sys_with_extra_root_ca_provider_inner(None, root_ca_pem).await
}
/// Initialize the global OIDC system with a reload-aware outbound root CA provider.
pub async fn init_oidc_sys_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<()> {
init_oidc_sys_with_extra_root_ca_provider_inner(Some(extra_root_ca_provider), None).await
}
async fn init_oidc_sys_with_extra_root_ca_provider_inner(
extra_root_ca_provider: Option<OidcExtraRootCaProvider>,
root_ca_pem: Option<&[u8]>,
) -> Result<()> {
if OIDC_SYS.get().is_some() {
debug!(
event = EVENT_OIDC_STATE,
@@ -303,7 +320,11 @@ pub async fn init_oidc_sys() -> Result<()> {
"OIDC runtime starting"
);
let oidc_sys = match OidcSys::new().await {
let oidc_sys_result = match extra_root_ca_provider {
Some(provider) => OidcSys::new_with_extra_root_ca_provider(provider).await,
None => OidcSys::new_with_extra_root_ca(root_ca_pem).await,
};
let oidc_sys = match oidc_sys_result {
Ok(sys) => {
if sys.has_providers() {
debug!(
+420 -48
View File
@@ -25,7 +25,7 @@ use openidconnect::{
JsonWebKeySetUrl, LogoutRequest, Nonce, PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl,
ProviderMetadataWithLogout, RedirectUrl, RequestTokenError, Scope,
};
use reqwest::Client;
use reqwest::{Certificate, Client};
use rustfs_config::oidc::*;
use rustfs_config::server_config::{Config as ServerConfig, KVS};
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_OIDC_RESPONSE_SIZE};
@@ -38,7 +38,6 @@ use std::fmt;
use std::future::Future;
use std::net::IpAddr;
use std::pin::Pin;
#[cfg(test)]
use std::sync::Arc;
use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
use std::time::{Duration as StdDuration, Instant};
@@ -258,6 +257,7 @@ fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String)
}
OidcHttpError::Reqwest(_) => ("request", String::new()),
OidcHttpError::Http(_) => ("http_build", String::new()),
OidcHttpError::ExtraRootCa(_) => ("extra_root_ca", String::new()),
OidcHttpError::ForbiddenOutbound(_) => ("forbidden_outbound", String::new()),
OidcHttpError::ResponseTooLarge(limit) => ("response_too_large", limit.to_string()),
}
@@ -270,6 +270,7 @@ fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String)
pub enum OidcHttpError {
Reqwest(reqwest::Error),
Http(http::Error),
ExtraRootCa(String),
/// The outbound destination was rejected by the shared egress policy before any
/// connection was attempted (invalid URL, loopback/link-local/metadata/private IP,
/// or a malformed allow-origins configuration).
@@ -284,6 +285,7 @@ impl std::fmt::Display for OidcHttpError {
match self {
Self::Reqwest(e) => write!(f, "{e}"),
Self::Http(e) => write!(f, "{e}"),
Self::ExtraRootCa(reason) => write!(f, "failed to load OIDC extra root CA bundle: {reason}"),
Self::ForbiddenOutbound(reason) => write!(f, "outbound request rejected: {reason}"),
Self::ResponseTooLarge(limit) => write!(f, "oidc response body exceeds {limit} bytes"),
}
@@ -295,11 +297,48 @@ impl std::error::Error for OidcHttpError {
match self {
Self::Reqwest(e) => Some(e),
Self::Http(e) => Some(e),
Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None,
Self::ExtraRootCa(_) | Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None,
}
}
}
#[derive(Clone, Debug, Default)]
pub struct OidcExtraRootCaMaterial {
pub generation: u64,
pub root_ca_pem: Option<Vec<u8>>,
}
type OidcExtraRootCaFuture = Pin<Box<dyn Future<Output = Result<OidcExtraRootCaMaterial, String>> + Send>>;
type OidcExtraRootCaLoader = dyn Fn() -> OidcExtraRootCaFuture + Send + Sync;
#[derive(Clone)]
pub struct OidcExtraRootCaProvider {
loader: Arc<OidcExtraRootCaLoader>,
}
impl OidcExtraRootCaProvider {
pub fn new<F, Fut>(loader: F) -> Self
where
F: Fn() -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<OidcExtraRootCaMaterial, String>> + Send + 'static,
{
Self {
loader: Arc::new(move || Box::pin(loader())),
}
}
async fn load(&self) -> Result<OidcExtraRootCaMaterial, String> {
(self.loader)().await
}
}
#[derive(Clone, Default)]
struct CachedOidcExtraRootCerts {
generation: u64,
initialized: bool,
certs: Vec<Certificate>,
}
/// HTTP client adapter bridging reqwest 0.13 to the `openidconnect` `AsyncHttpClient` trait.
///
/// A fresh client is built for every request so the destination is re-validated and the
@@ -311,10 +350,26 @@ pub(crate) struct ReqwestHttpClient {
/// `None` in production: the process-cached outbound policy from the environment is used.
/// `Some(..)` only in tests, to explicitly allow a loopback mock endpoint.
policy_override: Option<OutboundPolicy>,
extra_root_certs: Arc<RwLock<CachedOidcExtraRootCerts>>,
extra_root_ca_provider: Option<OidcExtraRootCaProvider>,
#[cfg(test)]
dns_resolver_override: Option<Arc<dyn reqwest::dns::Resolve>>,
}
fn parse_oidc_extra_root_certs(source: &str, pem: &[u8]) -> Result<Vec<Certificate>, String> {
if pem.iter().all(|byte| byte.is_ascii_whitespace()) {
return Ok(Vec::new());
}
Certificate::from_pem_bundle(pem).map_err(|err| format!("failed to parse OIDC extra root CA bundle from {source}: {err}"))
}
fn oidc_extra_root_certs(root_ca_pem: Option<&[u8]>) -> Result<Vec<Certificate>, String> {
match root_ca_pem {
Some(pem) => parse_oidc_extra_root_certs("RustFS outbound TLS material", pem),
None => Ok(Vec::new()),
}
}
/// Build a reqwest client pinned to the shared outbound egress policy for a single request.
///
/// [`OutboundPolicy::resolver_for`] validates the URL shape and rejects loopback,
@@ -326,6 +381,7 @@ pub(crate) struct ReqwestHttpClient {
fn build_oidc_http_client(
uri: &str,
policy_override: Option<&OutboundPolicy>,
extra_root_certs: &[Certificate],
#[cfg(test)] dns_resolver_override: Option<Arc<dyn reqwest::dns::Resolve>>,
) -> Result<(Client, Url), OidcHttpError> {
let url = Url::parse(uri).map_err(|_| OidcHttpError::ForbiddenOutbound("invalid outbound OIDC URL".to_string()))?;
@@ -356,6 +412,9 @@ fn build_oidc_http_client(
if bypass_proxy {
builder = builder.no_proxy();
}
if !extra_root_certs.is_empty() {
builder = builder.tls_certs_merge(extra_root_certs.iter().cloned());
}
builder.build().map(|client| (client, url)).map_err(OidcHttpError::Reqwest)
}
@@ -410,19 +469,93 @@ fn should_bypass_proxy_for_oidc_uri(uri: &str) -> bool {
impl ReqwestHttpClient {
fn new() -> Result<Self, String> {
Self::new_with_extra_root_certs(Vec::new())
}
fn extra_root_cert_cache(certs: Vec<Certificate>) -> Arc<RwLock<CachedOidcExtraRootCerts>> {
Arc::new(RwLock::new(CachedOidcExtraRootCerts {
generation: 0,
initialized: true,
certs,
}))
}
fn new_with_extra_root_certs(extra_root_certs: Vec<Certificate>) -> Result<Self, String> {
Ok(Self {
policy_override: None,
extra_root_certs: Self::extra_root_cert_cache(extra_root_certs),
extra_root_ca_provider: None,
#[cfg(test)]
dns_resolver_override: None,
})
}
fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<Self, String> {
Ok(Self {
policy_override: None,
extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())),
extra_root_ca_provider: Some(extra_root_ca_provider),
#[cfg(test)]
dns_resolver_override: None,
})
}
async fn current_extra_root_certs(&self) -> Result<Vec<Certificate>, OidcHttpError> {
let Some(provider) = self.extra_root_ca_provider.as_ref() else {
return self
.extra_root_certs
.read()
.map(|cache| cache.certs.clone())
.map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}")));
};
let material = provider.load().await.map_err(OidcHttpError::ExtraRootCa)?;
if let Ok(cache) = self.extra_root_certs.read()
&& cache.initialized
&& cache.generation == material.generation
{
return Ok(cache.certs.clone());
}
let certs = oidc_extra_root_certs(material.root_ca_pem.as_deref()).map_err(OidcHttpError::ExtraRootCa)?;
let mut cache = self
.extra_root_certs
.write()
.map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}")))?;
cache.generation = material.generation;
cache.initialized = true;
cache.certs = certs.clone();
Ok(certs)
}
/// Test-only constructor that pins outbound requests to an explicit policy, so a
/// loopback mock server can be reached without depending on process-wide environment.
#[cfg(test)]
fn with_policy(policy: OutboundPolicy) -> Self {
Self {
policy_override: Some(policy),
extra_root_certs: Self::extra_root_cert_cache(Vec::new()),
extra_root_ca_provider: None,
dns_resolver_override: None,
}
}
#[cfg(test)]
fn with_policy_and_extra_root_certs(policy: OutboundPolicy, extra_root_certs: Vec<Certificate>) -> Self {
Self {
policy_override: Some(policy),
extra_root_certs: Self::extra_root_cert_cache(extra_root_certs),
extra_root_ca_provider: None,
dns_resolver_override: None,
}
}
#[cfg(test)]
fn with_policy_and_extra_root_ca_provider(policy: OutboundPolicy, extra_root_ca_provider: OidcExtraRootCaProvider) -> Self {
Self {
policy_override: Some(policy),
extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())),
extra_root_ca_provider: Some(extra_root_ca_provider),
dns_resolver_override: None,
}
}
@@ -431,6 +564,8 @@ impl ReqwestHttpClient {
fn with_policy_and_dns_resolver(policy: OutboundPolicy, resolver: Arc<dyn reqwest::dns::Resolve>) -> Self {
Self {
policy_override: Some(policy),
extra_root_certs: Self::extra_root_cert_cache(Vec::new()),
extra_root_ca_provider: None,
dns_resolver_override: Some(resolver),
}
}
@@ -461,9 +596,11 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
);
}
let extra_root_certs = self.current_extra_root_certs().await?;
let (client, url) = build_oidc_http_client(
&uri,
self.policy_override.as_ref(),
&extra_root_certs,
#[cfg(test)]
self.dns_resolver_override.clone(),
)?;
@@ -678,7 +815,22 @@ fn trusted_aud(other_audiences: &[String], audience: &Audience) -> bool {
impl OidcSys {
/// Parse environment variables and discover all configured OIDC providers.
pub async fn new() -> Result<Self, String> {
let http_client = ReqwestHttpClient::new()?;
Self::new_with_extra_root_ca(None).await
}
/// Parse environment variables and discover providers with an additional outbound root CA bundle.
pub(crate) async fn new_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result<Self, String> {
let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?;
Self::new_with_http_client(http_client).await
}
pub(crate) async fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<Self, String> {
let http_client = ReqwestHttpClient::new_with_extra_root_ca_provider(extra_root_ca_provider)?;
http_client.current_extra_root_certs().await.map_err(|err| err.to_string())?;
Self::new_with_http_client(http_client).await
}
async fn new_with_http_client(http_client: ReqwestHttpClient) -> Result<Self, String> {
let server_config = crate::server_config::current_server_config();
let parsed_configs = load_effective_oidc_provider_configs(server_config.as_ref());
let mut configs = HashMap::new();
@@ -1874,7 +2026,14 @@ pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>
}
pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
let http_client = ReqwestHttpClient::new()?;
validate_oidc_provider_config_with_extra_root_ca(config, None).await
}
pub async fn validate_oidc_provider_config_with_extra_root_ca(
config: &OidcProviderConfig,
root_ca_pem: Option<&[u8]>,
) -> Result<OidcProviderValidationResult, String> {
let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?;
let state = OidcSys::discover_provider(config, &http_client).await?;
Ok(OidcProviderValidationResult {
@@ -2438,6 +2597,50 @@ mod tests {
}
}
fn read_mock_oidc_request_path(stream: &mut impl std::io::Read) -> String {
let mut request_bytes = Vec::new();
let mut buffer = [0u8; 4096];
loop {
match stream.read(&mut buffer) {
Ok(0) => break,
Ok(n) => request_bytes.extend_from_slice(&buffer[..n]),
Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => break,
Err(_) => break,
}
if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") {
break;
}
if request_bytes.len() >= 8192 {
break;
}
}
let request = String::from_utf8_lossy(&request_bytes);
request
.lines()
.next()
.unwrap_or("")
.split_whitespace()
.nth(1)
.unwrap_or("")
.to_string()
}
fn mock_oidc_response(path: &str, discovery_body: &str, expected_jwks_path: &str, jwks_body: &str) -> String {
let (status, body) = if path.contains("/.well-known/openid-configuration") {
(200, discovery_body)
} else if path == expected_jwks_path {
(200, jwks_body)
} else {
(404, r#"{"error":"not found"}"#)
};
format!(
"HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
if status == 200 { "OK" } else { "Not Found" },
body.len()
)
}
fn start_mock_oidc_discovery_server<F>(
build_discovery_issuer: F,
max_requests: usize,
@@ -2445,7 +2648,6 @@ mod tests {
where
F: Fn(&str) -> (String, String, String) + Send + 'static,
{
use std::io::Read;
use std::io::Write;
use std::net::{Shutdown, TcpListener};
use std::sync::mpsc;
@@ -2516,41 +2718,8 @@ mod tests {
.set_read_timeout(Some(Duration::from_secs(1)))
.expect("failed to set discovery mock read timeout");
let mut request_bytes = Vec::new();
let mut buffer = [0u8; 4096];
loop {
match stream.read(&mut buffer) {
Ok(0) => break,
Ok(n) => request_bytes.extend_from_slice(&buffer[..n]),
Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => {
break;
}
Err(_) => break,
}
if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") {
break;
}
if request_bytes.len() >= 8192 {
break;
}
}
let request = String::from_utf8_lossy(&request_bytes);
let path = request.lines().next().unwrap_or("").split_whitespace().nth(1).unwrap_or("");
let (status, body) = if path.contains("/.well-known/openid-configuration") {
(200, discovery_body.as_str())
} else if path == expected_jwks_path {
(200, jwks_body)
} else {
(404, r#"{"error":"not found"}"#)
};
let response = format!(
"HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
if status == 200 { "OK" } else { "Not Found" },
body.len()
);
let path = read_mock_oidc_request_path(&mut stream);
let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body);
let _ = stream.write_all(response.as_bytes());
let _ = stream.flush();
let _ = stream.shutdown(Shutdown::Both);
@@ -2568,6 +2737,114 @@ mod tests {
Some((base, handle))
}
fn start_mock_oidc_tls_discovery_server<F>(
build_discovery_issuer: F,
max_requests: usize,
) -> Option<(String, String, std::thread::JoinHandle<()>)>
where
F: Fn(&str) -> (String, String, String) + Send + 'static,
{
use std::io::Write;
use std::net::{Shutdown, TcpListener};
use std::sync::mpsc;
use std::time::{Duration, Instant};
const IDLE_SHUTDOWN: Duration = Duration::from_secs(1);
const ABSOLUTE_CAP: Duration = Duration::from_secs(5);
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let certified =
rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).expect("generate OIDC TLS test certificate");
let cert_pem = certified.cert.pem();
let server_config = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(
vec![certified.cert.der().clone()],
rustls_pki_types::PrivateKeyDer::try_from(certified.signing_key.serialize_der())
.expect("convert OIDC TLS test private key"),
)
.expect("build OIDC TLS mock server config");
let listener = match TcpListener::bind("127.0.0.1:0") {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("test TLS listener should bind: {err}"),
};
let base = format!("https://{}", listener.local_addr().expect("listener local address should be available"));
let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base);
let discovery_body = serde_json::json!({
"issuer": discovery_issuer,
"authorization_endpoint": format!("{base}/authorize"),
"token_endpoint": format!("{base}/token"),
"jwks_uri": discovery_jwks_uri,
"response_types_supported": ["code"],
"response_modes_supported": ["query"],
"subject_types_supported": ["public"],
"id_token_signing_alg_values_supported": ["RS256"],
})
.to_string();
let jwks_body = r#"{"keys":[]}"#;
let (ready_tx, ready_rx) = mpsc::channel();
let handle = std::thread::spawn(move || {
let server_config = Arc::new(server_config);
listener
.set_nonblocking(true)
.expect("failed to set TLS discovery mock listener non-blocking");
let _ = ready_tx.send(());
let mut seen = 0usize;
let start = Instant::now();
let mut last_completed = Instant::now();
loop {
if seen > 0 && last_completed.elapsed() >= IDLE_SHUTDOWN {
break;
}
if start.elapsed() >= ABSOLUTE_CAP {
break;
}
let tcp_stream = match listener.accept() {
Ok((stream, _)) => stream,
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(Duration::from_millis(5));
continue;
}
Err(_) => break,
};
tcp_stream
.set_nonblocking(false)
.expect("failed to set TLS discovery mock stream blocking");
tcp_stream
.set_read_timeout(Some(Duration::from_secs(1)))
.expect("failed to set TLS discovery mock read timeout");
seen += 1;
let connection = match rustls::ServerConnection::new(server_config.clone()) {
Ok(connection) => connection,
Err(_) => break,
};
let mut stream = rustls::StreamOwned::new(connection, tcp_stream);
let path = read_mock_oidc_request_path(&mut stream);
let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body);
let _ = stream.write_all(response.as_bytes());
let _ = stream.flush();
let _ = stream.sock.shutdown(Shutdown::Both);
last_completed = Instant::now();
if seen >= max_requests {
break;
}
}
});
ready_rx
.recv_timeout(Duration::from_millis(100))
.expect("mock TLS OIDC discovery server should become ready");
Some((base, cert_pem, handle))
}
fn discovery_error_contains_all_variants(err: &str, base: &str) -> bool {
err.contains(base) && err.contains(&format!("{base}/")) && err.contains("discovery failed for all issuer variants")
}
@@ -2590,6 +2867,100 @@ mod tests {
})
}
#[tokio::test]
async fn oidc_discovery_accepts_extra_root_ca_for_https_provider() {
let Some((base, ca_pem, handle)) = start_mock_oidc_tls_discovery_server(
|base| (format!("{base}/application/o/rustfs"), format!("{base}/jwks"), "/jwks".to_string()),
4,
) else {
return;
};
let config_url = format!("{base}/application/o/rustfs");
let config = build_mocked_oidc_provider_config("default", &config_url);
let origin = Url::parse(&config.config_url)
.expect("mock config_url should parse")
.origin()
.ascii_serialization();
let policy = OutboundPolicy::from_allowed_origins(&origin).expect("loopback TLS origin should be allowed");
let extra_root_certs =
parse_oidc_extra_root_certs("test OIDC TLS CA", ca_pem.as_bytes()).expect("test CA bundle should parse");
let http_client = ReqwestHttpClient::with_policy_and_extra_root_certs(policy, extra_root_certs);
let state = OidcSys::discover_provider(&config, &http_client)
.await
.expect("OIDC discovery should trust the extra root CA");
assert_eq!(state.metadata.issuer().to_string(), format!("{base}/application/o/rustfs"));
assert!(handle.join().is_ok());
}
#[tokio::test]
async fn oidc_discovery_refreshes_extra_root_ca_when_generation_changes() {
let Some((base_a, ca_pem_a, handle_a)) = start_mock_oidc_tls_discovery_server(
|base| (format!("{base}/application/o/rustfs-a"), format!("{base}/jwks"), "/jwks".to_string()),
4,
) else {
return;
};
let Some((base_b, ca_pem_b, handle_b)) = start_mock_oidc_tls_discovery_server(
|base| (format!("{base}/application/o/rustfs-b"), format!("{base}/jwks"), "/jwks".to_string()),
4,
) else {
assert!(handle_a.join().is_ok());
return;
};
let origin_a = Url::parse(&base_a)
.expect("mock base A should parse")
.origin()
.ascii_serialization();
let origin_b = Url::parse(&base_b)
.expect("mock base B should parse")
.origin()
.ascii_serialization();
let allowed_origins = format!("{origin_a},{origin_b}");
let policy = OutboundPolicy::from_allowed_origins(&allowed_origins).expect("loopback TLS origins should be allowed");
let material = Arc::new(Mutex::new(OidcExtraRootCaMaterial {
generation: 1,
root_ca_pem: Some(ca_pem_a.into_bytes()),
}));
let provider = OidcExtraRootCaProvider::new({
let material = material.clone();
move || {
let material = material.clone();
async move {
material
.lock()
.map(|material| material.clone())
.map_err(|e| format!("test OIDC extra CA material lock poisoned: {e}"))
}
}
});
let http_client = ReqwestHttpClient::with_policy_and_extra_root_ca_provider(policy, provider);
let config_a = build_mocked_oidc_provider_config("a", &format!("{base_a}/application/o/rustfs-a"));
let state_a = OidcSys::discover_provider(&config_a, &http_client)
.await
.expect("OIDC discovery should trust initial extra root CA");
assert_eq!(state_a.metadata.issuer().to_string(), format!("{base_a}/application/o/rustfs-a"));
{
let mut material = material
.lock()
.expect("test OIDC extra CA material lock should not be poisoned");
material.generation = 2;
material.root_ca_pem = Some(ca_pem_b.into_bytes());
}
let config_b = build_mocked_oidc_provider_config("b", &format!("{base_b}/application/o/rustfs-b"));
let state_b = OidcSys::discover_provider(&config_b, &http_client)
.await
.expect("OIDC discovery should refresh extra root CA after generation change");
assert_eq!(state_b.metadata.issuer().to_string(), format!("{base_b}/application/o/rustfs-b"));
assert!(handle_a.join().is_ok());
assert!(handle_b.join().is_ok());
}
#[tokio::test]
async fn test_validate_oidc_provider_config_retries_with_issuer_candidates() {
// Discovery document must advertise the canonical issuer path. The first candidate has no
@@ -2945,7 +3316,7 @@ mod tests {
// Cloud metadata endpoint is never allowed.
assert!(
matches!(
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, None),
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, &[], None),
Err(OidcHttpError::ForbiddenOutbound(_))
),
"metadata endpoint must be rejected"
@@ -2953,7 +3324,7 @@ mod tests {
// Loopback is rejected by default (no allow-origins configured).
assert!(
matches!(
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, None),
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, &[], None),
Err(OidcHttpError::ForbiddenOutbound(_))
),
"loopback must be rejected by default"
@@ -2961,7 +3332,7 @@ mod tests {
// A public hostname passes the up-front shape/host check; the resolved IP is still
// re-classified at connection time by the pinned resolver.
assert!(
build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, None).is_ok(),
build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, &[], None).is_ok(),
"public https endpoint should build"
);
}
@@ -2981,13 +3352,13 @@ mod tests {
fn build_oidc_http_client_honors_explicit_allowlist_for_loopback() {
let policy = OutboundPolicy::from_allowed_origins("http://127.0.0.1:8080").expect("origin should parse");
assert!(
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), None).is_ok(),
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), &[], None).is_ok(),
"explicitly allow-listed loopback origin should build"
);
// A metadata endpoint stays forbidden even when a loopback origin is allow-listed.
assert!(
matches!(
build_oidc_http_client("http://169.254.169.254/", Some(&policy), None),
build_oidc_http_client("http://169.254.169.254/", Some(&policy), &[], None),
Err(OidcHttpError::ForbiddenOutbound(_))
),
"metadata endpoint stays forbidden despite an unrelated allow-list entry"
@@ -3190,8 +3561,9 @@ mod tests {
#[test]
fn oidc_metadata_endpoint_rejection_does_not_offer_allowlist_bypass() {
let error = build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), None)
.expect_err("metadata endpoint must remain forbidden");
let error =
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), &[], None)
.expect_err("metadata endpoint must remain forbidden");
let message = error.to_string();
assert!(message.contains("metadata endpoint"));
+7 -1
View File
@@ -449,9 +449,15 @@ impl Operation for ValidateOidcConfigHandler {
request.provider_id.trim().to_string()
};
let provider_config = build_provider_config_from_validate(request, &provider_id)?;
let validation = rustfs_iam::oidc::validate_oidc_provider_config(&provider_config)
let oidc_extra_root_ca = crate::startup_auth::current_oidc_extra_root_ca_material()
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("validation failed: {e}")))?;
let validation = rustfs_iam::oidc::validate_oidc_provider_config_with_extra_root_ca(
&provider_config,
oidc_extra_root_ca.root_ca_pem.as_deref(),
)
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRequest, format!("validation failed: {e}")))?;
json_response(
StatusCode::OK,
+139 -3
View File
@@ -24,9 +24,10 @@ use crate::startup_runtime_sources;
use rustfs_common::MtlsIdentityPem;
use rustfs_config::{
DEFAULT_SERVER_MTLS_ENABLE, DEFAULT_TLS_KEYLOG, DEFAULT_TLS_RELOAD_ENABLE, DEFAULT_TLS_RELOAD_INTERVAL,
DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_SERVER_MTLS_ENABLE,
ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_SYSTEM_CA,
RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME, RUSTFS_TLS_CERT,
DEFAULT_TRUST_LEAF_CERT_AS_CA, DEFAULT_TRUST_SYSTEM_CA, ENV_MTLS_CLIENT_CERT, ENV_MTLS_CLIENT_KEY, ENV_RUSTFS_EXTRA_CA_CERT,
ENV_SERVER_MTLS_ENABLE, ENV_TLS_KEYLOG, ENV_TLS_RELOAD_ENABLE, ENV_TLS_RELOAD_INTERVAL, ENV_TRUST_LEAF_CERT_AS_CA,
ENV_TRUST_SYSTEM_CA, RUSTFS_CA_CERT, RUSTFS_CLIENT_CA_CERT_FILENAME, RUSTFS_CLIENT_CERT_FILENAME, RUSTFS_CLIENT_KEY_FILENAME,
RUSTFS_TLS_CERT,
};
use rustfs_tls_runtime::{
ServerTlsMaterial as RuntimeServerTlsMaterial, TlsGeneration, TlsSource, WebPkiClientVerifierOptions,
@@ -34,6 +35,7 @@ use rustfs_tls_runtime::{
};
use rustfs_utils::{get_env_bool, get_env_opt_str};
use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject};
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::RwLock;
@@ -267,6 +269,60 @@ fn map_runtime_tls_error(err: rustfs_tls_runtime::TlsRuntimeError) -> TlsMateria
}
}
pub(crate) async fn validate_configured_oidc_extra_ca_cert() -> Result<(), TlsMaterialError> {
if let Some(path) = configured_oidc_extra_ca_cert_path() {
let _ = load_configured_oidc_extra_ca_cert().await?;
info!(
component = LOG_COMPONENT_TLS,
subsystem = LOG_SUBSYSTEM_TLS,
event = "oidc_extra_ca_validated",
source = "oidc_extra_ca_bundle",
env_var = ENV_RUSTFS_EXTRA_CA_CERT,
path = ?path,
"OIDC extra root CA bundle validated"
);
}
Ok(())
}
pub(crate) async fn load_configured_oidc_extra_ca_cert() -> Result<Option<Vec<u8>>, TlsMaterialError> {
let Some(path) = configured_oidc_extra_ca_cert_path() else {
return Ok(None);
};
let data = tokio::fs::read(&path)
.await
.map_err(|e| TlsMaterialError::Io(format!("read extra CA bundle {path:?}: {e}")))?;
validate_cert_bundle(&data, &path)?;
Ok(Some(data))
}
fn configured_oidc_extra_ca_cert_path() -> Option<PathBuf> {
let path = get_env_opt_str(ENV_RUSTFS_EXTRA_CA_CERT)?;
let path = path.trim();
if path.is_empty() {
return None;
}
Some(PathBuf::from(path))
}
fn validate_cert_bundle(data: &[u8], path: &Path) -> Result<(), TlsMaterialError> {
let mut reader = Cursor::new(data);
let mut found = false;
let mut store = rustls::RootCertStore::empty();
for cert in CertificateDer::pem_reader_iter(&mut reader) {
let cert = cert.map_err(|e| TlsMaterialError::Parse(format!("invalid extra CA bundle {path:?}: {e}")))?;
store
.add(cert)
.map_err(|e| TlsMaterialError::Parse(format!("invalid extra CA bundle {path:?}: {e}")))?;
found = true;
}
if !found {
return Err(TlsMaterialError::Parse(format!("no certificate found in extra CA bundle {path:?}")));
}
Ok(())
}
/// Load a single certificate file and append PEM data.
/// Returns true if the file was successfully loaded.
async fn load_cert_file(path: &Path, pem_data: &mut Vec<u8>, desc: &str) -> bool {
@@ -681,6 +737,86 @@ mod tests {
fs::write(dir.join(rustfs_config::RUSTFS_TLS_KEY), signing_key.serialize_pem()).unwrap();
}
#[tokio::test]
#[serial_test::serial]
async fn oidc_extra_ca_cert_loads_configured_bundle() {
let CertifiedKey { cert, .. } =
rcgen::generate_simple_self_signed(vec!["extra-ca.example".to_string()]).expect("generate extra CA cert");
let temp_file = tempfile::NamedTempFile::new().expect("create extra CA file");
fs::write(temp_file.path(), cert.pem()).expect("write extra CA file");
let path = temp_file.path().to_string_lossy().to_string();
temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async {
let extra_ca = load_configured_oidc_extra_ca_cert()
.await
.expect("OIDC extra CA should load")
.expect("configured OIDC extra CA should be present");
assert!(extra_ca.starts_with(cert.pem().as_bytes()));
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn oidc_extra_ca_cert_rejects_invalid_pem() {
let temp_file = tempfile::NamedTempFile::new().expect("create invalid extra CA file");
fs::write(temp_file.path(), b"not a certificate").expect("write invalid extra CA file");
let path = temp_file.path().to_string_lossy().to_string();
temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async {
let err = load_configured_oidc_extra_ca_cert()
.await
.expect_err("invalid extra CA should fail");
assert!(err.to_string().contains("no certificate found in extra CA bundle"));
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn oidc_extra_ca_cert_rejects_malformed_der_certificate() {
let temp_file = tempfile::NamedTempFile::new().expect("create malformed extra CA file");
fs::write(
temp_file.path(),
b"-----BEGIN CERTIFICATE-----\nbm90IGEgY2VydA==\n-----END CERTIFICATE-----\n",
)
.expect("write malformed extra CA file");
let path = temp_file.path().to_string_lossy().to_string();
temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async {
let err = load_configured_oidc_extra_ca_cert()
.await
.expect_err("malformed DER in PEM framing should fail");
assert!(err.to_string().contains("invalid extra CA bundle"));
})
.await;
}
#[tokio::test]
#[serial_test::serial]
async fn load_tls_material_does_not_append_oidc_extra_ca_cert() {
let temp_dir = TempDir::new().expect("create TLS material dir");
write_test_cert_pair(temp_dir.path(), "server.example");
let CertifiedKey { cert, .. } =
rcgen::generate_simple_self_signed(vec!["extra-ca.example".to_string()]).expect("generate extra CA cert");
let temp_file = tempfile::NamedTempFile::new().expect("create extra CA file");
fs::write(temp_file.path(), cert.pem()).expect("write extra CA file");
let path = temp_file.path().to_string_lossy().to_string();
temp_env::async_with_vars([(ENV_RUSTFS_EXTRA_CA_CERT, Some(path.as_str()))], async {
let snapshot = load_tls_material(temp_dir.path().to_str().expect("TLS material dir should be utf-8"))
.await
.expect("TLS material should load");
assert!(snapshot.outbound.root_ca_pem.is_empty());
assert!(snapshot.server.is_some());
})
.await;
}
#[tokio::test]
async fn build_acceptor_accepts_root_single_cert_with_trailing_slash() {
ensure_rustls_crypto_provider();
+42 -2
View File
@@ -14,9 +14,12 @@
use rustfs_iam::{
federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter},
get_oidc, init_oidc_sys,
get_oidc, init_oidc_sys_with_extra_root_ca_provider,
oidc::{OidcExtraRootCaMaterial, OidcExtraRootCaProvider},
};
use std::{
collections::hash_map::DefaultHasher,
hash::{Hash, Hasher},
io::{Error, Result},
sync::Arc,
};
@@ -50,7 +53,7 @@ pub(crate) async fn init_auth_integrations() -> Result<()> {
}
}
match init_oidc_sys().await {
match init_oidc_sys_with_extra_root_ca_provider(oidc_extra_root_ca_provider()).await {
Ok(()) => {
if let Some(oidc) = get_oidc() {
let adapter = Arc::new(StandardOidcAdapter::new(oidc));
@@ -72,3 +75,40 @@ pub(crate) async fn init_auth_integrations() -> Result<()> {
Ok(())
}
pub(crate) fn oidc_extra_root_ca_provider() -> OidcExtraRootCaProvider {
OidcExtraRootCaProvider::new(current_oidc_extra_root_ca_material)
}
pub(crate) async fn current_oidc_extra_root_ca_material() -> std::result::Result<OidcExtraRootCaMaterial, String> {
let outbound_tls = crate::runtime_sources::current_outbound_tls_state().await;
let outbound_generation = outbound_tls.as_ref().map(|state| state.generation.0).unwrap_or_default();
let mut root_ca_pem = outbound_tls.as_ref().and_then(|state| state.root_ca_pem.clone());
if let Some(extra_ca_pem) = crate::server::tls_material::load_configured_oidc_extra_ca_cert()
.await
.map_err(|err| err.to_string())?
{
match root_ca_pem.as_mut() {
Some(root_ca_pem) => {
if !root_ca_pem.is_empty() && !root_ca_pem.ends_with(b"\n") {
root_ca_pem.push(b'\n');
}
root_ca_pem.extend_from_slice(&extra_ca_pem);
}
None => root_ca_pem = Some(extra_ca_pem),
}
}
Ok(OidcExtraRootCaMaterial {
generation: oidc_extra_root_ca_generation(outbound_generation, root_ca_pem.as_deref()),
root_ca_pem,
})
}
fn oidc_extra_root_ca_generation(outbound_generation: u64, root_ca_pem: Option<&[u8]>) -> u64 {
let mut hasher = DefaultHasher::new();
outbound_generation.hash(&mut hasher);
root_ca_pem.hash(&mut hasher);
hasher.finish()
}
+23 -12
View File
@@ -24,21 +24,14 @@ const EVENT_TLS_OUTBOUND_INITIALIZATION_FAILED: &str = "tls_outbound_initializat
const TLS_STARTUP_GENERATION_CONSUMER: &str = "rustfs_server_startup";
pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> {
crate::server::tls_material::validate_configured_oidc_extra_ca_cert()
.await
.map_err(|err| Error::other(err.to_string()))?;
if let Some(tls_path) = normalized_tls_path(config.tls_path.as_deref()) {
match crate::server::tls_material::load_tls_material(tls_path).await {
Ok(snapshot) => {
let generation = next_tls_generation(startup_runtime_sources::current_outbound_tls_generation());
startup_runtime_sources::publish_outbound_tls_state(generation, &snapshot.outbound).await;
startup_runtime_sources::record_tls_generation(TLS_STARTUP_GENERATION_CONSUMER, generation.0);
info!(
target: "rustfs::main",
event = EVENT_TLS_OUTBOUND_INITIALIZED,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
tls_path,
generation = generation.0,
"Initialized TLS outbound material"
);
publish_outbound_tls_material(&snapshot.outbound, Some(tls_path)).await;
}
Err(err) => {
error!(
@@ -61,6 +54,24 @@ pub(crate) async fn init_outbound_tls_material(config: &Config) -> Result<()> {
Ok(())
}
async fn publish_outbound_tls_material(outbound: &rustfs_tls_runtime::OutboundTlsMaterial, tls_path: Option<&str>) {
let generation = next_tls_generation(startup_runtime_sources::current_outbound_tls_generation());
startup_runtime_sources::publish_outbound_tls_state(generation, outbound).await;
startup_runtime_sources::record_tls_generation(TLS_STARTUP_GENERATION_CONSUMER, generation.0);
info!(
target: "rustfs::main",
event = EVENT_TLS_OUTBOUND_INITIALIZED,
component = LOG_COMPONENT_MAIN,
subsystem = LOG_SUBSYSTEM_STARTUP,
state = "initialized",
tls_path = tls_path.unwrap_or(""),
generation = generation.0,
has_root_ca = !outbound.root_ca_pem.is_empty(),
has_mtls_identity = outbound.mtls_identity.is_some(),
"Initialized TLS outbound material"
);
}
fn normalized_tls_path(path: Option<&str>) -> Option<&str> {
path.map(str::trim).filter(|value| !value.is_empty())
}