fix(startup): never panic when the system CA bundle is absent (#6769)

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -352,6 +352,28 @@ pub struct BucketTargetSys {
     heartbeat_started: OnceLock<()>,
 }
 
+/// Build the bucket-target health-check HTTP client without panicking when
+/// the host has no system CA bundle (issue #6734).
+///
+/// `BucketTargetSys::get()` initializes lazily on the startup path (bucket
+/// metadata install calls it on the main thread), and `reqwest::Client::new()`
+/// panics when the TLS backend cannot load any system trust root — the state
+/// of a minimal container image. Fall back to a client with an explicit empty
+/// trust store: HTTP health checks keep working, and HTTPS targets fail closed
+/// at the TLS handshake with a clear certificate error instead of aborting
+/// the whole process at startup.
+fn build_health_check_client() -> HttpClient {
+    HttpClient::builder().build().unwrap_or_else(|error| {
+        warn!(
+            "bucket target health-check HTTP client could not load system TLS roots ({error}); continuing with an empty trust store — HTTPS target health checks will fail until a CA bundle is installed"
+        );
+        HttpClient::builder()
+            .tls_certs_only(std::iter::empty::<reqwest::Certificate>())
+            .build()
+            .expect("HTTP client construction must succeed with an explicit empty trust store")
+    })
+}
+
 impl BucketTargetSys {
     pub fn get() -> &'static Self {
         GLOBAL_BUCKET_TARGET_SYS.get_or_init(Self::new)
@@ -364,7 +386,7 @@ impl BucketTargetSys {
             targets_map: Arc::new(RwLock::new(HashMap::new())),
             h_mutex: Arc::new(RwLock::new(HashMap::new())),
             target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
-            hc_client: Arc::new(HttpClient::new()),
+            hc_client: Arc::new(build_health_check_client()),
             a_mutex: Arc::new(Mutex::new(HashMap::new())),
             arn_errs_map: Arc::new(RwLock::new(HashMap::new())),
             target_update_mutexes: Arc::new(Mutex::new(HashMap::new())),
@@ -2490,6 +2512,18 @@ mod tests {
     use super::*;
     use rcgen::generate_simple_self_signed;
 
+    // The startup panic fix for hosts without a CA bundle (issue #6734) rests
+    // on two properties: the health-check client constructor never panics, and
+    // its degraded fallback — an explicit empty trust store — always builds.
+    #[test]
+    fn health_check_client_construction_never_panics() {
+        let _ = build_health_check_client();
+        HttpClient::builder()
+            .tls_certs_only(std::iter::empty::<reqwest::Certificate>())
+            .build()
+            .expect("empty-trust-store client build must succeed without touching system roots");
+    }
+
     #[derive(Clone, Debug)]
     struct RecordingHttpConnector {
         request_uris: Arc<std::sync::Mutex<Vec<String>>>,

crates/policy/src/policy/opa.rs

@@ -173,20 +173,28 @@ pub async fn lookup_config() -> Result<Args, OpaConfigError> {
 
 impl AuthZPlugin {
     pub fn new(config: Args) -> Self {
-        let client = reqwest::Client::builder()
-            .timeout(Duration::from_secs(5))
-            .connect_timeout(Duration::from_secs(1))
-            .pool_max_idle_per_host(10)
-            .pool_idle_timeout(Some(Duration::from_secs(60)))
-            .tcp_keepalive(Some(Duration::from_secs(30)))
-            .tcp_nodelay(true)
-            .http2_keep_alive_interval(Some(Duration::from_secs(30)))
-            .http2_keep_alive_timeout(Duration::from_secs(15))
-            .build()
-            .unwrap_or_else(|err| {
-                error!("failed to build OPA HTTP client, falling back to default reqwest client: {err}");
-                reqwest::Client::new()
-            });
+        let builder = || {
+            reqwest::Client::builder()
+                .timeout(Duration::from_secs(5))
+                .connect_timeout(Duration::from_secs(1))
+                .pool_max_idle_per_host(10)
+                .pool_idle_timeout(Some(Duration::from_secs(60)))
+                .tcp_keepalive(Some(Duration::from_secs(30)))
+                .tcp_nodelay(true)
+                .http2_keep_alive_interval(Some(Duration::from_secs(30)))
+                .http2_keep_alive_timeout(Duration::from_secs(15))
+        };
+        // Never fall back to `reqwest::Client::new()`: it panics for the same
+        // reason the first build failed (e.g. no system CA bundle, issue
+        // #6734). Retry with an explicit empty trust store instead — an HTTP
+        // OPA endpoint keeps working, an HTTPS one fails closed per request.
+        let client = builder().build().unwrap_or_else(|err| {
+            error!("failed to build OPA HTTP client ({err}); continuing with an empty trust store");
+            builder()
+                .tls_certs_only(std::iter::empty::<reqwest::Certificate>())
+                .build()
+                .expect("HTTP client construction must succeed with an explicit empty trust store")
+        });
 
         Self { client, args: config }
     }

crates/trusted-proxies/src/cloud/metadata/aws.rs

@@ -43,7 +43,7 @@ impl AwsMetadataFetcher {
     ///
     /// Returns a new instance of `AwsMetadataFetcher`.
     pub fn new(timeout: Duration) -> Self {
-        let client = Client::builder().timeout(timeout).build().unwrap_or_else(|_| Client::new());
+        let client = super::metadata_http_client(timeout);
 
         Self {
             client,

crates/trusted-proxies/src/cloud/metadata/azure.rs

@@ -34,7 +34,7 @@ pub struct AzureMetadataFetcher {
 impl AzureMetadataFetcher {
     /// Creates a new `AzureMetadataFetcher`.
     pub fn new(timeout: Duration) -> Self {
-        let client = Client::builder().timeout(timeout).build().unwrap_or_else(|_| Client::new());
+        let client = super::metadata_http_client(timeout);
 
         Self {
             client,

crates/trusted-proxies/src/cloud/metadata/gcp.rs

@@ -34,7 +34,7 @@ pub struct GcpMetadataFetcher {
 impl GcpMetadataFetcher {
     /// Creates a new `GcpMetadataFetcher`.
     pub fn new(timeout: Duration) -> Self {
-        let client = Client::builder().timeout(timeout).build().unwrap_or_else(|_| Client::new());
+        let client = super::metadata_http_client(timeout);
 
         Self {
             client,

crates/trusted-proxies/src/cloud/metadata/mod.rs

@@ -24,3 +24,40 @@ mod gcp;
 pub use aws::*;
 pub use azure::*;
 pub use gcp::*;
+
+/// Build the metadata HTTP client without panicking when the host has no
+/// system CA bundle (issue #6734).
+///
+/// `reqwest::Client::new()` panics when the TLS backend cannot load any
+/// system trust root, which is exactly the state of a minimal container
+/// image. Cloud metadata endpoints are plain HTTP link-local addresses, so a
+/// client with an explicit empty trust store is fully functional here; TLS
+/// requests through it fail closed at the handshake.
+pub(crate) fn metadata_http_client(timeout: std::time::Duration) -> reqwest::Client {
+    reqwest::Client::builder().timeout(timeout).build().unwrap_or_else(|error| {
+        tracing::warn!(
+            "cloud metadata HTTP client could not load system TLS roots ({error}); continuing with an empty trust store"
+        );
+        reqwest::Client::builder()
+            .timeout(timeout)
+            .tls_certs_only(std::iter::empty::<reqwest::Certificate>())
+            .build()
+            .expect("HTTP client construction must succeed with an explicit empty trust store")
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    // The startup panic fix for hosts without a CA bundle (issue #6734) rests
+    // on the constructor never panicking and its degraded fallback — an
+    // explicit empty trust store — always building.
+    #[test]
+    fn metadata_http_client_construction_never_panics() {
+        let _ = super::metadata_http_client(std::time::Duration::from_secs(1));
+        reqwest::Client::builder()
+            .timeout(std::time::Duration::from_secs(1))
+            .tls_certs_only(std::iter::empty::<reqwest::Certificate>())
+            .build()
+            .expect("empty-trust-store client build must succeed without touching system roots");
+    }
+}