From 32346f159a9f6a191d6b893125a98b7667ee3fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Wed, 26 Aug 2026 13:32:29 +0800 Subject: [PATCH] feat(kms): wire Vault custom CA and mTLS client identity (#6638) --- Cargo.lock | 1 + crates/kms/Cargo.toml | 6 +- crates/kms/src/api_types.rs | 146 +++++++++++-- crates/kms/src/backends/vault.rs | 3 + crates/kms/src/backends/vault_credentials.rs | 205 ++++++++++++++++++ crates/kms/src/backends/vault_transit.rs | 3 + crates/kms/src/backup/vault_restore.rs | 6 +- crates/kms/src/config.rs | 180 ++++++++++++--- ...sts__kms_vault_transit_config_summary.snap | 4 +- docs/operations/kms-backend-security.md | 15 ++ 10 files changed, 517 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1768d6849..c0614289d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9869,6 +9869,7 @@ dependencies = [ "metrics-util", "moka", "rand 0.10.2", + "rcgen", "reqwest", "rustfs-s3-types", "rustfs-security-governance", diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index f1a41c888..57a60b1c6 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -74,7 +74,10 @@ rustfs-security-governance = { workspace = true } rustfs-s3-types = { workspace = true } # HTTP client for Vault -reqwest = { workspace = true } +# `rustls` named explicitly: `reqwest::Identity::from_pem` (Vault mTLS) is only +# available on the rustls TLS backend, and relying on another crate's feature +# unification to enable it would break silently if that crate changed. +reqwest = { workspace = true, features = ["rustls"] } vaultrs = { workspace = true } # vaultrs surfaces transport-level failures as wrapped rustify errors; the # operation policy needs the concrete type to classify them for retry decisions. @@ -98,6 +101,7 @@ metrics-util = { workspace = true, features = ["debugging"] } insta = { workspace = true, features = ["yaml", "json"] } tempfile = { workspace = true } temp-env = { workspace = true } +rcgen = { workspace = true } # "net" backs the scripted loopback Vault used by the policy wiring tests. tokio = { workspace = true, features = ["net", "test-util"] } # Replays canned AWS KMS HTTP exchanges so the AWS backend tests stay offline. diff --git a/crates/kms/src/api_types.rs b/crates/kms/src/api_types.rs index 6faf6cacc..64cafc723 100644 --- a/crates/kms/src/api_types.rs +++ b/crates/kms/src/api_types.rs @@ -94,6 +94,12 @@ pub struct ConfigureVaultKmsRequest { pub key_path_prefix: Option, /// Skip TLS verification (insecure, for development only) pub skip_tls_verify: Option, + /// Path to a PEM CA bundle trusted for the Vault connection (server-local path) + pub ca_cert_path: Option, + /// Path to a PEM client certificate presented to Vault for mTLS (requires client_key_path) + pub client_cert_path: Option, + /// Path to the PEM private key matching client_cert_path + pub client_key_path: Option, /// Default master key ID for auto-encryption pub default_key_id: Option, /// Operation timeout in seconds @@ -125,6 +131,12 @@ pub struct ConfigureVaultTransitKmsRequest { pub mount_path: Option, /// Skip TLS verification (insecure, for development only) pub skip_tls_verify: Option, + /// Path to a PEM CA bundle trusted for the Vault connection (server-local path) + pub ca_cert_path: Option, + /// Path to a PEM client certificate presented to Vault for mTLS (requires client_key_path) + pub client_cert_path: Option, + /// Path to the PEM private key matching client_cert_path + pub client_key_path: Option, /// Default master key ID for auto-encryption pub default_key_id: Option, /// Operation timeout in seconds @@ -461,6 +473,12 @@ pub enum BackendSummary { key_path_prefix: String, /// Skip TLS verification skip_tls_verify: bool, + /// Whether a custom CA bundle is configured for the connection + #[serde(default)] + has_custom_ca: bool, + /// Whether an mTLS client certificate/key pair is configured + #[serde(default)] + has_client_identity: bool, }, /// Vault Transit backend summary VaultTransit { @@ -476,6 +494,12 @@ pub enum BackendSummary { mount_path: String, /// Skip TLS verification skip_tls_verify: bool, + /// Whether a custom CA bundle is configured for the connection + #[serde(default)] + has_custom_ca: bool, + /// Whether an mTLS client certificate/key pair is configured + #[serde(default)] + has_client_identity: bool, }, /// Static single-key backend summary Static { @@ -528,6 +552,8 @@ impl From<&KmsConfig> for KmsConfigSummary { kv_mount: vault_config.kv_mount.clone(), key_path_prefix: vault_config.key_path_prefix.clone(), skip_tls_verify: vault_config.tls.as_ref().is_some_and(|tls| tls.skip_verify), + has_custom_ca: vault_config.tls.as_ref().is_some_and(|tls| tls.ca_cert_path.is_some()), + has_client_identity: vault_config.tls.as_ref().is_some_and(|tls| tls.client_cert_path.is_some()), }, BackendConfig::VaultTransit(vault_config) => BackendSummary::VaultTransit { address: vault_config.address.clone(), @@ -541,6 +567,8 @@ impl From<&KmsConfig> for KmsConfigSummary { namespace: vault_config.namespace.clone(), mount_path: vault_config.mount_path.clone(), skip_tls_verify: vault_config.tls.as_ref().is_some_and(|tls| tls.skip_verify), + has_custom_ca: vault_config.tls.as_ref().is_some_and(|tls| tls.ca_cert_path.is_some()), + has_client_identity: vault_config.tls.as_ref().is_some_and(|tls| tls.client_cert_path.is_some()), }, BackendConfig::Static(static_config) => BackendSummary::Static { key_id: static_config.key_id.clone(), @@ -593,6 +621,25 @@ impl ConfigureLocalKmsRequest { } } +/// Assemble the Vault TLS settings named by a configure request. +/// +/// Mirrors the env-side assembly: `None` when nothing TLS-related was +/// requested, so the persisted config keeps its historical shape. +fn vault_tls_from_request( + skip_tls_verify: Option, + ca_cert_path: Option<&String>, + client_cert_path: Option<&String>, + client_key_path: Option<&String>, +) -> Option { + let skip_verify = skip_tls_verify.unwrap_or(false); + (skip_verify || ca_cert_path.is_some() || client_cert_path.is_some() || client_key_path.is_some()).then(|| TlsConfig { + ca_cert_path: ca_cert_path.map(PathBuf::from), + client_cert_path: client_cert_path.map(PathBuf::from), + client_key_path: client_key_path.map(PathBuf::from), + skip_verify, + }) +} + impl ConfigureVaultKmsRequest { /// Convert to KmsConfig pub fn to_kms_config(&self) -> KmsConfig { @@ -606,16 +653,12 @@ impl ConfigureVaultKmsRequest { mount_path: self.mount_path.clone().unwrap_or_else(|| "transit".to_string()), kv_mount: self.kv_mount.clone().unwrap_or_else(|| "secret".to_string()), key_path_prefix: self.key_path_prefix.clone().unwrap_or_else(|| "rustfs/kms/keys".to_string()), - tls: if self.skip_tls_verify.unwrap_or(false) { - Some(TlsConfig { - ca_cert_path: None, - client_cert_path: None, - client_key_path: None, - skip_verify: true, - }) - } else { - None - }, + tls: vault_tls_from_request( + self.skip_tls_verify, + self.ca_cert_path.as_ref(), + self.client_cert_path.as_ref(), + self.client_key_path.as_ref(), + ), })), allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false), // Read from server configuration, never from the request body: the @@ -647,16 +690,12 @@ impl ConfigureVaultTransitKmsRequest { mount_path: self.mount_path.clone().unwrap_or_else(|| "transit".to_string()), metadata_kv_mount: DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT.to_string(), metadata_key_prefix: DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX.to_string(), - tls: if self.skip_tls_verify.unwrap_or(false) { - Some(TlsConfig { - ca_cert_path: None, - client_cert_path: None, - client_key_path: None, - skip_verify: true, - }) - } else { - None - }, + tls: vault_tls_from_request( + self.skip_tls_verify, + self.ca_cert_path.as_ref(), + self.client_cert_path.as_ref(), + self.client_key_path.as_ref(), + ), })), allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false), // Read from server configuration, never from the request body: the @@ -1016,6 +1055,67 @@ mod tests { assert!(request.to_kms_config().validate().is_ok()); } + /// TLS certificate paths named by a configure request must reach the + /// backend configuration, the summary must report them only as booleans, + /// and a request without TLS settings must keep the historical `tls: None` + /// shape in the persisted configuration. + #[test] + fn test_configure_request_tls_paths_reach_the_config_and_summary() { + let mut request = ConfigureVaultTransitKmsRequest { + address: "https://vault.example.com:8200".to_string(), + auth_method: VaultAuthMethod::Token { + token: "vault-token".to_string(), + }, + namespace: None, + mount_path: None, + skip_tls_verify: None, + ca_cert_path: Some("/certs/vault-ca.pem".to_string()), + client_cert_path: Some("/certs/client.pem".to_string()), + client_key_path: Some("/certs/client.key".to_string()), + default_key_id: None, + timeout_seconds: None, + retry_attempts: None, + enable_cache: None, + max_cached_keys: None, + cache_ttl_seconds: None, + allow_insecure_dev_defaults: None, + }; + + let config = request.to_kms_config(); + let tls = config + .vault_transit_config() + .and_then(|transit| transit.tls.as_ref()) + .expect("certificate paths in the request must produce TLS settings"); + assert_eq!(tls.ca_cert_path.as_deref(), Some(std::path::Path::new("/certs/vault-ca.pem"))); + assert_eq!(tls.client_cert_path.as_deref(), Some(std::path::Path::new("/certs/client.pem"))); + assert_eq!(tls.client_key_path.as_deref(), Some(std::path::Path::new("/certs/client.key"))); + assert!(!tls.skip_verify); + + match &KmsConfigSummary::from(&config).backend_summary { + BackendSummary::VaultTransit { + has_custom_ca, + has_client_identity, + .. + } => { + assert!(*has_custom_ca); + assert!(*has_client_identity); + } + other => panic!("expected vault transit summary, got {other:?}"), + } + + request.ca_cert_path = None; + request.client_cert_path = None; + request.client_key_path = None; + assert!( + request + .to_kms_config() + .vault_transit_config() + .and_then(|t| t.tls.as_ref()) + .is_none(), + "a request without TLS settings must keep tls: None" + ); + } + /// The AWS summary carries only non-credential settings, because the /// backend never holds AWS credential material to begin with. #[test] @@ -1180,6 +1280,9 @@ mod tests { namespace: None, mount_path: Some("transit".to_string()), skip_tls_verify: Some(false), + ca_cert_path: None, + client_cert_path: None, + client_key_path: None, default_key_id: None, timeout_seconds: None, retry_attempts: None, @@ -1196,6 +1299,9 @@ mod tests { kv_mount: Some("secret".to_string()), key_path_prefix: Some("rustfs/kms/keys".to_string()), skip_tls_verify: Some(false), + ca_cert_path: None, + client_cert_path: None, + client_key_path: None, default_key_id: None, timeout_seconds: None, retry_attempts: None, diff --git a/crates/kms/src/backends/vault.rs b/crates/kms/src/backends/vault.rs index a430a269b..9e9ce6a8d 100644 --- a/crates/kms/src/backends/vault.rs +++ b/crates/kms/src/backends/vault.rs @@ -546,11 +546,14 @@ impl VaultKmsClient { /// request issued through this client, plus the retry and fail-closed /// budgets for credential refresh. pub async fn new(config: VaultConfig, kms_config: &KmsConfig) -> Result { + let (ca_cert_paths, client_identity) = crate::backends::vault_credentials::vault_tls_materials(config.tls.as_ref())?; let settings = VaultConnectionSettings { address: config.address.clone(), namespace: config.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify), + ca_cert_paths, + client_identity, }; let source = token_source_for(&config.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/backends/vault_credentials.rs b/crates/kms/src/backends/vault_credentials.rs index d5d8a564f..a9b95703d 100644 --- a/crates/kms/src/backends/vault_credentials.rs +++ b/crates/kms/src/backends/vault_credentials.rs @@ -583,6 +583,68 @@ pub(crate) struct VaultConnectionSettings { /// Whether to accept an unverified Vault server certificate. Gated on /// `allow_insecure_dev_defaults` by `KmsConfig::validate`. pub(crate) skip_tls_verify: bool, + /// Additional CA bundle paths trusted for the Vault connection. vaultrs + /// reads and parses the files itself; [`vault_tls_materials`] has already + /// validated them so a bad path fails at configuration time. + pub(crate) ca_cert_paths: Vec, + /// Client certificate + key presented to Vault for mTLS, already loaded + /// from disk. Built once at configuration time so every generation reuses + /// the same identity and a bad file fails fast instead of on refresh. + pub(crate) client_identity: Option, +} + +/// Resolve TLS trust and identity material from the configured [`TlsConfig`]. +/// +/// Reads the files eagerly and fails on the first problem: vaultrs and reqwest +/// would otherwise surface an unreadable bundle only when a client generation +/// is built, or - worse - silently fall back to the `VAULT_CACERT` / +/// `VAULT_CLIENT_CERT` environment variables. +pub(crate) fn vault_tls_materials(tls: Option<&crate::config::TlsConfig>) -> Result<(Vec, Option)> { + let Some(tls) = tls else { + return Ok((Vec::new(), None)); + }; + + let mut ca_cert_paths = Vec::new(); + if let Some(path) = &tls.ca_cert_path { + let content = std::fs::read(path) + .map_err(|e| KmsError::configuration_error(format!("failed to read Vault CA certificate {}: {e}", path.display())))?; + reqwest::Certificate::from_pem_bundle(&content).map_err(|e| { + KmsError::configuration_error(format!("Vault CA certificate {} is not a PEM bundle: {e}", path.display())) + })?; + let path = path.to_str().ok_or_else(|| { + KmsError::configuration_error(format!("Vault CA certificate path {} is not valid UTF-8", path.display())) + })?; + ca_cert_paths.push(path.to_string()); + } + + let client_identity = match (&tls.client_cert_path, &tls.client_key_path) { + (Some(cert_path), Some(key_path)) => { + let mut pem = std::fs::read(cert_path).map_err(|e| { + KmsError::configuration_error(format!("failed to read Vault client certificate {}: {e}", cert_path.display())) + })?; + pem.extend_from_slice(b"\n"); + pem.extend_from_slice(&std::fs::read(key_path).map_err(|e| { + KmsError::configuration_error(format!("failed to read Vault client key {}: {e}", key_path.display())) + })?); + Some(reqwest::Identity::from_pem(&pem).map_err(|e| { + KmsError::configuration_error(format!( + "Vault client certificate {} / key {} do not form a usable identity: {e}", + cert_path.display(), + key_path.display() + )) + })?) + } + (None, None) => None, + // `KmsConfig::validate` rejects unpaired cert/key configuration; this + // guard keeps the invariant for callers that skip validation. + _ => { + return Err(KmsError::configuration_error( + "Vault client certificate and key must be configured together for mTLS", + )); + } + }; + + Ok((ca_cert_paths, client_identity)) } impl VaultConnectionSettings { @@ -601,6 +663,12 @@ impl VaultConnectionSettings { // disable certificate verification behind the KMS configuration and its // insecure-defaults gate. settings_builder.verify(!self.skip_tls_verify); + // Same reasoning for trust roots and the client identity: unset, they + // default to VAULT_CACERT / VAULT_CAPATH and VAULT_CLIENT_CERT / + // VAULT_CLIENT_KEY, splicing TLS material into the connection behind + // the KMS configuration. An empty list / None neutralizes them. + settings_builder.ca_certs(self.ca_cert_paths.clone()); + settings_builder.identity(self.client_identity.clone()); if let Some(namespace) = &self.namespace { settings_builder.namespace(Some(namespace.clone())); @@ -1000,6 +1068,8 @@ mod tests { namespace: Some("team-namespace".to_string()), attempt_timeout: Duration::from_secs(30), skip_tls_verify: false, + ca_cert_paths: Vec::new(), + client_identity: None, } } @@ -1248,6 +1318,8 @@ mod tests { namespace: None, attempt_timeout: Duration::from_secs(30), skip_tls_verify, + ca_cert_paths: Vec::new(), + client_identity: None, }; let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build"); @@ -1272,6 +1344,139 @@ mod tests { }); } + /// Write a self-signed certificate + key pair usable both as a CA bundle + /// and as an mTLS client identity for TLS material tests. + fn write_test_cert_pair(dir: &std::path::Path) -> (std::path::PathBuf, std::path::PathBuf) { + let rcgen::CertifiedKey { cert, signing_key } = + rcgen::generate_simple_self_signed(vec!["vault.example.com".to_string()]).expect("cert should generate"); + let cert_path = dir.join("client_cert.pem"); + let key_path = dir.join("client_key.pem"); + std::fs::write(&cert_path, cert.pem()).expect("cert should write"); + std::fs::write(&key_path, signing_key.serialize_pem()).expect("key should write"); + (cert_path, key_path) + } + + /// Like `verify`, the configured trust roots and client identity have to + /// reach the HTTP client on every generation, or an mTLS Vault rejects the + /// handshake for whichever generation missed them. + #[test] + fn test_tls_materials_reach_every_vault_client_generation() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_test_cert_pair(dir.path()); + + let tls = crate::config::TlsConfig { + ca_cert_path: Some(cert_path.clone()), + client_cert_path: Some(cert_path.clone()), + client_key_path: Some(key_path), + skip_verify: false, + }; + let (ca_cert_paths, client_identity) = + vault_tls_materials(Some(&tls)).expect("valid certificate files must produce TLS materials"); + assert_eq!(ca_cert_paths, vec![cert_path.to_str().expect("utf-8 path").to_string()]); + assert!(client_identity.is_some(), "cert + key must produce a client identity"); + + let settings = VaultConnectionSettings { + address: "https://vault.example.com:8200".to_string(), + namespace: None, + attempt_timeout: Duration::from_secs(30), + skip_tls_verify: false, + ca_cert_paths: ca_cert_paths.clone(), + client_identity, + }; + + let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build"); + assert_eq!(authenticated.settings.ca_certs, ca_cert_paths); + assert!(authenticated.settings.identity.is_some()); + + let login = settings.build_login_client().expect("login client must build"); + assert_eq!(login.settings.ca_certs, ca_cert_paths); + assert!(login.settings.identity.is_some()); + } + + /// vaultrs defaults `ca_certs` from VAULT_CACERT / VAULT_CAPATH and + /// `identity` from VAULT_CLIENT_CERT / VAULT_CLIENT_KEY when the builder + /// leaves them unset, which would splice TLS material into the connection + /// behind the KMS configuration. + #[test] + fn test_vaultrs_tls_env_material_cannot_reach_the_client() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, key_path) = write_test_cert_pair(dir.path()); + let cert = cert_path.to_str().expect("utf-8 path"); + let key = key_path.to_str().expect("utf-8 path"); + let ca_dir = dir.path().to_str().expect("utf-8 path"); + + temp_env::with_vars( + [ + ("VAULT_CACERT", Some(cert)), + ("VAULT_CAPATH", Some(ca_dir)), + ("VAULT_CLIENT_CERT", Some(cert)), + ("VAULT_CLIENT_KEY", Some(key)), + ], + || { + let client = test_settings().build_client(TEST_TOKEN).expect("client must build"); + assert!( + client.settings.ca_certs.is_empty(), + "stray VAULT_CACERT / VAULT_CAPATH must not add trust roots behind the KMS configuration" + ); + assert!( + client.settings.identity.is_none(), + "stray VAULT_CLIENT_CERT / VAULT_CLIENT_KEY must not attach an mTLS identity behind the KMS configuration" + ); + }, + ); + } + + /// Misconfigured TLS material has to fail at configuration time, not on a + /// later credential refresh, and never by silently ignoring the files. + #[test] + fn test_tls_materials_fail_closed_on_bad_configuration() { + let dir = tempfile::tempdir().expect("tempdir"); + let (cert_path, _key_path) = write_test_cert_pair(dir.path()); + + let missing_ca = crate::config::TlsConfig { + ca_cert_path: Some(dir.path().join("absent.pem")), + client_cert_path: None, + client_key_path: None, + skip_verify: false, + }; + assert!( + vault_tls_materials(Some(&missing_ca)).is_err(), + "an unreadable CA bundle must fail configuration" + ); + + let garbage_path = dir.path().join("garbage.pem"); + std::fs::write( + &garbage_path, + b"-----BEGIN CERTIFICATE-----\nnot base64 at all!!\n-----END CERTIFICATE-----\n", + ) + .expect("garbage should write"); + let garbage_ca = crate::config::TlsConfig { + ca_cert_path: Some(garbage_path), + client_cert_path: None, + client_key_path: None, + skip_verify: false, + }; + assert!( + vault_tls_materials(Some(&garbage_ca)).is_err(), + "a CA file that is not a PEM bundle must fail configuration" + ); + + let unpaired = crate::config::TlsConfig { + ca_cert_path: None, + client_cert_path: Some(cert_path), + client_key_path: None, + skip_verify: false, + }; + assert!( + vault_tls_materials(Some(&unpaired)).is_err(), + "a client certificate without its key must fail configuration" + ); + + let (ca_cert_paths, client_identity) = vault_tls_materials(None).expect("absent TLS config is valid"); + assert!(ca_cert_paths.is_empty()); + assert!(client_identity.is_none()); + } + /// The projected token is read fresh per login attempt and trimmed, so a /// kubelet rotation is picked up without a restart and a trailing newline /// does not corrupt the assertion sent to Vault. diff --git a/crates/kms/src/backends/vault_transit.rs b/crates/kms/src/backends/vault_transit.rs index 39bac2be1..e869b2689 100644 --- a/crates/kms/src/backends/vault_transit.rs +++ b/crates/kms/src/backends/vault_transit.rs @@ -411,11 +411,14 @@ impl VaultTransitKmsClient { /// request issued through this client, plus the retry and fail-closed /// budgets for credential refresh. pub async fn new(config: VaultTransitConfig, kms_config: &KmsConfig) -> Result { + let (ca_cert_paths, client_identity) = crate::backends::vault_credentials::vault_tls_materials(config.tls.as_ref())?; let settings = VaultConnectionSettings { address: config.address.clone(), namespace: config.namespace.clone(), attempt_timeout: kms_config.effective_timeout(), skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify), + ca_cert_paths, + client_identity, }; let source = token_source_for(&config.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/backup/vault_restore.rs b/crates/kms/src/backup/vault_restore.rs index b033c4d5f..57c36945e 100644 --- a/crates/kms/src/backup/vault_restore.rs +++ b/crates/kms/src/backup/vault_restore.rs @@ -452,8 +452,12 @@ impl VaultRestoreClient { attempt_timeout: kms_config.effective_timeout(), // A restore target carries no TLS settings, so certificates are // always verified: recovery is the last path that should accept an - // unauthenticated Vault. + // unauthenticated Vault. The empty trust list and absent identity + // also neutralize the VAULT_CACERT / VAULT_CLIENT_CERT environment + // fallbacks, exactly as on the primary connection. skip_tls_verify: false, + ca_cert_paths: Vec::new(), + client_identity: None, }; let source = token_source_for(&target.auth_method, &settings)?; let policy = VaultCredentialPolicy::from_kms_config( diff --git a/crates/kms/src/config.rs b/crates/kms/src/config.rs index 2f177af61..1a4bde079 100644 --- a/crates/kms/src/config.rs +++ b/crates/kms/src/config.rs @@ -30,6 +30,9 @@ pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN"; pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE"; pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH"; pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY"; +pub const ENV_KMS_VAULT_CA_CERT: &str = "RUSTFS_KMS_VAULT_CA_CERT"; +pub const ENV_KMS_VAULT_CLIENT_CERT: &str = "RUSTFS_KMS_VAULT_CLIENT_CERT"; +pub const ENV_KMS_VAULT_CLIENT_KEY: &str = "RUSTFS_KMS_VAULT_CLIENT_KEY"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX"; pub const ENV_KMS_STATIC_SECRET_KEY: &str = "RUSTFS_KMS_STATIC_SECRET_KEY"; @@ -936,22 +939,18 @@ impl KmsConfig { return Err(KmsError::configuration_error("Vault KV2 mount cannot be empty")); } + if let Some(ref tls) = config.tls { + validate_vault_tls_pairing(tls)?; + } + // Validate TLS configuration if using HTTPS if config.address.starts_with("https://") && let Some(ref tls) = config.tls && !tls.skip_verify + && tls.ca_cert_path.is_none() + && tls.client_cert_path.is_none() { - if tls.ca_cert_path.is_some() || tls.client_cert_path.is_some() || tls.client_key_path.is_some() { - // No configuration surface sets these paths today and the - // Vault client does not consume them; warn loudly instead - // of implying the certificates take effect. - tracing::warn!( - "Vault TLS certificate paths are configured but not applied to the Vault client; \ - the connection still relies on the system CA store without a client identity" - ); - } else { - tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA"); - } + tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA"); } } BackendConfig::VaultTransit(config) => { @@ -982,20 +981,17 @@ impl KmsConfig { return Err(KmsError::configuration_error("Vault Transit metadata key prefix cannot be empty")); } + if let Some(ref tls) = config.tls { + validate_vault_tls_pairing(tls)?; + } + if config.address.starts_with("https://") && let Some(ref tls) = config.tls && !tls.skip_verify + && tls.ca_cert_path.is_none() + && tls.client_cert_path.is_none() { - if tls.ca_cert_path.is_some() || tls.client_cert_path.is_some() || tls.client_key_path.is_some() { - // Same as the KV2 branch: these paths are dead - // configuration until the client consumes them. - tracing::warn!( - "Vault TLS certificate paths are configured but not applied to the Vault client; \ - the connection still relies on the system CA store without a client identity" - ); - } else { - tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA"); - } + tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA"); } } BackendConfig::Static(config) => { @@ -1220,15 +1216,36 @@ pub fn kms_config_from_persisted_json(data: &[u8]) -> serde_json::Result Option { - skip_tls_verify.then_some(TlsConfig { - ca_cert_path: None, - client_cert_path: None, - client_key_path: None, - skip_verify: true, +/// Assemble the Vault TLS settings from the environment. +/// +/// Returns `None` when nothing TLS-related is configured so the config +/// serializes without an empty `tls` block, matching the previous behavior. +fn vault_tls_config_from_env() -> Option { + let skip_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false); + let ca_cert_path = get_env_opt_str(ENV_KMS_VAULT_CA_CERT).map(PathBuf::from); + let client_cert_path = get_env_opt_str(ENV_KMS_VAULT_CLIENT_CERT).map(PathBuf::from); + let client_key_path = get_env_opt_str(ENV_KMS_VAULT_CLIENT_KEY).map(PathBuf::from); + + (skip_verify || ca_cert_path.is_some() || client_cert_path.is_some() || client_key_path.is_some()).then_some(TlsConfig { + ca_cert_path, + client_cert_path, + client_key_path, + skip_verify, }) } +/// A client certificate without its key (or the reverse) cannot form an mTLS +/// identity; rejecting it at validation names the missing setting instead of +/// failing when the backend loads the files. +fn validate_vault_tls_pairing(tls: &TlsConfig) -> Result<()> { + if tls.client_cert_path.is_some() != tls.client_key_path.is_some() { + return Err(KmsError::configuration_error( + "Vault client_cert_path and client_key_path must be configured together for mTLS", + )); + } + Ok(()) +} + fn development_default_error(reason: &str) -> KmsError { KmsError::configuration_error(format!("{reason}; set {ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS}=true only for development")) } @@ -1280,7 +1297,7 @@ pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result) -> Result .unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")), metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT), metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX), - tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)), + tls: vault_tls_config_from_env(), }) } @@ -1663,6 +1680,111 @@ mod tests { assert!(skip_tls_config.with_insecure_development_defaults().validate().is_ok()); } + #[test] + fn test_vault_tls_client_cert_and_key_must_be_paired() { + fn kv2_with_tls(tls: TlsConfig) -> KmsConfig { + KmsConfig { + backend: KmsBackend::VaultKv2, + backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig { + address: "https://vault.example.com:8200".to_string(), + auth_method: VaultAuthMethod::Token { + token: "vault-token".to_string(), + }, + namespace: None, + mount_path: "transit".to_string(), + kv_mount: "secret".to_string(), + key_path_prefix: "rustfs/kms/keys".to_string(), + tls: Some(tls), + })), + ..Default::default() + } + } + + let cert_only = kv2_with_tls(TlsConfig { + ca_cert_path: None, + client_cert_path: Some(PathBuf::from("/certs/client.pem")), + client_key_path: None, + skip_verify: false, + }); + let error = cert_only + .with_insecure_development_defaults() + .validate() + .expect_err("a client certificate without its key cannot form an mTLS identity"); + assert!(error.to_string().contains("must be configured together"), "{error}"); + + let key_only = kv2_with_tls(TlsConfig { + ca_cert_path: None, + client_cert_path: None, + client_key_path: Some(PathBuf::from("/certs/client.key")), + skip_verify: false, + }); + assert!(key_only.with_insecure_development_defaults().validate().is_err()); + + let paired = kv2_with_tls(TlsConfig { + ca_cert_path: Some(PathBuf::from("/certs/vault-ca.pem")), + client_cert_path: Some(PathBuf::from("/certs/client.pem")), + client_key_path: Some(PathBuf::from("/certs/client.key")), + skip_verify: false, + }); + assert!(paired.with_insecure_development_defaults().validate().is_ok()); + + // The Transit branch shares the pairing guard. + let transit_cert_only = KmsConfig { + backend: KmsBackend::VaultTransit, + backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig { + address: "https://vault.example.com:8200".to_string(), + auth_method: VaultAuthMethod::Token { + token: "vault-token".to_string(), + }, + namespace: None, + mount_path: "transit".to_string(), + metadata_kv_mount: DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT.to_string(), + metadata_key_prefix: DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX.to_string(), + tls: Some(TlsConfig { + ca_cert_path: None, + client_cert_path: Some(PathBuf::from("/certs/client.pem")), + client_key_path: None, + skip_verify: false, + }), + })), + ..Default::default() + }; + assert!(transit_cert_only.with_insecure_development_defaults().validate().is_err()); + } + + #[test] + fn test_vault_tls_config_from_env_reads_certificate_paths() { + temp_env::with_vars( + [ + (ENV_KMS_VAULT_CA_CERT, Some("/certs/vault-ca.pem")), + (ENV_KMS_VAULT_CLIENT_CERT, Some("/certs/client.pem")), + (ENV_KMS_VAULT_CLIENT_KEY, Some("/certs/client.key")), + ], + || { + let tls = vault_tls_config_from_env().expect("certificate paths in the environment must produce TLS settings"); + assert_eq!(tls.ca_cert_path.as_deref(), Some(Path::new("/certs/vault-ca.pem"))); + assert_eq!(tls.client_cert_path.as_deref(), Some(Path::new("/certs/client.pem"))); + assert_eq!(tls.client_key_path.as_deref(), Some(Path::new("/certs/client.key"))); + assert!(!tls.skip_verify); + }, + ); + + temp_env::with_vars_unset( + [ + ENV_KMS_VAULT_CA_CERT, + ENV_KMS_VAULT_CLIENT_CERT, + ENV_KMS_VAULT_CLIENT_KEY, + ENV_KMS_VAULT_SKIP_TLS_VERIFY, + ], + || { + assert!( + vault_tls_config_from_env().is_none(), + "no TLS-related environment must keep the config free of a TLS block" + ); + }, + ); + } + #[test] fn test_vault_kv2_backend_serialization_uses_pascal_case() { let serialized = serde_json::to_string(&KmsBackend::VaultKv2).expect("backend should serialize"); diff --git a/crates/kms/src/snapshots/rustfs_kms__api_types__tests__kms_vault_transit_config_summary.snap b/crates/kms/src/snapshots/rustfs_kms__api_types__tests__kms_vault_transit_config_summary.snap index 24c7fa954..b9f20bb5b 100644 --- a/crates/kms/src/snapshots/rustfs_kms__api_types__tests__kms_vault_transit_config_summary.snap +++ b/crates/kms/src/snapshots/rustfs_kms__api_types__tests__kms_vault_transit_config_summary.snap @@ -1,12 +1,14 @@ --- source: crates/kms/src/api_types.rs -expression: "serde_json::to_value(&summary).expect(\"KMS config summary should serialize\")" +expression: stable_json_value(&summary) --- { "backend_summary": { "address": "http://127.0.0.1:8200", "auth_method_type": "token", "backend_type": "vault-transit", + "has_client_identity": false, + "has_custom_ca": false, "has_stored_credentials": true, "mount_path": "transit", "namespace": "tenant-a", diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 5135b3fb1..5d1d4fa95 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -261,6 +261,21 @@ The KMS admin API accepts the AWS backend as `"backend_type": "AWS"` (aliases `a `region` is mandatory on this path even though `RUSTFS_KMS_AWS_REGION` is optional at startup: the admin configuration is persisted once and replayed on every node, so a request that left the region to each node's ambient chain would let nodes address different regions, and therefore different keys, while reporting an identical configuration. `default_key_id` must be an AWS key id or ARN that already exists — this backend never creates keys by name. +## Vault TLS: custom CA and mutual TLS + +Both Vault backends (KV2 and Transit) support a private certificate authority and client-certificate (mTLS) authentication at the connection layer: + +| Setting | Environment variable | Admin configure field | Meaning | +| --- | --- | --- | --- | +| CA bundle | `RUSTFS_KMS_VAULT_CA_CERT` | `ca_cert_path` | Path to a PEM CA bundle trusted for the Vault connection, in addition to nothing else: when set, only this bundle is trusted | +| Client certificate | `RUSTFS_KMS_VAULT_CLIENT_CERT` | `client_cert_path` | Path to a PEM client certificate presented to Vault; requires the client key | +| Client key | `RUSTFS_KMS_VAULT_CLIENT_KEY` | `client_key_path` | Path to the PEM private key matching the client certificate | +| Skip verification | `RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY` | `skip_tls_verify` | Disables server certificate verification; gated on the insecure development defaults opt-in | + +Paths are read on the node applying the configuration, so the files must exist at the same path on every node. The certificate and key must be configured together; configuration validation rejects one without the other, and the files are read and parsed when the backend starts, so a bad path or malformed PEM fails the configuration instead of a later request. The `kms/status` backend summary reports `has_custom_ca` and `has_client_identity` booleans (never the file contents). + +The Vault client library would otherwise fall back to the `VAULT_CACERT`, `VAULT_CAPATH`, `VAULT_CLIENT_CERT` and `VAULT_CLIENT_KEY` process environment variables; RustFS always sets the trust roots and identity explicitly - to the configured values or to empty - so stray Vault environment variables cannot splice TLS material into the connection behind the KMS configuration. + ## Local backend durability and deployment support matrix The Local backend stores one JSON record per key (`.key`) plus an Argon2id salt file (`.master-key.salt`) inside the configured `key_dir`. This section documents which deployments that layout supports and how the backend recovers from a crash or power loss. For where the key material lives and who can read it, see the [backend comparison](#backend-comparison) above.