feat(kms): introduce KMS unavailability error and enhance data key handling (#5184)

crates/ecstore/src/object_api/readers.rs

@@ -25,7 +25,7 @@ use chacha20poly1305::ChaCha20Poly1305;
 #[cfg(feature = "rio-v2")]
 use hmac::{Hmac, Mac};
 use md5::{Digest, Md5};
-use rustfs_kms::types::ObjectEncryptionContext;
+use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext};
 use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
 use rustfs_utils::path::path_join_buf;
 #[cfg(feature = "rio-v2")]
@@ -1604,7 +1604,13 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap
     let kms_context: Option<HashMap<String, String>> = None;
     let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref());
 
-    let decrypted_key = if let Some(service) = crate::runtime::sources::object_encryption_service().await {
+    // Persisted wrapping format is the read-side source of truth. The
+    // advertised SSE scheme and current KMS availability are write policy
+    // and runtime state, neither of which identifies the historical provider.
+    let decrypted_key = if is_data_key_envelope(&encrypted_dek) {
+        let service = crate::runtime::sources::object_encryption_service()
+            .await
+            .ok_or_else(|| Error::other(KmsUnavailableError))?;
         #[cfg(feature = "rio-v2")]
         let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) {
             service.decrypt_legacy_data_key(&encrypted_dek).await
@@ -2412,6 +2418,70 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    async fn resolve_managed_material_selects_provider_from_persisted_dek() {
+        use rustfs_kms::KmsConfig;
+        use tempfile::TempDir;
+
+        let key_dir = TempDir::new().expect("create KMS key directory");
+        let manager = rustfs_kms::init_global_kms_service_manager();
+        manager
+            .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
+            .await
+            .expect("start test KMS service");
+
+        async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async {
+            let data_key = [0x24; 32];
+            let base_nonce = [0x14; 12];
+            let encrypted_dek = encrypt_managed_dek_for_test(data_key, [7u8; 32]);
+            let metadata = HashMap::from([
+                (
+                    INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
+                    BASE64_STANDARD.encode(encrypted_dek.as_bytes()),
+                ),
+                (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(base_nonce)),
+                (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "legacy-local-key".to_string()),
+            ]);
+
+            let material = resolve_managed_material("bucket", "object", &metadata)
+                .await
+                .expect("legacy local DEK should not be routed to the running KMS");
+            assert_eq!(material.key_bytes, data_key);
+            assert_eq!(material.base_nonce, base_nonce);
+        })
+        .await;
+
+        manager.stop().await.expect("stop test KMS service");
+
+        let kms_envelope = br#"{
+            "key_id": "test-key-id",
+            "master_key_id": "master-key-id",
+            "key_spec": "AES_256",
+            "encrypted_key": [1, 2, 3, 4],
+            "nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+            "encryption_context": {},
+            "created_at": "2024-01-01T00:00:00+00:00"
+        }"#;
+        let metadata = HashMap::from([
+            (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(kms_envelope)),
+            (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x14; 12])),
+            (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "test-key-id".to_string()),
+        ]);
+        let error = match resolve_managed_material("bucket", "object", &metadata).await {
+            Ok(_) => panic!("KMS envelope must not fall back to the local provider"),
+            Err(error) => error,
+        };
+        let Error::Io(io_error) = error else {
+            panic!("KMS absence should retain its typed source");
+        };
+        assert!(
+            io_error
+                .get_ref()
+                .and_then(|source| source.downcast_ref::<KmsUnavailableError>())
+                .is_some()
+        );
+    }
+
     #[cfg(feature = "rio-v2")]
     #[tokio::test]
     async fn resolve_managed_material_accepts_chacha20_poly1305_header_variant() {

crates/kms/Cargo.toml

@@ -44,6 +44,7 @@ argon2 = { workspace = true }
 chacha20poly1305 = { workspace = true }
 rand = { workspace = true, features = ["serde"] }
 base64 = { workspace = true }
+sha2 = { workspace = true }
 zeroize = { workspace = true, features = ["derive"] }
 
 # Configuration and storage

crates/kms/src/backends/local.rs

@@ -30,6 +30,7 @@ use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
 use jiff::Zoned;
 use rand::RngExt;
 use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
 use std::collections::HashMap;
 use std::path::PathBuf;
 use std::time::Duration;
@@ -51,6 +52,8 @@ pub struct LocalKmsClient {
     key_cache: RwLock<HashMap<String, MasterKeyInfo>>,
     /// Master encryption key for encrypting stored keys
     master_cipher: Option<Aes256Gcm>,
+    /// Legacy pre-beta.9 master cipher for reading pre-Argon2 key files
+    legacy_master_cipher: Option<Aes256Gcm>,
     /// DEK encryption implementation
     dek_crypto: AesDekCrypto,
 }
@@ -97,19 +100,21 @@ impl LocalKmsClient {
         }
 
         // Initialize master cipher if master key is provided
-        let master_cipher = if let Some(ref master_key) = config.master_key {
+        let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key {
             let salt = Self::load_or_create_master_key_salt(&config).await?;
             let key = Self::derive_master_key(master_key, &salt)?;
-            Some(Aes256Gcm::new(&key))
+            let legacy_key = Self::derive_legacy_master_key(master_key)?;
+            (Some(Aes256Gcm::new(&key)), Some(Aes256Gcm::new(&legacy_key)))
         } else {
             warn!("No master key provided - local KMS key material will use explicit plaintext-dev-only storage");
-            None
+            (None, None)
         };
 
         Ok(Self {
             config,
             key_cache: RwLock::new(HashMap::new()),
             master_cipher,
+            legacy_master_cipher,
             dek_crypto: AesDekCrypto::new(),
         })
     }
@@ -132,6 +137,14 @@ impl LocalKmsClient {
         Ok(key)
     }
 
+    fn derive_legacy_master_key(master_key: &str) -> Result<Key<Aes256Gcm>> {
+        let mut hasher = Sha256::new();
+        hasher.update(master_key.as_bytes());
+        hasher.update(b"rustfs-kms-local");
+        Key::<Aes256Gcm>::try_from(hasher.finalize().as_slice())
+            .map_err(|_| KmsError::cryptographic_error("legacy_key", "Invalid key length"))
+    }
+
     fn master_key_salt_path(config: &LocalConfig) -> PathBuf {
         config.key_dir.join(LOCAL_KMS_MASTER_KEY_SALT_FILE)
     }
@@ -206,6 +219,9 @@ impl LocalKmsClient {
         // Decrypt key material if master cipher is available.
         let key_material = match effective_protection {
             StoredKeyProtection::EncryptedMasterKey => {
+                // RUSTFS_COMPAT_TODO(rustfs-5063): Remove after upgrades rewrite all pre-beta.9 key files.
+                // Pre-beta.9 files have no protection marker and use the legacy
+                // SHA-256 KDF, while later pre-marker files use Argon2.
                 let cipher = self.master_cipher.as_ref().ok_or_else(|| {
                     KmsError::configuration_error(format!(
                         "Local KMS key {key_id} is encrypted at rest and requires a configured master key"
@@ -219,9 +235,20 @@ impl LocalKmsClient {
                 nonce_array.copy_from_slice(&stored_key.nonce);
                 let nonce = Nonce::from(nonce_array);
 
-                cipher
-                    .decrypt(&nonce, encrypted_bytes.as_ref())
-                    .map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?
+                match cipher.decrypt(&nonce, encrypted_bytes.as_ref()) {
+                    Ok(key_material) => key_material,
+                    Err(current_error) if stored_key.at_rest_protection == StoredKeyProtection::LegacyUnspecified => {
+                        let legacy_cipher = self.legacy_master_cipher.as_ref().ok_or_else(|| {
+                            KmsError::configuration_error(format!(
+                                "Local KMS key {key_id} is encrypted at rest and requires a configured master key"
+                            ))
+                        })?;
+                        legacy_cipher
+                            .decrypt(&nonce, encrypted_bytes.as_ref())
+                            .map_err(|_| KmsError::cryptographic_error("decrypt", current_error.to_string()))?
+                    }
+                    Err(error) => return Err(KmsError::cryptographic_error("decrypt", error.to_string())),
+                }
             }
             StoredKeyProtection::PlaintextDevOnly | StoredKeyProtection::LegacyUnspecified => {
                 if self.master_cipher.is_some() && stored_key.at_rest_protection == StoredKeyProtection::PlaintextDevOnly {
@@ -1226,4 +1253,71 @@ mod tests {
             .expect("legacy encrypted record should remain readable");
         assert_eq!(key_info.key_id, "legacy-encrypted-key");
     }
+
+    #[tokio::test]
+    async fn test_load_master_key_accepts_beta5_sha256_encrypted_record() {
+        let temp_dir = TempDir::new().expect("create beta.5 fixture directory");
+        let stored_key = serde_json::json!({
+            "key_id": "beta5-key",
+            "version": 1u32,
+            "algorithm": "AES_256",
+            "usage": "EncryptDecrypt",
+            "status": "Active",
+            "description": serde_json::Value::Null,
+            "metadata": HashMap::<String, String>::new(),
+            "created_at": "2024-01-01T00:00:00+00:00",
+            "rotated_at": serde_json::Value::Null,
+            "created_by": "beta5-fixture",
+            "encrypted_key_material": "xjwGa4Lj4qzKg6XQl8s2btyFkPHPChMAkjqs268TFGyvFUv8WjDD5HQCUDLViZmt",
+            "nonce": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+        });
+        fs::write(
+            temp_dir.path().join("beta5-key.key"),
+            serde_json::to_vec_pretty(&stored_key).expect("serialize beta.5 fixture"),
+        )
+        .await
+        .expect("write beta.5 fixture");
+        let mut explicit_protection = stored_key.clone();
+        let explicit_object = explicit_protection.as_object_mut().expect("beta.5 fixture is a JSON object");
+        explicit_object.insert("key_id".to_string(), serde_json::json!("beta5-explicit-key"));
+        explicit_object.insert("at_rest_protection".to_string(), serde_json::json!("encrypted-master-key"));
+        fs::write(
+            temp_dir.path().join("beta5-explicit-key.key"),
+            serde_json::to_vec_pretty(&explicit_protection).expect("serialize explicit-protection fixture"),
+        )
+        .await
+        .expect("write explicit-protection fixture");
+
+        let client = LocalKmsClient::new(LocalConfig {
+            key_dir: temp_dir.path().to_path_buf(),
+            master_key: Some("beta5-test-master-key".to_string()),
+            file_permissions: Some(0o600),
+        })
+        .await
+        .expect("initialize local KMS for beta.5 fixture");
+
+        let material = client
+            .get_key_material("beta5-key")
+            .await
+            .expect("decrypt beta.5 SHA-256 protected key");
+        assert_eq!(material, vec![0x42; 32]);
+        let explicit_error = client
+            .get_key_material("beta5-explicit-key")
+            .await
+            .expect_err("explicit current protection must not fall back to the beta.5 KDF");
+        assert!(matches!(explicit_error, KmsError::CryptographicError { .. }));
+
+        let wrong_key_client = LocalKmsClient::new(LocalConfig {
+            key_dir: temp_dir.path().to_path_buf(),
+            master_key: Some("wrong-beta5-master-key".to_string()),
+            file_permissions: Some(0o600),
+        })
+        .await
+        .expect("initialize local KMS with wrong beta.5 master key");
+        let error = wrong_key_client
+            .get_key_material("beta5-key")
+            .await
+            .expect_err("wrong beta.5 master key must not decrypt the fixture");
+        assert!(matches!(error, KmsError::CryptographicError { .. }));
+    }
 }

crates/kms/src/encryption/dek.rs

@@ -24,6 +24,7 @@ use crate::error::{KmsError, Result};
 use async_trait::async_trait;
 use jiff::Zoned;
 use rand::Rng;
+use serde::de::IgnoredAny;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
@@ -44,6 +45,30 @@ pub struct DataKeyEnvelope {
     pub created_at: Zoned,
 }
 
+#[derive(Deserialize)]
+struct DataKeyEnvelopeMarker {
+    #[serde(rename = "key_id")]
+    _key_id: IgnoredAny,
+    #[serde(rename = "master_key_id")]
+    _master_key_id: IgnoredAny,
+    #[serde(rename = "key_spec")]
+    _key_spec: IgnoredAny,
+    #[serde(rename = "encrypted_key")]
+    _encrypted_key: IgnoredAny,
+    #[serde(rename = "nonce")]
+    _nonce: IgnoredAny,
+    #[serde(rename = "encryption_context")]
+    _encryption_context: IgnoredAny,
+    #[serde(rename = "created_at")]
+    _created_at: IgnoredAny,
+}
+
+/// Returns whether ciphertext is a RustFS KMS data-key envelope.
+pub fn is_data_key_envelope(ciphertext: &[u8]) -> bool {
+    ciphertext.iter().copied().find(|byte| !byte.is_ascii_whitespace()) == Some(b'{')
+        && serde_json::from_slice::<DataKeyEnvelopeMarker>(ciphertext).is_ok()
+}
+
 /// Trait for encrypting and decrypting data encryption keys (DEK)
 ///
 /// This trait abstracts the encryption operations used to protect
@@ -329,4 +354,46 @@ mod tests {
         assert_eq!(deserialized.key_id, "test-key-id");
         assert_eq!(deserialized.master_key_id, "master-key-id");
     }
+
+    #[test]
+    fn test_data_key_envelope_discriminator_rejects_local_formats() {
+        let kms_envelope = br#"{
+            "key_id": "test-key-id",
+            "master_key_id": "master-key-id",
+            "key_spec": "AES_256",
+            "encrypted_key": [1, 2, 3, 4],
+            "nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+            "encryption_context": {},
+            "created_at": "2024-01-01T00:00:00+00:00"
+        }"#;
+        let minio_legacy = br#"{"aead":"AES-256-GCM-HMAC-SHA-256","iv":[1],"nonce":[2],"bytes":[3]}"#;
+        let duplicate_key_id = [b"{\"key_id\":\"duplicate\",".as_slice(), &kms_envelope[1..]].concat();
+
+        assert!(is_data_key_envelope(kms_envelope));
+        assert!(is_data_key_envelope(&[b" \n".as_slice(), kms_envelope].concat()));
+        assert!(!is_data_key_envelope(&duplicate_key_id));
+        assert!(!is_data_key_envelope(b"bm9uY2U=:Y2lwaGVydGV4dA=="));
+        assert!(!is_data_key_envelope(minio_legacy));
+
+        let envelope_value: serde_json::Value = serde_json::from_slice(kms_envelope).expect("parse KMS envelope fixture");
+        for required_field in [
+            "key_id",
+            "master_key_id",
+            "key_spec",
+            "encrypted_key",
+            "nonce",
+            "encryption_context",
+            "created_at",
+        ] {
+            let mut incomplete = envelope_value.clone();
+            incomplete
+                .as_object_mut()
+                .expect("KMS envelope fixture is an object")
+                .remove(required_field);
+            assert!(
+                !is_data_key_envelope(&serde_json::to_vec(&incomplete).expect("serialize incomplete envelope")),
+                "missing {required_field} must not classify as a KMS envelope"
+            );
+        }
+    }
 }

crates/kms/src/encryption/mod.rs

@@ -17,4 +17,4 @@
 pub mod ciphers;
 pub mod dek;
 
-pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
+pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material, is_data_key_envelope};

crates/kms/src/error.rs

@@ -19,6 +19,11 @@ use thiserror::Error;
 /// Result type for KMS operations
 pub type Result<T> = std::result::Result<T, KmsError>;
 
+/// KMS runtime is unavailable for an encryption operation.
+#[derive(Error, Debug, Clone, Copy)]
+#[error("KMS encryption service is unavailable")]
+pub struct KmsUnavailableError;
+
 /// KMS error types covering all possible failure scenarios
 #[derive(Error, Debug, Clone)]
 pub enum KmsError {

crates/kms/src/lib.rs

@@ -83,7 +83,8 @@ pub use api_types::{
     TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
 };
 pub use config::*;
-pub use error::{KmsError, Result};
+pub use encryption::is_data_key_envelope;
+pub use error::{KmsError, KmsUnavailableError, Result};
 pub use manager::KmsManager;
 pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{