diff --git a/Cargo.lock b/Cargo.lock index dd420db90..3cde4a2ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9099,7 +9099,6 @@ dependencies = [ name = "rustfs-ecstore" version = "1.0.0-beta.12" dependencies = [ - "aes-gcm", "arc-swap", "async-channel", "async-recursion", @@ -9115,7 +9114,6 @@ dependencies = [ "byteorder", "bytes", "bytesize", - "chacha20poly1305", "chrono", "criterion", "enumset", @@ -9169,7 +9167,6 @@ dependencies = [ "rustfs-erasure-codec", "rustfs-filemeta", "rustfs-io-metrics", - "rustfs-kms", "rustfs-lifecycle", "rustfs-lock", "rustfs-madmin", diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 31a5446d1..78186b52a 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -57,7 +57,6 @@ rustfs-policy.workspace = true rustfs-protos.workspace = true rustfs-replication.workspace = true rustfs-lifecycle.workspace = true -rustfs-kms.workspace = true rustfs-s3-types = { workspace = true } rustfs-data-usage.workspace = true rustfs-object-capacity.workspace = true @@ -125,8 +124,6 @@ libc.workspace = true rustix = { workspace = true, features = ["process", "fs"] } rustfs-madmin.workspace = true reqwest = { workspace = true } -aes-gcm = { workspace = true, features = ["rand_core"] } -chacha20poly1305.workspace = true aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } urlencoding = { workspace = true } smallvec = { workspace = true, features = ["serde"] } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index d5bed138b..15aebe9d9 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -391,10 +391,12 @@ pub mod notification { pub mod object { pub use crate::object_api::{ - BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, - GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer, - get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, - register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, + GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo, + ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, + ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, + register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook, + unregister_object_mutation_hook, }; pub use crate::store::PreparedGetObjectReader; } diff --git a/crates/ecstore/src/client/utils.rs b/crates/ecstore/src/client/utils.rs index 5d6c975be..3a05e7cf3 100644 --- a/crates/ecstore/src/client/utils.rs +++ b/crates/ecstore/src/client/utils.rs @@ -46,16 +46,6 @@ lazy_static! { m.insert("x-amz-replication-status".to_string(), true); m }; - static ref SSE_HEADERS: HashMap = { - let mut m = HashMap::new(); - m.insert("x-amz-server-side-encryption".to_string(), true); - m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true); - m.insert("x-amz-server-side-encryption-context".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-key".to_string(), true); - m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true); - m - }; } pub fn is_standard_query_value(qs_key: &str) -> bool { @@ -70,16 +60,12 @@ pub fn is_standard_header(header_key: &str) -> bool { *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) } -pub fn is_sse_header(header_key: &str) -> bool { - *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false) -} - pub fn is_amz_header(header_key: &str) -> bool { let key = header_key.to_lowercase(); key.starts_with("x-amz-meta-") || key.starts_with("x-amz-grant-") || key == "x-amz-acl" - || is_sse_header(header_key) + || rustfs_utils::http::is_sse_header(header_key) || key.starts_with("x-amz-checksum-") } diff --git a/crates/ecstore/src/object_api/encryption.rs b/crates/ecstore/src/object_api/encryption.rs new file mode 100644 index 000000000..07413cdd9 --- /dev/null +++ b/crates/ecstore/src/object_api/encryption.rs @@ -0,0 +1,80 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use async_trait::async_trait; +use http::{HeaderMap, HeaderValue}; +use std::collections::HashMap; +use std::error::Error; +use std::fmt::{Display, Formatter}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReadEncryptionMode { + Direct { base_nonce: [u8; 12] }, + Object, +} + +pub struct ReadEncryptionMaterial { + pub key_bytes: [u8; 32], + pub mode: ReadEncryptionMode, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncryptionResolutionErrorKind { + InvalidRequest, + InvalidMetadata, + ServiceUnavailable, + DecryptionFailed, +} + +#[derive(Debug)] +pub struct EncryptionResolutionError { + kind: EncryptionResolutionErrorKind, + message: String, +} + +impl EncryptionResolutionError { + pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } + + pub fn kind(&self) -> EncryptionResolutionErrorKind { + self.kind + } +} + +impl Display for EncryptionResolutionError { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl Error for EncryptionResolutionError {} + +pub struct ReadEncryptionRequest<'a> { + pub bucket: &'a str, + pub object: &'a str, + pub metadata: &'a HashMap, + pub headers: &'a HeaderMap, +} + +#[async_trait] +pub trait ObjectEncryptionResolver: Send + Sync { + async fn resolve_read_material( + &self, + request: ReadEncryptionRequest<'_>, + ) -> Result, EncryptionResolutionError>; +} diff --git a/crates/ecstore/src/object_api/mod.rs b/crates/ecstore/src/object_api/mod.rs index 5c57b69b2..28d567e75 100644 --- a/crates/ecstore/src/object_api/mod.rs +++ b/crates/ecstore/src/object_api/mod.rs @@ -84,6 +84,7 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool { } mod body_cache_hook; +mod encryption; mod hook_slot; mod object_mutation_hook; mod readers; @@ -98,6 +99,10 @@ pub use body_cache_hook::{ pub(crate) use body_cache_hook::{ get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook, }; +pub use encryption::{ + EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial, + ReadEncryptionMode, ReadEncryptionRequest, +}; pub(crate) use object_mutation_hook::notify_object_mutation; pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook}; pub use readers::*; diff --git a/crates/ecstore/src/object_api/readers.rs b/crates/ecstore/src/object_api/readers.rs index 1330cc10b..ff82e2a4b 100644 --- a/crates/ecstore/src/object_api/readers.rs +++ b/crates/ecstore/src/object_api/readers.rs @@ -13,110 +13,13 @@ // limitations under the License. use super::*; -#[cfg(feature = "rio-v2")] -use aes_gcm::aead::Payload; -use aes_gcm::{ - Aes256Gcm, Key, Nonce, - aead::{Aead, KeyInit}, -}; -use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; -#[cfg(feature = "rio-v2")] -use chacha20poly1305::ChaCha20Poly1305; -#[cfg(feature = "rio-v2")] -use hmac::{Hmac, Mac}; -use md5::{Digest, Md5}; -use rustfs_kms::{KmsUnavailableError, is_data_key_envelope, types::ObjectEncryptionContext}; -use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; -use rustfs_utils::path::path_join_buf; -use serde::Deserialize; -#[cfg(feature = "rio-v2")] -use sha2::Sha256; -use std::collections::HashMap; -use std::env; use crate::io_support::rio::Index; -const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id"; -const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key"; -const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context"; -const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv"; -const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size"; -const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size"; -const DEFAULT_SSE_ALGORITHM: &str = "AES256"; -const LOCAL_SSE_DEK_FORMAT_VERSION: u8 = 1; #[cfg(feature = "rio-v2")] const DARE_PAYLOAD_SIZE: i64 = 64 * 1024; #[cfg(feature = "rio-v2")] const DARE_PACKAGE_SIZE: i64 = DARE_PAYLOAD_SIZE + 32; -const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"; -#[cfg(feature = "rio-v2")] -const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256"; -#[cfg(feature = "rio-v2")] -const DARE_VERSION_20: u8 = 0x20; -#[cfg(feature = "rio-v2")] -const DARE_CIPHER_AES_256_GCM: u8 = 0x00; -#[cfg(feature = "rio-v2")] -const DARE_CIPHER_CHACHA20_POLY1305: u8 = 0x01; -#[cfg(feature = "rio-v2")] -const DARE_HEADER_SIZE: usize = 16; -#[cfg(feature = "rio-v2")] -const DARE_TAG_SIZE: usize = 16; -#[cfg(feature = "rio-v2")] -const SEALED_KEY_IV_SIZE: usize = 32; -#[cfg(feature = "rio-v2")] -const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE; -#[cfg(feature = "rio-v2")] -const MINIO_SECRET_KEY_RANDOM_SIZE: usize = 28; -#[cfg(feature = "rio-v2")] -const MINIO_SECRET_KEY_IV_SIZE: usize = 16; -#[cfg(feature = "rio-v2")] -const MINIO_SECRET_KEY_NONCE_SIZE: usize = 12; - -#[cfg(feature = "rio-v2")] -type HmacSha256 = Hmac; - -fn canonical_kms_bucket_path(bucket: &str, object: &str) -> String { - path_join_buf(&[bucket, object]) -} - -fn build_object_encryption_context( - bucket: &str, - object: &str, - provided_context: Option<&HashMap>, -) -> ObjectEncryptionContext { - let mut context = provided_context.cloned().unwrap_or_default(); - context - .entry(bucket.to_string()) - .or_insert_with(|| canonical_kms_bucket_path(bucket, object)); - - let mut object_context = ObjectEncryptionContext::new(bucket.to_string(), object.to_string()); - for (ctx_key, ctx_value) in context { - object_context = object_context.with_encryption_context(ctx_key, ctx_value); - } - object_context -} - -#[cfg(feature = "rio-v2")] -fn is_legacy_rustfs_managed_metadata(metadata: &HashMap) -> bool { - metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).is_some() - && metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER).is_some() - && metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER).is_none() - && metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_none() -} fn part_plaintext_size(part: &ObjectPartInfo) -> i64 { if part.actual_size > 0 { @@ -544,21 +447,6 @@ impl GetObjectReader { } } -#[derive(Debug, Clone, Copy)] -struct EncryptionMaterial { - key_bytes: [u8; 32], - base_nonce: [u8; 12], - key_kind: EncryptionKeyKind, - reader_backend: crate::io_support::rio::ReadEncryptionBackend, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum EncryptionKeyKind { - Direct, - Object, -} - -#[derive(Debug, Clone)] enum ReadTransform { Plain { visible_offset: usize, @@ -572,7 +460,7 @@ enum ReadTransform { total_plaintext_size: usize, }, Encrypted { - material: EncryptionMaterial, + material: ReadEncryptionMaterial, is_multipart: bool, part_numbers: Vec, sequence_number: u32, @@ -584,7 +472,6 @@ enum ReadTransform { }, } -#[derive(Debug, Clone)] struct ReadPlan { storage_offset: usize, storage_length: i64, @@ -593,7 +480,18 @@ struct ReadPlan { } impl ReadPlan { + #[cfg(test)] async fn build(rs: Option, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap) -> Result { + Self::build_with_resolver(rs, oi, opts, h, Some(&tests::TEST_RESOLVER)).await + } + + async fn build_with_resolver( + rs: Option, + oi: &ObjectInfo, + opts: &ObjectOptions, + h: &HeaderMap, + resolver: Option<&dyn ObjectEncryptionResolver>, + ) -> Result { let mut rs = rs; if let Some(part_number) = opts.part_number && rs.is_none() @@ -657,9 +555,20 @@ impl ReadPlan { } if is_encrypted { - let material = resolve_encryption_material(oi, h).await?; + let resolver = resolver.ok_or_else(|| Error::other("object encryption resolver is unavailable"))?; + let resolved = resolver + .resolve_read_material(ReadEncryptionRequest { + bucket: &oi.bucket, + object: &oi.name, + metadata: &oi.user_defined, + headers: h, + }) + .await + .map_err(Error::other)? + .ok_or_else(|| Error::other("encrypted object metadata is incomplete"))?; + let material = resolved; #[cfg(feature = "rio-v2")] - let encryption_backend = material.reader_backend; + let uses_legacy_encryption = matches!(material.mode, ReadEncryptionMode::Direct { .. }); let is_multipart = is_multipart_encrypted_object(&oi.parts, oi.etag.as_deref()); let recorded_plaintext_size = oi.encryption_original_size()?; let plaintext_size = encrypted_plaintext_size(oi, is_multipart, is_compressed, recorded_plaintext_size)?; @@ -678,7 +587,7 @@ impl ReadPlan { let (requested_offset, requested_length) = rs.get_offset_length(plaintext_size)?; #[cfg(feature = "rio-v2")] { - if encryption_backend == crate::io_support::rio::ReadEncryptionBackend::Legacy { + if uses_legacy_encryption { legacy_encrypted_range_plan( oi, is_multipart, @@ -869,32 +778,32 @@ impl ReadPlan { #[cfg(not(feature = "rio-v2"))] let _ = sequence_number; let decrypted_reader: Box = if is_multipart { - match material.key_kind { - EncryptionKeyKind::Object => crate::io_support::rio::decrypt_multipart_reader_with_object_key( + match material.mode { + ReadEncryptionMode::Object => crate::io_support::rio::decrypt_multipart_reader_with_object_key( reader, material.key_bytes, part_numbers, sequence_number, ), - EncryptionKeyKind::Direct => crate::io_support::rio::decrypt_multipart_reader( + ReadEncryptionMode::Direct { base_nonce } => crate::io_support::rio::decrypt_multipart_reader( reader, material.key_bytes, - material.base_nonce, + base_nonce, part_numbers, - material.reader_backend, + crate::io_support::rio::ReadEncryptionBackend::Legacy, sequence_number, ), } } else { - match material.key_kind { - EncryptionKeyKind::Object => { + match material.mode { + ReadEncryptionMode::Object => { crate::io_support::rio::decrypt_reader_with_object_key(reader, material.key_bytes, sequence_number) } - EncryptionKeyKind::Direct => crate::io_support::rio::decrypt_reader( + ReadEncryptionMode::Direct { base_nonce } => crate::io_support::rio::decrypt_reader( reader, material.key_bytes, - material.base_nonce, - material.reader_backend, + base_nonce, + crate::io_support::rio::ReadEncryptionBackend::Legacy, sequence_number, ), } @@ -962,14 +871,28 @@ impl ReadPlan { } impl GetObjectReader { - pub async fn new( + #[cfg(test)] + pub(crate) async fn new( reader: Box, rs: Option, oi: &ObjectInfo, opts: &ObjectOptions, h: &HeaderMap, ) -> Result<(Self, usize, i64)> { - ReadPlan::build(rs, oi, opts, h).await?.into_reader(reader, oi) + Self::new_with_resolver(reader, rs, oi, opts, h, Some(&tests::TEST_RESOLVER)).await + } + + pub async fn new_with_resolver( + reader: Box, + rs: Option, + oi: &ObjectInfo, + opts: &ObjectOptions, + h: &HeaderMap, + resolver: Option<&dyn ObjectEncryptionResolver>, + ) -> Result<(Self, usize, i64)> { + ReadPlan::build_with_resolver(rs, oi, opts, h, resolver) + .await? + .into_reader(reader, oi) } pub async fn read_all(&mut self) -> Result> { let mut data = Vec::new(); @@ -1327,599 +1250,80 @@ fn multipart_part_numbers(parts: &[ObjectPartInfo]) -> Vec { parts.iter().map(|part| part.number).collect() } -fn metadata_get<'a>(metadata: &'a HashMap, key: &str) -> Option<&'a str> { - metadata.get(key).map(String::as_str).or_else(|| { - metadata - .iter() - .find_map(|(candidate, value)| candidate.eq_ignore_ascii_case(key).then_some(value.as_str())) - }) -} - -#[cfg(feature = "rio-v2")] -fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool { - matches!(cipher, DARE_CIPHER_AES_256_GCM | DARE_CIPHER_CHACHA20_POLY1305) -} - -#[cfg(feature = "rio-v2")] -fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result> { - let nonce = &header[4..16]; - let ciphertext = &sealed_key[DARE_HEADER_SIZE..]; - let aad = &header[..4]; - match header[1] { - DARE_CIPHER_AES_256_GCM => { - let cipher = Aes256Gcm::new_from_slice(&sealing_key) - .map_err(|err| Error::other(format!("invalid AES-GCM sealing key: {err}")))?; - let nonce = Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?; - cipher.decrypt(&nonce, Payload { msg: ciphertext, aad }) - } - DARE_CIPHER_CHACHA20_POLY1305 => { - let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key) - .map_err(|err| Error::other(format!("invalid ChaCha20-Poly1305 sealing key: {err}")))?; - let nonce = - chacha20poly1305::Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?; - cipher.decrypt(&nonce, Payload { msg: ciphertext, aad }) - } - _ => return Err(Error::other("unsupported sealed object-key DARE header")), - } - .map_err(|err| Error::other(format!("failed to unseal object key: {err}"))) -} - -async fn resolve_encryption_material(oi: &ObjectInfo, headers: &HeaderMap) -> Result { - if metadata_get(&oi.user_defined, SSEC_ALGORITHM_HEADER).is_some() { - return resolve_ssec_material(oi, headers); - } - - if contains_managed_encryption_metadata(&oi.user_defined) { - return resolve_managed_material(&oi.bucket, &oi.name, &oi.user_defined).await; - } - - Err(Error::other("encrypted object metadata is incomplete")) -} - -fn contains_managed_encryption_metadata(metadata: &HashMap) -> bool { - if metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).is_some() { - return true; - } - - #[cfg(feature = "rio-v2")] - { - metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER).is_some() - || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_some() - || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER).is_some() - } - - #[cfg(not(feature = "rio-v2"))] - { - false - } -} - -#[cfg(feature = "rio-v2")] -fn canonical_sse_path(bucket: &str, object: &str) -> String { - let bucket = bucket.trim_matches('/'); - let object = object.trim_matches('/'); - if object.is_empty() { - bucket.to_string() - } else if bucket.is_empty() { - object.to_string() - } else { - format!("{bucket}/{object}") - } -} - -#[cfg(feature = "rio-v2")] -fn managed_sse_domain(metadata: &HashMap) -> &'static str { - if metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_some() - || metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER).is_some() - || matches!(metadata_get(metadata, "x-amz-server-side-encryption"), Some("aws:kms")) - { - "SSE-KMS" - } else if metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER).is_some() { - "SSE-C" - } else { - "SSE-S3" - } -} - -#[cfg(feature = "rio-v2")] -fn derive_sealing_key( - external_key: [u8; 32], - iv: [u8; SEALED_KEY_IV_SIZE], - domain: &str, - bucket: &str, - object: &str, -) -> [u8; 32] { - let mut mac = HmacSha256::new_from_slice(&external_key).expect("32-byte HMAC key"); - mac.update(&iv); - mac.update(domain.as_bytes()); - mac.update(MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.as_bytes()); - mac.update(canonical_sse_path(bucket, object).as_bytes()); - - let mut sealing_key = [0u8; 32]; - sealing_key.copy_from_slice(mac.finalize().into_bytes().as_slice()); - sealing_key -} - -#[cfg(feature = "rio-v2")] -fn try_decode_minio_sealed_key(bytes: &str) -> Result> { - let decoded = BASE64_STANDARD - .decode(bytes) - .map_err(|e| Error::other(format!("failed to decode sealed object key: {e}")))?; - match decoded.as_slice().try_into() { - Ok(sealed_key) => Ok(Some(sealed_key)), - Err(_) => Ok(None), - } -} - -#[cfg(feature = "rio-v2")] -fn try_decode_minio_sealing_iv(bytes: &str) -> Result> { - let decoded = BASE64_STANDARD - .decode(bytes) - .map_err(|e| Error::other(format!("failed to decode sealing IV: {e}")))?; - match decoded.as_slice().try_into() { - Ok(iv) => Ok(Some(iv)), - Err(_) => Ok(None), - } -} - -#[cfg(feature = "rio-v2")] -fn try_unseal_minio_object_key( - metadata: &HashMap, - bucket: &str, - object: &str, - external_key: [u8; 32], -) -> Result> { - let Some(algorithm) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER) else { - return Ok(None); - }; - if algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM { - return Ok(None); - } - - let Some(iv_b64) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER) else { - return Ok(None); - }; - let Some(iv) = try_decode_minio_sealing_iv(iv_b64)? else { - return Ok(None); - }; - - let sealed_key_b64 = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER)); - let Some(sealed_key_b64) = sealed_key_b64 else { - return Ok(None); - }; - let Some(sealed_key) = try_decode_minio_sealed_key(sealed_key_b64)? else { - return Ok(None); - }; - let header = &sealed_key[..DARE_HEADER_SIZE]; - if header[0] != DARE_VERSION_20 || !is_supported_sealed_object_key_cipher(header[1]) { - return Err(Error::other("unsupported sealed object-key DARE header")); - } - if u16::from_le_bytes([header[2], header[3]]) != 31 || header[4] & 0x80 == 0 { - return Err(Error::other("invalid sealed object-key payload header")); - } - - let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object); - let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed_key)?; - let object_key: [u8; 32] = plaintext - .as_slice() - .try_into() - .map_err(|_| Error::other("sealed object key must decrypt to 32 bytes"))?; - Ok(Some(object_key)) -} - -fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap) -> Result { - let algorithm = headers - .get(SSEC_ALGORITHM_HEADER) - .ok_or_else(|| Error::other("missing SSE-C algorithm header"))? - .to_str() - .map_err(|_| Error::other("invalid SSE-C algorithm header"))?; - if algorithm != DEFAULT_SSE_ALGORITHM { - return Err(Error::other(format!("unsupported SSE-C algorithm {algorithm}"))); - } - - let key_b64 = headers - .get(SSEC_KEY_HEADER) - .ok_or_else(|| Error::other("missing SSE-C key header"))? - .to_str() - .map_err(|_| Error::other("invalid SSE-C key header"))?; - let key_md5 = headers - .get(SSEC_KEY_MD5_HEADER) - .ok_or_else(|| Error::other("missing SSE-C key md5 header"))? - .to_str() - .map_err(|_| Error::other("invalid SSE-C key md5 header"))?; - - let key_bytes_vec = BASE64_STANDARD - .decode(key_b64) - .map_err(|_| Error::other("failed to decode SSE-C key"))?; - let key_bytes: [u8; 32] = key_bytes_vec - .try_into() - .map_err(|_| Error::other("SSE-C key must be 32 bytes"))?; - - let expected_md5 = BASE64_STANDARD.encode(md5_bytes(key_bytes)); - if expected_md5 != key_md5 { - return Err(Error::other("SSE-C key MD5 mismatch")); - } - - let stored_md5 = - metadata_get(&oi.user_defined, SSEC_KEY_MD5_HEADER).ok_or_else(|| Error::other("missing stored SSE-C key md5"))?; - if stored_md5 != expected_md5 { - return Err(Error::other("SSE-C key does not match object metadata")); - } - - #[cfg(feature = "rio-v2")] - if let Some(object_key) = try_unseal_minio_object_key(&oi.user_defined, &oi.bucket, &oi.name, key_bytes)? { - return Ok(EncryptionMaterial { - key_bytes: object_key, - base_nonce: [0u8; 12], - key_kind: EncryptionKeyKind::Object, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2, - }); - } - - Ok(EncryptionMaterial { - key_bytes, - base_nonce: read_stored_ssec_nonce(&oi.user_defined, &oi.bucket, &oi.name), - key_kind: EncryptionKeyKind::Direct, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy, - }) -} - -/// Resolve the SSE-C Direct base nonce for decryption. -/// -/// Since #4576 the encrypt side uses a fresh random nonce per encryption and -/// persists it under `x-rustfs-encryption-iv` (plus the MinIO interop key); -/// this reader-side resolver must read that stored value back or every SSE-C -/// GET fails its first AEAD block. Legacy objects written before random -/// nonces were persisted carry no stored IV and were encrypted with the -/// deterministic `(bucket, key)` nonce, so fall back to recomputing it. Must -/// stay in lockstep with `read_stored_ssec_nonce` in rustfs/src/storage/sse.rs -/// (the API-layer twin of this resolver). -fn read_stored_ssec_nonce(metadata: &HashMap, bucket: &str, key: &str) -> [u8; 12] { - metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER)) - .and_then(|encoded| BASE64_STANDARD.decode(encoded).ok()) - .and_then(|bytes| <[u8; 12]>::try_from(bytes.as_slice()).ok()) - .unwrap_or_else(|| generate_ssec_nonce(bucket, key)) -} - -async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap) -> Result { - let normalized_metadata = normalize_managed_metadata(metadata); - let encrypted_dek = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_HEADER) - .ok_or_else(|| Error::other("missing managed encrypted DEK"))?; - let encrypted_dek = BASE64_STANDARD - .decode(encrypted_dek) - .map_err(|e| Error::other(format!("failed to decode managed encrypted DEK: {e}")))?; - - let kms_key_id = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).unwrap_or("default"); - #[cfg(feature = "rio-v2")] - let kms_context = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_CONTEXT_HEADER) - .map(|value| { - serde_json::from_str::>(value) - .map_err(|e| Error::other(format!("failed to parse managed KMS context: {e}"))) - }) - .transpose()?; - #[cfg(not(feature = "rio-v2"))] - let kms_context: Option> = None; - let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref()); - - // Persisted wrapping format is the read-side source of truth. The - // advertised SSE scheme and current KMS availability are write policy - // and runtime state, neither of which identifies the historical provider. - let decrypted_key = if is_data_key_envelope(&encrypted_dek) { - let service = crate::runtime::sources::object_encryption_service() - .await - .ok_or_else(|| Error::other(KmsUnavailableError))?; - #[cfg(feature = "rio-v2")] - let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) { - service.decrypt_legacy_data_key(&encrypted_dek).await - } else { - service.decrypt_data_key(&encrypted_dek, &object_context).await - }; - #[cfg(not(feature = "rio-v2"))] - let data_key = service.decrypt_data_key(&encrypted_dek, &object_context).await; - - data_key - .map_err(|e| Error::other(format!("failed to decrypt managed data key: {e}")))? - .plaintext_key - } else { - decrypt_local_sse_dek(&encrypted_dek, kms_key_id, &object_context)? - }; - - #[cfg(feature = "rio-v2")] - if let Some(object_key) = try_unseal_minio_object_key(&normalized_metadata, bucket, object, decrypted_key)? { - return Ok(EncryptionMaterial { - key_bytes: object_key, - base_nonce: [0u8; 12], - key_kind: EncryptionKeyKind::Object, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2, - }); - } - - let iv_b64 = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_IV_HEADER) - .ok_or_else(|| Error::other("missing managed encryption IV"))?; - let iv = BASE64_STANDARD - .decode(iv_b64) - .map_err(|e| Error::other(format!("failed to decode managed encryption IV: {e}")))?; - let base_nonce: [u8; 12] = iv - .as_slice() - .try_into() - .map_err(|_| Error::other("managed encryption IV must be 12 bytes"))?; - - Ok(EncryptionMaterial { - key_bytes: decrypted_key, - base_nonce, - key_kind: EncryptionKeyKind::Direct, - reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy, - }) -} - -fn normalize_managed_metadata(metadata: &HashMap) -> HashMap { - #[cfg(feature = "rio-v2")] - { - let mut normalized = metadata.clone(); - if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)) - .or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)) - { - normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.to_string()); - } - - if metadata_get(&normalized, INTERNAL_ENCRYPTION_IV_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER) - { - normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.to_string()); - } - - if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_ID_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER) - { - normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.to_string()); - } - - if metadata_get(&normalized, INTERNAL_ENCRYPTION_CONTEXT_HEADER).is_none() - && let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER) - && let Ok(decoded) = BASE64_STANDARD.decode(value) - && let Ok(context) = serde_json::from_slice::>(&decoded) - && let Ok(encoded) = serde_json::to_string(&context) - { - normalized.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), encoded); - } - - normalized - } - - #[cfg(not(feature = "rio-v2"))] - { - metadata.clone() - } -} - -fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> { - if let Ok(plaintext) = decrypt_rustfs_local_sse_dek(encrypted_dek) { - return Ok(plaintext); - } - - #[cfg(feature = "rio-v2")] - { - decrypt_minio_secret_key_dek(encrypted_dek, object_context) - } - - #[cfg(not(feature = "rio-v2"))] - { - let _ = object_context; - Err(Error::other("invalid managed DEK format")) - } -} - -fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> { - let encrypted_dek = std::str::from_utf8(encrypted_dek).map_err(|_| Error::other("managed DEK is not valid UTF-8"))?; - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct LocalSseDekEnvelope<'a> { - version: u8, - nonce: &'a str, - ciphertext: &'a str, - } - - let (nonce, ciphertext) = match serde_json::from_str::>(encrypted_dek) { - Ok(envelope) => { - if envelope.version != LOCAL_SSE_DEK_FORMAT_VERSION { - return Err(Error::other(format!("unsupported managed DEK format version: {}", envelope.version))); - } - (envelope.nonce, envelope.ciphertext) - } - Err(_) => { - // DEPRECATED: read-only compatibility for persisted colon-delimited DEKs. - // RUSTFS_COMPAT_TODO(sse-local-dek-json-v1): Remove after all supported upgrades have rewritten legacy DEKs. - let Some((nonce, ciphertext)) = encrypted_dek.split_once(':') else { - return Err(Error::other("invalid managed DEK format")); - }; - if ciphertext.contains(':') { - return Err(Error::other("invalid managed DEK format")); - } - (nonce, ciphertext) - } - }; - - let nonce_vec = BASE64_STANDARD - .decode(nonce) - .map_err(|_| Error::other("invalid managed DEK nonce"))?; - let ciphertext = BASE64_STANDARD - .decode(ciphertext) - .map_err(|_| Error::other("invalid managed DEK ciphertext"))?; - - let nonce_array: [u8; 12] = nonce_vec - .as_slice() - .try_into() - .map_err(|_| Error::other("invalid managed DEK nonce length"))?; - - let key = Key::::from(local_sse_master_key()?); - let cipher = Aes256Gcm::new(&key); - let plaintext = cipher - .decrypt(&Nonce::from(nonce_array), ciphertext.as_slice()) - .map_err(|e| Error::other(format!("failed to decrypt managed DEK: {e}")))?; - - plaintext - .as_slice() - .try_into() - .map_err(|_| Error::other("managed DEK has invalid plaintext length")) -} - -#[cfg(feature = "rio-v2")] -#[derive(Deserialize)] -struct MinioLegacyCiphertext { - #[serde(rename = "aead")] - algorithm: String, - iv: Vec, - nonce: Vec, - bytes: Vec, -} - -#[cfg(feature = "rio-v2")] -fn decrypt_minio_secret_key_dek(encrypted_dek: &[u8], object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> { - let key = local_sse_master_key()?; - let (ciphertext, iv, nonce) = parse_minio_secret_key_ciphertext(encrypted_dek)?; - let associated_data = marshal_minio_kms_context(&object_context.encryption_context); - - let mut mac = HmacSha256::new_from_slice(&key).map_err(|err| Error::other(format!("invalid local SSE master key: {err}")))?; - mac.update(&iv); - let sealing_key = mac.finalize().into_bytes(); - let cipher = Aes256Gcm::new_from_slice(sealing_key.as_slice()) - .map_err(|err| Error::other(format!("invalid MinIO sealing key: {err}")))?; - let nonce = Nonce::try_from(&nonce[..]).map_err(|_| Error::other("invalid MinIO managed DEK nonce"))?; - let plaintext = cipher - .decrypt( - &nonce, - aes_gcm::aead::Payload { - msg: &ciphertext, - aad: &associated_data, - }, - ) - .map_err(|err| Error::other(format!("failed to decrypt MinIO managed DEK: {err}")))?; - - plaintext - .as_slice() - .try_into() - .map_err(|_| Error::other("MinIO managed DEK has invalid plaintext length")) -} - -#[cfg(feature = "rio-v2")] -fn parse_minio_secret_key_ciphertext( - encrypted_dek: &[u8], -) -> Result<(Vec, [u8; MINIO_SECRET_KEY_IV_SIZE], [u8; MINIO_SECRET_KEY_NONCE_SIZE])> { - if encrypted_dek.first() == Some(&b'{') && encrypted_dek.last() == Some(&b'}') { - let legacy: MinioLegacyCiphertext = serde_json::from_slice(encrypted_dek) - .map_err(|err| Error::other(format!("failed to parse MinIO legacy managed DEK: {err}")))?; - if legacy.algorithm != "AES-256-GCM-HMAC-SHA-256" { - return Err(Error::other(format!( - "unsupported MinIO legacy managed DEK algorithm {}", - legacy.algorithm - ))); - } - let iv = legacy - .iv - .as_slice() - .try_into() - .map_err(|_| Error::other("invalid MinIO legacy managed DEK IV length"))?; - let nonce = legacy - .nonce - .as_slice() - .try_into() - .map_err(|_| Error::other("invalid MinIO legacy managed DEK nonce length"))?; - return Ok((legacy.bytes, iv, nonce)); - } - - if encrypted_dek.len() <= MINIO_SECRET_KEY_RANDOM_SIZE { - return Err(Error::other("invalid MinIO managed DEK length")); - } - - let split_at = encrypted_dek.len() - MINIO_SECRET_KEY_RANDOM_SIZE; - let (ciphertext, random) = encrypted_dek.split_at(split_at); - let iv = random[..MINIO_SECRET_KEY_IV_SIZE] - .try_into() - .map_err(|_| Error::other("invalid MinIO managed DEK IV length"))?; - let nonce = random[MINIO_SECRET_KEY_IV_SIZE..] - .try_into() - .map_err(|_| Error::other("invalid MinIO managed DEK nonce length"))?; - Ok((ciphertext.to_vec(), iv, nonce)) -} - -#[cfg(feature = "rio-v2")] -fn marshal_minio_kms_context(context: &HashMap) -> Vec { - let mut entries: Vec<_> = context.iter().collect(); - entries.sort_by_key(|(left, _)| *left); - - let mut json = String::from("{"); - for (index, (key, value)) in entries.into_iter().enumerate() { - if index > 0 { - json.push(','); - } - json.push_str(&serde_json::to_string(key).expect("string key serializes")); - json.push(':'); - json.push_str(&serde_json::to_string(value).expect("string value serializes")); - } - json.push('}'); - json.into_bytes() -} - -fn local_sse_master_key() -> Result<[u8; 32]> { - if let Some(key) = decode_master_key_env("__RUSTFS_SSE_SIMPLE_CMK")? { - return Ok(key); - } - - if let Some(key) = decode_master_key_env("RUSTFS_SSE_S3_MASTER_KEY")? { - return Ok(key); - } - - Ok([0u8; 32]) -} - -fn decode_master_key_env(name: &str) -> Result> { - let Ok(value) = env::var(name) else { - return Ok(None); - }; - - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - - let decoded = BASE64_STANDARD - .decode(value) - .map_err(|e| Error::other(format!("{name} is not valid base64: {e}")))?; - let key = - <[u8; 32]>::try_from(decoded.as_slice()).map_err(|_| Error::other(format!("{name} must decode to exactly 32 bytes")))?; - - Ok(Some(key)) -} - -fn generate_ssec_nonce(bucket: &str, key: &str) -> [u8; 12] { - let digest = md5_bytes(format!("{bucket}-{key}").as_bytes()); - let mut nonce = [0u8; 12]; - nonce.copy_from_slice(&digest[..12]); - nonce -} - -fn md5_bytes(data: impl AsRef<[u8]>) -> [u8; 16] { - let digest = Md5::digest(data.as_ref()); - let mut out = [0u8; 16]; - out.copy_from_slice(&digest); - out -} - #[cfg(test)] mod tests { use super::*; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use md5::{Digest, Md5}; + use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER}; + use std::collections::HashMap; use std::io::Cursor; use temp_env::async_with_vars; use tokio::io::AsyncReadExt; + const TEST_DIRECT_KEY_HEADER: &str = "x-rustfs-test-direct-key"; + const TEST_OBJECT_KEY_HEADER: &str = "x-rustfs-test-object-key"; + const TEST_NONCE_HEADER: &str = "x-rustfs-test-nonce"; + + pub(super) static TEST_RESOLVER: TestObjectEncryptionResolver = TestObjectEncryptionResolver; + + pub(super) struct TestObjectEncryptionResolver; + + #[async_trait::async_trait] + impl ObjectEncryptionResolver for TestObjectEncryptionResolver { + async fn resolve_read_material( + &self, + request: ReadEncryptionRequest<'_>, + ) -> std::result::Result, EncryptionResolutionError> { + if let Some(encoded) = request.metadata.get(TEST_OBJECT_KEY_HEADER) { + let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidMetadata, "invalid test object key") + })?; + let key_bytes = decoded.try_into().map_err(|_| { + EncryptionResolutionError::new( + EncryptionResolutionErrorKind::InvalidMetadata, + "invalid test object key length", + ) + })?; + return Ok(Some(ReadEncryptionMaterial { + key_bytes, + mode: ReadEncryptionMode::Object, + })); + } + + let encoded = request + .headers + .get(TEST_DIRECT_KEY_HEADER) + .ok_or_else(|| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "missing test direct key") + })? + .to_str() + .map_err(|_| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key") + })?; + let decoded = BASE64_STANDARD.decode(encoded).map_err(|_| { + EncryptionResolutionError::new(EncryptionResolutionErrorKind::InvalidRequest, "invalid test encryption key") + })?; + let key_bytes = decoded.try_into().map_err(|_| { + EncryptionResolutionError::new( + EncryptionResolutionErrorKind::InvalidRequest, + "invalid test encryption key length", + ) + })?; + let base_nonce = request + .metadata + .get(TEST_NONCE_HEADER) + .and_then(|encoded| BASE64_STANDARD.decode(encoded).ok()) + .and_then(|bytes| bytes.try_into().ok()) + .unwrap_or_else(|| fixture_nonce(request.bucket, request.object)); + Ok(Some(ReadEncryptionMaterial { + key_bytes, + mode: ReadEncryptionMode::Direct { base_nonce }, + })) + } + } + fn md5_bytes(data: impl AsRef<[u8]>) -> [u8; 16] { let digest = Md5::digest(data.as_ref()); let mut bytes = [0u8; 16]; @@ -1927,6 +1331,22 @@ mod tests { bytes } + fn fixture_nonce(bucket: &str, object: &str) -> [u8; 12] { + let digest = md5_bytes(format!("{bucket}-{object}")); + let mut nonce = [0; 12]; + nonce.copy_from_slice(&digest[..12]); + nonce + } + + fn ssec_headers_from_key(key_bytes: [u8; 32]) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + TEST_DIRECT_KEY_HEADER, + HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("test key header is valid"), + ); + headers + } + #[tokio::test] async fn cache_body_uses_plaintext_length_for_compressed_metadata() { let mut metadata = HashMap::new(); @@ -1961,104 +1381,6 @@ mod tests { assert_eq!(restored, body); } - /// Regression for the #4576 fallout: the encrypt side persists a random - /// SSE-C nonce, and this reader-side resolver must read it back — falling - /// back to the deterministic legacy nonce only when no IV was stored. - /// Reverting the stored-nonce lookup breaks the first two cases. - #[test] - fn read_stored_ssec_nonce_prefers_persisted_iv_and_falls_back_for_legacy() { - let stored = [7u8; 12]; - let deterministic = generate_ssec_nonce("bucket", "object"); - assert_ne!(stored, deterministic, "test nonce must differ from the deterministic value"); - - let mut metadata = HashMap::new(); - metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(stored)); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), stored); - - // MinIO interop key only, in non-canonical casing: the lookup is - // case-insensitive like every other internal-metadata read here. - let mut metadata = HashMap::new(); - metadata.insert(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_ascii_lowercase(), BASE64_STANDARD.encode(stored)); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), stored); - - // Legacy object: no stored IV → deterministic fallback. - assert_eq!(read_stored_ssec_nonce(&HashMap::new(), "bucket", "object"), deterministic); - - // Corrupt values (bad base64 / wrong length) also fall back instead of erroring. - let mut metadata = HashMap::new(); - metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "not-base64!!".to_string()); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), deterministic); - let mut metadata = HashMap::new(); - metadata.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([1u8; 8])); - assert_eq!(read_stored_ssec_nonce(&metadata, "bucket", "object"), deterministic); - } - - fn ssec_headers_from_key(key_bytes: [u8; 32]) -> HeaderMap { - let mut headers = HeaderMap::new(); - headers.insert(SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256")); - headers.insert( - SSEC_KEY_HEADER, - HeaderValue::from_str(&BASE64_STANDARD.encode(key_bytes)).expect("valid base64 header"), - ); - headers.insert( - SSEC_KEY_MD5_HEADER, - HeaderValue::from_str(&BASE64_STANDARD.encode(md5_bytes(key_bytes))).expect("valid md5 header"), - ); - headers - } - - #[cfg(feature = "rio-v2")] - #[test] - fn test_legacy_managed_metadata_excludes_sealed_keys() { - let legacy_metadata = HashMap::from([ - (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()), - ]); - assert!(is_legacy_rustfs_managed_metadata(&legacy_metadata)); - - let sealed_metadata = HashMap::from([ - (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()), - (MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), "sealed-key".to_string()), - ]); - - assert!(!is_legacy_rustfs_managed_metadata(&sealed_metadata)); - } - - #[cfg(feature = "rio-v2")] - fn seal_ssec_object_key_for_test( - bucket: &str, - object: &str, - customer_key: [u8; 32], - object_key: [u8; 32], - ) -> ([u8; 32], Vec) { - let iv = [0x23u8; SEALED_KEY_IV_SIZE]; - let sealing_key = derive_sealing_key(customer_key, iv, "SSE-C", bucket, object); - let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key"); - - let mut header = [0u8; DARE_HEADER_SIZE]; - header[0] = DARE_VERSION_20; - header[1] = DARE_CIPHER_AES_256_GCM; - header[2..4].copy_from_slice(&31u16.to_le_bytes()); - header[4] = 0x80; - header[5..16].copy_from_slice(&[0x45u8; 11]); - - let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce"); - let mut sealed = header.to_vec(); - sealed.extend_from_slice( - &cipher - .encrypt( - &nonce, - aes_gcm::aead::Payload { - msg: &object_key, - aad: &header[..4], - }, - ) - .expect("seal object key"), - ); - (iv, sealed) - } - #[tokio::test] async fn test_ranged_decompress_reader() { // Create test data @@ -2342,261 +1664,6 @@ mod tests { assert_eq!(actual, b"fghijkl"); } - fn encrypt_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String { - let key = Key::::from(master_key); - let cipher = Aes256Gcm::new(&key); - let nonce = Nonce::from([0u8; 12]); - let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt managed dek"); - serde_json::json!({ - "version": LOCAL_SSE_DEK_FORMAT_VERSION, - "nonce": BASE64_STANDARD.encode(nonce), - "ciphertext": BASE64_STANDARD.encode(ciphertext), - }) - .to_string() - } - - fn encrypt_legacy_managed_dek_for_test(dek: [u8; 32], master_key: [u8; 32]) -> String { - let key = Key::::from(master_key); - let cipher = Aes256Gcm::new(&key); - let nonce = Nonce::from([0u8; 12]); - let ciphertext = cipher.encrypt(&nonce, dek.as_slice()).expect("encrypt legacy managed dek"); - format!("{}:{}", BASE64_STANDARD.encode(nonce), BASE64_STANDARD.encode(ciphertext)) - } - - #[test] - fn decrypt_rustfs_local_sse_dek_rejects_unknown_json_version() { - let envelope = serde_json::json!({ - "version": LOCAL_SSE_DEK_FORMAT_VERSION + 1, - "nonce": BASE64_STANDARD.encode([0u8; 12]), - "ciphertext": BASE64_STANDARD.encode([0u8; 48]), - }) - .to_string(); - - let error = - decrypt_rustfs_local_sse_dek(envelope.as_bytes()).expect_err("unknown local SSE DEK versions must fail closed"); - assert!(error.to_string().contains("unsupported managed DEK format version")); - } - - #[cfg(feature = "rio-v2")] - fn seal_managed_s3_object_key_for_test( - bucket: &str, - object: &str, - data_key: [u8; 32], - object_key: [u8; 32], - ) -> ([u8; 32], Vec) { - seal_managed_s3_object_key_for_test_with_cipher(bucket, object, data_key, object_key, DARE_CIPHER_AES_256_GCM) - } - - #[cfg(feature = "rio-v2")] - fn seal_managed_s3_object_key_for_test_with_cipher( - bucket: &str, - object: &str, - data_key: [u8; 32], - object_key: [u8; 32], - cipher_id: u8, - ) -> ([u8; 32], Vec) { - let iv = [0x24u8; SEALED_KEY_IV_SIZE]; - let sealing_key = derive_sealing_key(data_key, iv, "SSE-S3", bucket, object); - - let mut header = [0u8; DARE_HEADER_SIZE]; - header[0] = DARE_VERSION_20; - header[1] = cipher_id; - header[2..4].copy_from_slice(&31u16.to_le_bytes()); - header[4] = 0x80; - header[5..16].copy_from_slice(&[0x46u8; 11]); - - let ciphertext = match cipher_id { - DARE_CIPHER_AES_256_GCM => { - let cipher = Aes256Gcm::new_from_slice(&sealing_key).expect("valid sealing key"); - let nonce = Nonce::try_from(&header[4..16]).expect("valid nonce"); - cipher - .encrypt( - &nonce, - Payload { - msg: &object_key, - aad: &header[..4], - }, - ) - .expect("seal managed object key") - } - DARE_CIPHER_CHACHA20_POLY1305 => { - let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key).expect("valid sealing key"); - let nonce = chacha20poly1305::Nonce::try_from(&header[4..16]).expect("valid nonce"); - cipher - .encrypt( - &nonce, - Payload { - msg: &object_key, - aad: &header[..4], - }, - ) - .expect("seal managed object key") - } - _ => panic!("unsupported test cipher"), - }; - let mut sealed = header.to_vec(); - sealed.extend_from_slice(&ciphertext); - (iv, sealed) - } - - #[cfg(feature = "rio-v2")] - #[test] - fn test_supported_sealed_object_key_cipher_accepts_current_minio_fixture_value() { - assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_AES_256_GCM)); - assert!(is_supported_sealed_object_key_cipher(DARE_CIPHER_CHACHA20_POLY1305)); - assert!(!is_supported_sealed_object_key_cipher(0x02)); - } - - #[tokio::test] - async fn resolve_managed_material_accepts_case_insensitive_metadata_keys() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let data_key = [0x24; 32]; - let base_nonce = [0x14; 12]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let metadata = HashMap::from([ - ("X-Rustfs-Encryption-Key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("X-Rustfs-Encryption-IV".to_string(), BASE64_STANDARD.encode(base_nonce)), - ]); - - let material = resolve_managed_material("", "", &metadata) - .await - .expect("managed material should resolve mixed-case metadata keys"); - - assert_eq!(material.key_bytes, data_key); - assert_eq!(material.base_nonce, base_nonce); - }) - .await; - } - - #[tokio::test] - async fn resolve_managed_material_selects_provider_from_persisted_dek() { - use rustfs_kms::KmsConfig; - use tempfile::TempDir; - - let key_dir = TempDir::new().expect("create KMS key directory"); - let manager = rustfs_kms::init_global_kms_service_manager(); - manager - .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("start test KMS service"); - - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([7u8; 32])))], async { - let data_key = [0x24; 32]; - let base_nonce = [0x14; 12]; - let encrypted_dek = encrypt_legacy_managed_dek_for_test(data_key, [7u8; 32]); - let metadata = HashMap::from([ - ( - INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(base_nonce)), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "legacy-local-key".to_string()), - ]); - - let material = resolve_managed_material("bucket", "object", &metadata) - .await - .expect("legacy local DEK should not be routed to the running KMS"); - assert_eq!(material.key_bytes, data_key); - assert_eq!(material.base_nonce, base_nonce); - }) - .await; - - manager.stop().await.expect("stop test KMS service"); - - let kms_envelope = br#"{ - "key_id": "test-key-id", - "master_key_id": "master-key-id", - "key_spec": "AES_256", - "encrypted_key": [1, 2, 3, 4], - "nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], - "encryption_context": {}, - "created_at": "2024-01-01T00:00:00+00:00" - }"#; - let metadata = HashMap::from([ - (INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(kms_envelope)), - (INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x14; 12])), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "test-key-id".to_string()), - ]); - let error = match resolve_managed_material("bucket", "object", &metadata).await { - Ok(_) => panic!("KMS envelope must not fall back to the local provider"), - Err(error) => error, - }; - let Error::Io(io_error) = error else { - panic!("KMS absence should retain its typed source"); - }; - assert!( - io_error - .get_ref() - .and_then(|source| source.downcast_ref::()) - .is_some() - ); - } - - #[cfg(feature = "rio-v2")] - #[tokio::test] - async fn resolve_managed_material_accepts_chacha20_poly1305_header_variant() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let data_key = [0x24; 32]; - let object_key = [0x33; 32]; - let (iv, sealed_key) = seal_managed_s3_object_key_for_test_with_cipher( - "bucket", - "object", - data_key, - object_key, - DARE_CIPHER_CHACHA20_POLY1305, - ); - - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let metadata = HashMap::from([ - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(iv)), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - ( - MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), - ]); - - let material = resolve_managed_material("bucket", "object", &metadata) - .await - .expect("managed material should accept current MinIO header variant"); - assert_eq!(material.key_kind, EncryptionKeyKind::Object); - assert_eq!(material.key_bytes, object_key); - }) - .await; - } - - #[tokio::test] - async fn resolve_encryption_material_accepts_case_insensitive_metadata_keys() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let data_key = [0x24; 32]; - let base_nonce = [0x14; 12]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let metadata = HashMap::from([ - ("X-Rustfs-Encryption-Key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("X-Rustfs-Encryption-IV".to_string(), BASE64_STANDARD.encode(base_nonce)), - ]); - let object_info = ObjectInfo { - user_defined: Arc::new(metadata), - ..Default::default() - }; - let material = resolve_encryption_material(&object_info, &HeaderMap::new()) - .await - .expect("resolve_encryption_material should accept mixed-case managed metadata"); - - assert_eq!(material.key_bytes, data_key); - assert_eq!(material.base_nonce, base_nonce); - }) - .await; - } - #[tokio::test] async fn test_get_object_reader_rejects_ssec_read_without_headers() { let object_info = ObjectInfo { @@ -2791,310 +1858,6 @@ mod tests { )); } - #[tokio::test] - async fn test_get_object_reader_allows_encrypted_full_object_passthrough() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let plaintext = b"managed-full-object".to_vec(); - let data_key = [0x21; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-full-object"; - - let mut encrypted = Vec::new(); - #[cfg(feature = "rio-v2")] - let user_defined = { - let object_key = [0x41; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - ]) - }; - #[cfg(not(feature = "rio-v2"))] - let user_defined = { - let base_nonce = [0x11; 12]; - crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ]) - }; - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(user_defined), - ..Default::default() - }; - - let (mut reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(encrypted.clone())), - None, - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted full-object reads should decrypt inside ecstore"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed plaintext"); - - assert_eq!(offset, 0); - assert_eq!(length, object_info.size); - assert_eq!(reader.object_info.size, plaintext.len() as i64); - assert_eq!(actual, plaintext); - }) - .await; - } - - #[tokio::test] - async fn test_get_object_reader_decrypts_managed_sse_range_on_plaintext_semantics() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let plaintext = b"0123456789abcdefghijklmnopqrstuvwxyz".to_vec(); - let data_key = [0x23; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-range-object"; - - let mut encrypted = Vec::new(); - #[cfg(feature = "rio-v2")] - let user_defined = { - let object_key = [0x43; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed ranged object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - ]) - }; - #[cfg(not(feature = "rio-v2"))] - let user_defined = { - let base_nonce = [0x13; 12]; - crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed ranged object"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ]) - }; - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(user_defined), - ..Default::default() - }; - let range = HTTPRangeSpec { - is_suffix_length: false, - start: 5, - end: 11, - }; - - let (mut reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(encrypted.clone())), - Some(range), - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted range reads should decrypt inside ecstore"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed ranged plaintext"); - - assert_eq!(offset, 0); - assert_eq!(length, encrypted.len() as i64); - assert_eq!(reader.object_info.size, 7); - assert_eq!(actual, b"56789ab"); - }) - .await; - } - - #[tokio::test] - async fn test_get_object_reader_uses_local_managed_fallback_with_explicit_sse_s3_key() { - async_with_vars( - [ - ("__RUSTFS_SSE_SIMPLE_CMK", None::), - ("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode([0u8; 32]))), - ], - async { - let plaintext = b"managed-local-fallback".to_vec(); - let data_key = [0x22; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-local-fallback"; - - let mut encrypted = Vec::new(); - #[cfg(feature = "rio-v2")] - let user_defined = { - let object_key = [0x42; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object with local fallback key"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - ]) - }; - #[cfg(not(feature = "rio-v2"))] - let user_defined = { - let base_nonce = [0x12; 12]; - crate::io_support::rio::EncryptReader::new(Cursor::new(plaintext.clone()), data_key, base_nonce) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object with local fallback key"); - HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())), - ("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(base_nonce)), - ("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()), - ]) - }; - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(user_defined), - ..Default::default() - }; - - let (mut reader, _, _) = GetObjectReader::new( - Box::new(Cursor::new(encrypted)), - None, - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted reads should use the configured local SSE-S3 key"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed plaintext"); - - assert_eq!(reader.object_info.size, plaintext.len() as i64); - assert_eq!(actual, plaintext); - }, - ) - .await; - } - - #[cfg(feature = "rio-v2")] - #[tokio::test] - async fn test_get_object_reader_accepts_minio_only_managed_metadata() { - async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async { - let plaintext = b"managed-minio-metadata".to_vec(); - let data_key = [0x23; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let bucket = "bucket"; - let object = "managed-minio-metadata"; - let object_key = [0x44; 32]; - let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, data_key, object_key); - - let mut encrypted = Vec::new(); - crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) - .read_to_end(&mut encrypted) - .await - .expect("encrypt managed object"); - - let object_info = ObjectInfo { - bucket: bucket.to_string(), - name: object.to_string(), - size: encrypted.len() as i64, - user_defined: Arc::new(HashMap::from([ - ("x-amz-server-side-encryption".to_string(), "AES256".to_string()), - ( - MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - ( - MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()), - ("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()), - ])), - ..Default::default() - }; - - let (mut reader, offset, length) = GetObjectReader::new( - Box::new(Cursor::new(encrypted.clone())), - None, - &object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - .expect("managed encrypted reads should accept MinIO-style metadata"); - - let mut actual = Vec::new(); - reader.read_to_end(&mut actual).await.expect("read managed plaintext"); - - assert_eq!(offset, 0); - assert_eq!(length, object_info.size); - assert_eq!(reader.object_info.size, plaintext.len() as i64); - assert_eq!(actual, plaintext); - }) - .await; - } - #[tokio::test] async fn test_get_object_reader_compressed_range_returns_physical_offset_from_index() { let mut index = Index::new(); @@ -3384,8 +2147,6 @@ mod tests { let object_key = [0x67; 32]; let bucket = "bucket"; let object = "sealed-object"; - let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); - let mut encrypted = Vec::new(); crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) @@ -3397,6 +2158,7 @@ mod tests { name: object.to_string(), size: encrypted.len() as i64, user_defined: Arc::new(HashMap::from([ + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), @@ -3406,15 +2168,6 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), plaintext.len().to_string(), ), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), ])), ..Default::default() }; @@ -3623,10 +2376,7 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), total_plaintext.to_string(), ), - ( - INTERNAL_ENCRYPTION_IV_HEADER.to_string(), - BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE), - ), + (TEST_NONCE_HEADER.to_string(), BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE)), ]) } @@ -4005,65 +2755,6 @@ mod tests { .await; } - #[tokio::test] - async fn test_legacy_managed_multipart_range_seek_byte_exact() { - async_with_vars( - [ - ("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32]))), - (ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true".to_string())), - ], - async { - let data_key = [0x74; 32]; - let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]); - let total_plaintext: usize = 20_000 + 9_000 + 5_000; - let metadata = HashMap::from([ - ( - INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(encrypted_dek.as_bytes()), - ), - (INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "default".to_string()), - ( - INTERNAL_ENCRYPTION_IV_HEADER.to_string(), - BASE64_STANDARD.encode(LEGACY_FIXTURE_BASE_NONCE), - ), - (INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), total_plaintext.to_string()), - ]); - let fixture = - build_legacy_multipart_fixture("bucket", "managed-multipart", data_key, &[20_000, 9_000, 5_000], metadata) - .await; - let headers = HeaderMap::new(); - let opts = ObjectOptions::default(); - - for (rs, expected_offset, expected_length, label) in [ - ( - range(33_900, 33_999), - fixture.physical_part_start(2), - fixture.part_physical_sizes[2] as i64, - "managed tail range", - ), - ( - range(19_990, 20_010), - 0, - (fixture.part_physical_sizes[0] + fixture.part_physical_sizes[1]) as i64, - "managed boundary straddle", - ), - ] { - let (start, len) = rs.get_offset_length(total_plaintext as i64).expect("valid managed range"); - let expected_body = - &fixture.plaintext[start..start + usize::try_from(len).expect("valid managed range length fits usize")]; - - let (body, offset, length, reported_size) = read_via_seek_window(&fixture, Some(rs), &opts, &headers).await; - - assert_eq!(offset, expected_offset, "{label}: physical offset"); - assert_eq!(length, expected_length, "{label}: physical length"); - assert_eq!(reported_size, len, "{label}: reported plaintext size"); - assert_eq!(body, expected_body, "{label}: body bytes"); - } - }, - ) - .await; - } - #[tokio::test] async fn test_legacy_single_part_multipart_object_keeps_full_read_shape() { async_with_vars([(ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async { @@ -4299,50 +2990,6 @@ mod tests { .await; } - #[tokio::test] - async fn test_legacy_ssec_multipart_range_rejects_wrong_or_missing_key() { - async_with_vars([(ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async { - let key_bytes = [0x79; 32]; - let fixture = build_legacy_ssec_multipart_fixture(key_bytes, &[20_000, 9_000, 5_000]).await; - let rs = range(33_900, 33_999); - - let missing = match GetObjectReader::new( - Box::new(Cursor::new(fixture.ciphertext.clone())), - Some(rs.clone()), - &fixture.object_info, - &ObjectOptions::default(), - &HeaderMap::new(), - ) - .await - { - Ok(_) => panic!("missing SSE-C key must fail before any body is produced"), - Err(err) => err, - }; - assert!( - missing.to_string().contains("SSE-C"), - "missing-key failure must come from SSE-C validation: {missing}" - ); - - let wrong = match GetObjectReader::new( - Box::new(Cursor::new(fixture.ciphertext.clone())), - Some(rs), - &fixture.object_info, - &ObjectOptions::default(), - &ssec_headers_from_key([0x00; 32]), - ) - .await - { - Ok(_) => panic!("wrong SSE-C key must fail before any body is produced"), - Err(err) => err, - }; - assert!( - wrong.to_string().contains("SSE-C key does not match object metadata"), - "wrong-key failure must come from the stored key check: {wrong}" - ); - }) - .await; - } - #[tokio::test] async fn test_legacy_ssec_multipart_seek_tamper_fails_hard_with_no_plaintext() { async_with_vars([(ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async { @@ -4394,8 +3041,6 @@ mod tests { let object_key = [0x68; 32]; let bucket = "bucket"; let object = "large-range-object"; - let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); - let mut encrypted = Vec::new(); crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key) .read_to_end(&mut encrypted) @@ -4407,6 +3052,7 @@ mod tests { name: object.to_string(), size: encrypted.len() as i64, user_defined: Arc::new(HashMap::from([ + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), @@ -4416,15 +3062,6 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), plaintext.len().to_string(), ), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), ])), ..Default::default() }; @@ -4536,7 +3173,6 @@ mod tests { let object_key = [0x74; 32]; let bucket = "bucket"; let object = "compressed-large-object"; - let (sealing_iv, sealed_key) = seal_ssec_object_key_for_test(bucket, object, customer_key, object_key); let mut compressor = crate::io_support::rio::CompressReader::with_encrypted_padding( Cursor::new(plaintext.clone()), CompressionAlgorithm::default(), @@ -4629,6 +3265,7 @@ mod tests { ..Default::default() }]), user_defined: Arc::new(HashMap::from([ + (TEST_OBJECT_KEY_HEADER.to_string(), BASE64_STANDARD.encode(object_key)), ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), ( "x-amz-server-side-encryption-customer-key-md5".to_string(), @@ -4638,15 +3275,6 @@ mod tests { "x-amz-server-side-encryption-customer-original-size".to_string(), plaintext.len().to_string(), ), - ( - MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(), - MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(), - ), - (MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)), - ( - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(), - BASE64_STANDARD.encode(sealed_key), - ), ( "x-minio-internal-compression".to_string(), crate::io_support::rio::compression_metadata_value(CompressionAlgorithm::default()), @@ -4680,7 +3308,7 @@ mod tests { assert_eq!(plaintext_offset as i64, range.start - uncomp_off); assert_eq!(plaintext_length, 64); } - other => panic!("expected encrypted read plan, got {other:?}"), + _ => panic!("expected encrypted read plan"), } let (mut reader, offset, length) = GetObjectReader::new( diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index dfd403e18..0248d76b2 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -273,29 +273,9 @@ impl ObjectInfo { } pub fn is_encrypted(&self) -> bool { - // Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function - use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER}; - - self.user_defined.keys().any(|key| { - let lower = key.to_ascii_lowercase(); - lower.starts_with("x-minio-encryption-") - || lower.starts_with("x-minio-internal-server-side-encryption-") - || matches!( - lower.as_str(), - "x-minio-internal-encrypted-multipart" - | "x-rustfs-encryption-key" - | "x-rustfs-encryption-algorithm" - | "x-rustfs-encryption-iv" - | "x-rustfs-encryption-key-id" - | "x-rustfs-encryption-context" - | "x-rustfs-encryption-tag" - | "x-amz-server-side-encryption-aws-kms-key-id" - | SSEC_ALGORITHM_HEADER - | SSEC_KEY_HEADER - | SSEC_KEY_MD5_HEADER - | "x-amz-server-side-encryption" - ) - }) + self.user_defined + .keys() + .any(|key| rustfs_utils::http::is_object_encryption_marker(key)) } /// Maximum inline size for non-versioned objects (128 KiB). @@ -339,26 +319,7 @@ impl ObjectInfo { } pub fn encryption_original_size(&self) -> std::io::Result> { - let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE); - if let Some(size_str) = self - .user_defined - .get("x-rustfs-encryption-original-size") - .map(String::as_str) - .or_else(|| { - self.user_defined - .get("x-amz-server-side-encryption-customer-original-size") - .map(String::as_str) - }) - .or(actual_size.as_deref()) - && !size_str.is_empty() - { - let size = size_str - .parse::() - .map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?; - return Ok(Some(size)); - } - - Ok(None) + rustfs_utils::http::get_object_encryption_original_size(&self.user_defined) } pub fn decrypted_size(&self) -> std::io::Result { @@ -388,9 +349,6 @@ impl ObjectInfo { return Ok(actual_size); } - // Check if object is encrypted - // Managed SSE stores original size in x-rustfs-encryption-original-size metadata - // SSE-C stores original size in x-amz-server-side-encryption-customer-original-size if let Some(size) = self.encryption_original_size()? { return Ok(size); } @@ -881,6 +839,19 @@ mod tests { assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back"); } + #[test] + fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() { + let object = ObjectInfo { + user_defined: Arc::new(HashMap::from([( + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(), + "sealed".to_string(), + )])), + ..Default::default() + }; + + assert!(object.is_encrypted()); + } + #[test] fn versions_after_marker_handles_null_version_marker() { let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap(); diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 2c8ab72c6..ed0bda1dd 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -46,6 +46,7 @@ use crate::bucket::metadata_sys::BucketMetadataSys; use crate::bucket::replication::{DynReplicationPool, ReplicationStats}; use crate::disk::DiskStore; use crate::layout::endpoints::{EndpointServerPools, SetupType}; +use crate::object_api::ObjectEncryptionResolver; use crate::services::event_notification::EventNotifier; use crate::services::tier::tier::TierConfigMgr; use rustfs_lock::{GlobalLockManager, get_global_lock_manager}; @@ -159,6 +160,8 @@ pub struct InstanceContext { /// workers (scanner/heal/tier/lifecycle) without touching another instance. /// Replaces the process-global cancel-token static. background_cancel_token: OnceLock, + /// Resolves object-encryption material at the application boundary. + object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, transition_transaction_recovery_stores: std::sync::Mutex>, #[cfg(test)] @@ -197,6 +200,7 @@ impl InstanceContext { local_disk_set_drives: Arc::new(RwLock::new(Vec::new())), bucket_metadata_sys: std::sync::Mutex::new(None), background_cancel_token: OnceLock::new(), + object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), #[cfg(test)] @@ -209,6 +213,19 @@ impl InstanceContext { self.lock_manager.clone() } + /// Install the application-owned object-encryption resolver once. + pub fn set_object_encryption_resolver( + &self, + resolver: Arc, + ) -> Result<(), Arc> { + self.object_encryption_resolver.set(resolver) + } + + /// Return the configured object-encryption resolver, if startup installed one. + pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> { + self.object_encryption_resolver.get().map(Arc::as_ref) + } + /// Set this instance's S3 region. /// /// Write-once: panics on a second write, preserving the startup fail-fast diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index 8a16f0c26..b85d6c395 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -46,7 +46,6 @@ use crate::{ use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; -use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service}; use rustfs_lock::client::LockClient; use s3s::dto::BucketLifecycleConfiguration; use s3s::region::Region; @@ -105,10 +104,6 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_ global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error); } -pub(crate) async fn object_encryption_service() -> Option> { - get_global_encryption_service().await -} - pub fn object_store_handle() -> Option> { resolve_object_store_handle() } diff --git a/crates/ecstore/src/set_disk/metadata.rs b/crates/ecstore/src/set_disk/metadata.rs index e1cfea0e0..b19d502a0 100644 --- a/crates/ecstore/src/set_disk/metadata.rs +++ b/crates/ecstore/src/set_disk/metadata.rs @@ -505,9 +505,7 @@ impl SetDisks { } fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool { - meta.metadata - .keys() - .any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name)) + meta.metadata.keys().any(|name| http::is_object_encryption_marker(name)) } fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index d356108d3..6ff7488a2 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -147,15 +147,17 @@ use rustfs_object_capacity::capacity_scope::{ CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope, }; use rustfs_s3_types::EventName; +#[cfg(test)] +use rustfs_utils::http::SSEC_ALGORITHM_HEADER; use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; use rustfs_utils::http::headers::{ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, }; use rustfs_utils::http::{ - SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, - SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str, - get_header_map, get_str, insert_str, is_encryption_metadata_key, remove_header_map, + SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, + SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker, + remove_header_map, }; use rustfs_utils::{ HashAlgorithm, @@ -670,10 +672,7 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap) -> bool { - metadata.keys().any(|key| is_encryption_metadata_key(key)) - || metadata.contains_key(SSEC_ALGORITHM_HEADER) - || metadata.contains_key(SSEC_KEY_HEADER) - || metadata.contains_key(SSEC_KEY_MD5_HEADER) + metadata.keys().any(|key| is_object_encryption_marker(key)) } /// Per-set memoized capacity dirty scope. diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 90f84db95..30cd9f94a 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -42,6 +42,7 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; use crate::store::ECStore; use futures::FutureExt as _; +use http::HeaderValue; use std::future::Future; fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result { @@ -49,6 +50,17 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result, + range: Option, + object_info: &ObjectInfo, + opts: &ObjectOptions, + headers: &HeaderMap, +) -> Result<(GetObjectReader, usize, i64)> { + GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await +} + /// Length of the full plaintext body when — and only when — this read's output /// is exactly the object's complete plaintext, so the app-layer body cache may /// serve it in place of the erasure read. @@ -713,7 +725,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { size_bucket, ); record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket); - let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?; + let (mut reader, _offset, _length) = + get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?; // Carry the hook probe result so the app layer skips its // now-redundant lookup on the streaming miss path (ODC-16). reader.body_source = body_source; @@ -745,7 +758,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { let (rd, wd) = tokio::io::duplex(duplex_buffer_size); debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer"); - let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?; + let (mut reader, offset, length) = + get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?; // Carry the hook probe result so the app layer skips its now-redundant // lookup on the streaming miss path (ODC-16). reader.body_source = body_source; @@ -4536,6 +4550,61 @@ mod erasure_construction_tests { } } +#[cfg(test)] +mod object_encryption_resolver_wiring_tests { + use super::*; + use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest}; + use std::io::Cursor; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CountingResolver { + calls: AtomicUsize, + } + + #[async_trait::async_trait] + impl ObjectEncryptionResolver for CountingResolver { + async fn resolve_read_material( + &self, + _request: ReadEncryptionRequest<'_>, + ) -> std::result::Result, EncryptionResolutionError> { + self.calls.fetch_add(1, Ordering::Relaxed); + Ok(None) + } + } + + #[tokio::test] + async fn get_object_reader_forwards_instance_resolver() { + let resolver = Arc::new(CountingResolver { + calls: AtomicUsize::new(0), + }); + let ctx = InstanceContext::new(); + assert!( + ctx.set_object_encryption_resolver(resolver.clone()).is_ok(), + "fresh context should accept resolver" + ); + let object_info = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 1, + user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])), + ..Default::default() + }; + + let result = get_object_reader_with_context( + &ctx, + Box::new(Cursor::new(Vec::::new())), + None, + &object_info, + &ObjectOptions::default(), + &HeaderMap::new(), + ) + .await; + + assert!(result.is_err(), "resolver returning no material must fail closed"); + assert_eq!(resolver.calls.load(Ordering::Relaxed), 1); + } +} + #[cfg(test)] pub(in crate::set_disk::ops) mod hermetic_set_disks_support { //! Shared hermetic `SetDisks` construction for the ops tests below: the diff --git a/crates/ecstore/tests/README.md b/crates/ecstore/tests/README.md index 0833a467d..1514ba7f3 100644 --- a/crates/ecstore/tests/README.md +++ b/crates/ecstore/tests/README.md @@ -2,7 +2,7 @@ ## MinIO-generated encrypted fixtures -`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by +`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by `.\rustfs\scripts\minio_fixture_lab\lab.py`. It currently covers multipart fixtures for: @@ -20,5 +20,5 @@ Example: ```powershell $env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key' $env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '' -cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored +cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored ``` diff --git a/crates/kms/AGENTS.md b/crates/kms/AGENTS.md index 3f6cb014a..5555c8071 100644 --- a/crates/kms/AGENTS.md +++ b/crates/kms/AGENTS.md @@ -25,3 +25,25 @@ For local KMS end-to-end tests, keep proxy bypass settings: NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1 ``` + +## Local Key Export for SSE-S3 Migration Tests + +Use the read-only `local_kms_key_decrypt` example to export an AES-256 Local +KMS key as the base64 value expected by `RUSTFS_SSE_S3_MASTER_KEY`: + +```bash +export RUSTFS_KMS_LOCAL_MASTER_KEY='' +export RUSTFS_SSE_S3_MASTER_KEY="$( + cargo run -q -p rustfs-kms --example local_kms_key_decrypt -- \ + /absolute/path/to/.key +)" +``` + +For a `plaintext-dev-only` Local KMS key file, +`RUSTFS_KMS_LOCAL_MASTER_KEY` is not required. + +The example writes only the base64-encoded 32-byte key to stdout. Diagnostics +go to stderr. Never paste its output into logs, shell history, issue comments, +or committed configuration. The export path must remain read-only and must +reuse `LocalKmsClient` decoding so current Argon2id and legacy key-file +compatibility stay aligned with the backend. diff --git a/crates/kms/examples/local_kms_key_decrypt.rs b/crates/kms/examples/local_kms_key_decrypt.rs new file mode 100644 index 000000000..37f174ccc --- /dev/null +++ b/crates/kms/examples/local_kms_key_decrypt.rs @@ -0,0 +1,112 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD}; +use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use zeroize::Zeroizing; + +const LOCAL_KMS_MASTER_KEY_ENV: &str = "RUSTFS_KMS_LOCAL_MASTER_KEY"; + +fn usage(program: &str) -> String { + format!( + "Usage: {program} \n\ + Reads {LOCAL_KMS_MASTER_KEY_ENV} when the key file is encrypted.\n\ + Writes only the base64-encoded 32-byte key to stdout." + ) +} + +fn resolve_key_file(path: &Path) -> Result<(PathBuf, String), String> { + let canonical = std::fs::canonicalize(path).map_err(|error| format!("cannot open Local KMS key file: {error}"))?; + if canonical.extension().and_then(|extension| extension.to_str()) != Some("key") { + return Err("Local KMS key file must have a .key extension".to_string()); + } + let key_dir = canonical + .parent() + .ok_or_else(|| "Local KMS key file must have a parent directory".to_string())? + .to_path_buf(); + let key_id = canonical + .file_stem() + .and_then(|stem| stem.to_str()) + .filter(|stem| !stem.is_empty()) + .ok_or_else(|| "Local KMS key file name must contain a valid UTF-8 key ID".to_string())? + .to_string(); + Ok((key_dir, key_id)) +} + +async fn run() -> Result<(), String> { + let mut args = std::env::args(); + let program = args.next().unwrap_or_else(|| "local_kms_key_decrypt".to_string()); + let Some(key_file) = args.next() else { + return Err(usage(&program)); + }; + if args.next().is_some() { + return Err(usage(&program)); + } + + let (key_dir, key_id) = resolve_key_file(Path::new(&key_file))?; + let master_key = std::env::var(LOCAL_KMS_MASTER_KEY_ENV).ok().filter(|value| !value.is_empty()); + let client = LocalKmsClient::new_for_key_export(LocalConfig { + key_dir, + master_key, + file_permissions: Some(0o600), + }) + .await + .map_err(|error| error.to_string())?; + let key_material = client + .decrypt_key_material_for_export(&key_id) + .await + .map_err(|error| error.to_string())?; + let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref())); + + let mut stdout = io::stdout().lock(); + writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}")) +} + +#[tokio::main] +async fn main() { + if let Err(error) = run().await { + let _ = writeln!(io::stderr().lock(), "local_kms_key_decrypt: {error}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolve_key_file_extracts_directory_and_key_id() { + let directory = tempfile::tempdir().expect("create temporary directory"); + let key_file = directory.path().join("migration-key.key"); + std::fs::write(&key_file, b"{}").expect("create key file"); + + let (key_dir, key_id) = resolve_key_file(&key_file).expect("resolve key file"); + + assert_eq!(key_dir, directory.path().canonicalize().expect("canonical directory")); + assert_eq!(key_id, "migration-key"); + } + + #[test] + fn resolve_key_file_rejects_non_key_extension() { + let directory = tempfile::tempdir().expect("create temporary directory"); + let key_file = directory.path().join("migration-key.json"); + std::fs::write(&key_file, b"{}").expect("create key file"); + + let error = resolve_key_file(&key_file).expect_err("non-key file must be rejected"); + + assert!(error.contains(".key")); + } +} diff --git a/crates/kms/src/backends/local.rs b/crates/kms/src/backends/local.rs index ada43f00b..60081963f 100644 --- a/crates/kms/src/backends/local.rs +++ b/crates/kms/src/backends/local.rs @@ -36,6 +36,7 @@ use std::path::{Component, Path, PathBuf}; use std::time::Duration; use tokio::fs; use tracing::{debug, warn}; +use zeroize::Zeroizing; /// Reject key identifiers that would not name a single file directly inside the key /// directory. @@ -146,6 +147,45 @@ impl LocalKmsClient { Ok(client) } + /// Open a Local KMS key directory without creating or modifying any files. + /// + /// This constructor is restricted to explicit key-export tooling. Normal + /// backend operation must use [`Self::new`]. + pub async fn new_for_key_export(config: LocalConfig) -> Result { + if !fs::try_exists(&config.key_dir).await? { + return Err(KmsError::configuration_error("Local KMS key directory does not exist")); + } + + let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key { + let legacy_key = Self::derive_legacy_master_key(master_key)?; + let legacy_master_cipher = Aes256Gcm::new(&legacy_key); + let salt_path = Self::master_key_salt_path(&config); + let master_cipher = if fs::try_exists(&salt_path).await? { + let salt = fs::read(&salt_path).await?; + let salt: [u8; LOCAL_KMS_MASTER_KEY_SALT_LEN] = salt.try_into().map_err(|_| { + KmsError::configuration_error(format!( + "Local KMS master key salt at {} must be exactly {} bytes", + salt_path.display(), + LOCAL_KMS_MASTER_KEY_SALT_LEN + )) + })?; + Aes256Gcm::new(&Self::derive_master_key(master_key, &salt)?) + } else { + Aes256Gcm::new(&legacy_key) + }; + (Some(master_cipher), Some(legacy_master_cipher)) + } else { + (None, None) + }; + + Ok(Self { + config, + master_cipher, + legacy_master_cipher, + dek_crypto: AesDekCrypto::new(), + }) + } + /// Derive a 256-bit key from the master key string using a persistent Argon2id salt. fn derive_master_key(master_key: &str, salt: &[u8]) -> Result> { let params = Params::new( @@ -435,6 +475,20 @@ impl LocalKmsClient { Ok(key_material) } + /// Decrypt an AES-256 Local KMS key for explicit migration tooling. + /// + /// The returned buffer is zeroized on drop. Callers must treat the value as + /// plaintext key material and avoid logging or persisting it. + pub async fn decrypt_key_material_for_export(&self, key_id: &str) -> Result> { + let (stored_key, key_material) = self.decode_stored_key(key_id).await?; + if stored_key.algorithm != "AES_256" { + return Err(KmsError::unsupported_algorithm(stored_key.algorithm)); + } + let actual = key_material.len(); + let key_material = key_material.try_into().map_err(|_| KmsError::invalid_key_size(32, actual))?; + Ok(Zeroizing::new(key_material)) + } + async fn validate_existing_keys(&self) -> Result<()> { let mut entries = fs::read_dir(&self.config.key_dir).await?; while let Some(entry) = entries.next_entry().await? { @@ -1208,6 +1262,52 @@ mod tests { assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. })); } + #[tokio::test] + async fn key_export_uses_existing_local_decryption_path_without_writing_files() { + let (client, _temp_dir) = create_test_client().await; + let key_id = "export-key"; + client + .create_key(key_id, "AES_256", None) + .await + .expect("create encrypted key"); + let expected = client.get_key_material(key_id).await.expect("load expected key material"); + let salt_path = LocalKmsClient::master_key_salt_path(&client.config); + let salt_before = fs::read(&salt_path).await.expect("read existing salt"); + + let export_client = LocalKmsClient::new_for_key_export(client.config.clone()) + .await + .expect("open read-only export client"); + let exported = export_client + .decrypt_key_material_for_export(key_id) + .await + .expect("decrypt key for export"); + + assert_eq!(exported.as_ref(), expected.as_slice()); + assert_eq!(fs::read(&salt_path).await.expect("read unchanged salt"), salt_before); + } + + #[tokio::test] + async fn key_export_accepts_plaintext_dev_only_key_without_master_key() { + let (client, _temp_dir) = create_dev_mode_client().await; + let key_id = "plaintext-export-key"; + client + .create_key(key_id, "AES_256", None) + .await + .expect("create plaintext-dev-only key"); + let expected = client.get_key_material(key_id).await.expect("load expected key material"); + + let export_client = LocalKmsClient::new_for_key_export(client.config.clone()) + .await + .expect("open read-only export client"); + let exported = export_client + .decrypt_key_material_for_export(key_id) + .await + .expect("export plaintext-dev-only key"); + + assert_eq!(exported.as_ref(), expected.as_slice()); + assert!(!LocalKmsClient::master_key_salt_path(&client.config).exists()); + } + #[tokio::test] async fn test_plaintext_dev_only_storage_is_explicit_and_loadable() { let (client, _temp_dir) = create_dev_mode_client().await; diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index 7806bbc7d..db4791ec3 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -89,7 +89,7 @@ pub use error::{KmsError, KmsUnavailableError, Result}; pub use manager::KmsManager; pub use service::{DataKey, ObjectEncryptionService}; pub use service_manager::{ - KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager, + KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager, init_global_kms_service_manager, }; pub use types::*; diff --git a/crates/kms/src/service_manager.rs b/crates/kms/src/service_manager.rs index 07b8628cf..be463c42d 100644 --- a/crates/kms/src/service_manager.rs +++ b/crates/kms/src/service_manager.rs @@ -21,12 +21,13 @@ use crate::manager::KmsManager; use crate::service::ObjectEncryptionService; use arc_swap::ArcSwap; use sha2::{Digest, Sha256}; +use std::future::Future; use std::sync::{ Arc, OnceLock, atomic::{AtomicU64, Ordering}, }; use subtle::ConstantTimeEq; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::Mutex; use tracing::{debug, error, info, warn}; const LOG_COMPONENT_KMS: &str = "kms"; @@ -93,6 +94,13 @@ pub enum KmsServiceStatus { Error(String), } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KmsStartOutcome { + Started, + Restarted, + AlreadyRunning, +} + /// Service version information for zero-downtime reconfiguration #[derive(Clone)] struct ServiceVersion { @@ -104,16 +112,17 @@ struct ServiceVersion { manager: Arc, } +#[derive(Clone)] +struct RuntimeState { + config: Option, + status: KmsServiceStatus, + current_service: Option, +} + /// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration pub struct KmsServiceManager { - /// Current service version (if running) - /// Uses ArcSwap for atomic, lock-free service switching - /// This allows instant atomic updates without blocking readers - current_service: ArcSwap>, - /// Current configuration - config: Arc>>, - /// Current status - status: Arc>, + /// Atomically published configuration, status, and current service. + state: ArcSwap, /// Version counter (monotonically increasing) version_counter: Arc, /// Mutex to protect lifecycle operations (start, stop, reconfigure) @@ -125,9 +134,11 @@ impl KmsServiceManager { /// Create a new KMS service manager (not configured) pub fn new() -> Self { Self { - current_service: ArcSwap::from_pointee(None), - config: Arc::new(RwLock::new(None)), - status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)), + state: ArcSwap::from_pointee(RuntimeState { + config: None, + status: KmsServiceStatus::NotConfigured, + current_service: None, + }), version_counter: Arc::new(AtomicU64::new(0)), lifecycle_mutex: Arc::new(Mutex::new(())), } @@ -135,44 +146,67 @@ impl KmsServiceManager { /// Get current service status pub async fn get_status(&self) -> KmsServiceStatus { - self.status.read().await.clone() + self.state.load().status.clone() } /// Get current configuration (if any) pub async fn get_config(&self) -> Option { - self.config.read().await.clone() + self.state.load().config.clone() } /// Get configuration for status and management responses without static key material. pub async fn get_redacted_config(&self) -> Option { - let mut config = self.config.read().await.clone()?; + let mut config = self.state.load().config.clone()?; + Self::redact_config(&mut config); + Some(config) + } + + /// Get status and redacted configuration from the same published snapshot. + pub async fn get_redacted_state(&self) -> (KmsServiceStatus, Option) { + let state = self.state.load(); + let mut config = state.config.clone(); + if let Some(config) = &mut config { + Self::redact_config(config); + } + (state.status.clone(), config) + } + + fn redact_config(config: &mut KmsConfig) { if let BackendConfig::Static(static_config) = &mut config.backend_config { use zeroize::Zeroize; static_config.secret_key.zeroize(); } - Some(config) } /// Configure KMS with new configuration pub async fn configure(&self, new_config: KmsConfig) -> Result<()> { - let _guard = self.lifecycle_mutex.lock().await; + self.configure_with_persistence(new_config, || async { Ok(()) }).await + } + + /// Configure KMS and publish the in-memory state only after persistence succeeds. + /// + /// The persistence callback runs under the lifecycle lock and must not call + /// another lifecycle method on this manager. + pub async fn configure_with_persistence(&self, new_config: KmsConfig, persist: Persist) -> Result<()> + where + Persist: FnOnce() -> PersistFuture, + PersistFuture: Future>, + { new_config.validate()?; - { - let config = self.config.read().await; - validate_local_transition(config.as_ref(), &new_config)?; - } - - // Update configuration - { - let mut config = self.config.write().await; - *config = Some(new_config.clone()); - } - - // Update status - { - let mut status = self.status.write().await; - *status = KmsServiceStatus::Configured; + let _guard = self.lifecycle_mutex.lock().await; + let current = self.state.load_full(); + validate_local_transition(current.config.as_ref(), &new_config)?; + if current.current_service.is_some() { + return Err(KmsError::configuration_error( + "Cannot configure KMS while it is running; use reconfigure instead", + )); } + persist().await?; + self.state.store(Arc::new(RuntimeState { + config: Some(new_config), + status: KmsServiceStatus::Configured, + current_service: None, + })); debug!( event = EVENT_KMS_SERVICE_STATE, @@ -190,19 +224,35 @@ impl KmsServiceManager { self.start_internal().await } + /// Start or restart KMS with the running-state decision serialized with the lifecycle action. + pub async fn start_or_restart(&self, force: bool) -> Result { + let _guard = self.lifecycle_mutex.lock().await; + let running = self.state.load().current_service.is_some(); + if running && !force { + return Ok(KmsStartOutcome::AlreadyRunning); + } + self.start_internal().await?; + Ok(if running { + KmsStartOutcome::Restarted + } else { + KmsStartOutcome::Started + }) + } + /// Internal start implementation (called within lifecycle mutex) async fn start_internal(&self) -> Result<()> { - let config = { - let config_guard = self.config.read().await; - match config_guard.as_ref() { - Some(config) => config.clone(), - None => { - let err_msg = "Cannot start KMS: no configuration provided"; - error!("{}", err_msg); - let mut status = self.status.write().await; - *status = KmsServiceStatus::Error(err_msg.to_string()); - return Err(KmsError::configuration_error(err_msg)); - } + let state = self.state.load_full(); + let config = match state.config.as_ref() { + Some(config) => config.clone(), + None => { + let err_msg = "Cannot start KMS: no configuration provided"; + error!("{}", err_msg); + self.state.store(Arc::new(RuntimeState { + config: None, + status: KmsServiceStatus::Error(err_msg.to_string()), + current_service: None, + })); + return Err(KmsError::configuration_error(err_msg)); } }; @@ -215,17 +265,9 @@ impl KmsServiceManager { "KMS service starting" ); - match self.create_service_version(&config).await { + match self.create_healthy_service_version(&config).await { Ok(service_version) => { - // Atomically update to new service version (lock-free, instant) - // ArcSwap::store() is a true atomic operation using CAS - self.current_service.store(Arc::new(Some(service_version))); - - // Update status - { - let mut status = self.status.write().await; - *status = KmsServiceStatus::Running; - } + self.publish_running(config, service_version); debug!( event = EVENT_KMS_SERVICE_STATE, @@ -239,13 +281,24 @@ impl KmsServiceManager { Err(e) => { let err_msg = format!("Failed to create KMS backend: {e}"); error!("{}", err_msg); - let mut status = self.status.write().await; - *status = KmsServiceStatus::Error(err_msg.clone()); + if state.current_service.is_none() { + self.state.store(Arc::new(RuntimeState { + config: state.config.clone(), + status: KmsServiceStatus::Error(err_msg.clone()), + current_service: None, + })); + } Err(KmsError::backend_error(&err_msg)) } } } + /// Replace the running service without exposing a stopped interval. + pub async fn restart(&self) -> Result<()> { + let _guard = self.lifecycle_mutex.lock().await; + self.start_internal().await + } + /// Stop KMS service /// /// Note: This stops accepting new operations, but existing operations using @@ -267,15 +320,16 @@ impl KmsServiceManager { // Atomically clear current service version (lock-free, instant) // Note: Existing Arc references will keep the service alive until operations complete - self.current_service.store(Arc::new(None)); - - // Update status (keep configuration) - { - let mut status = self.status.write().await; - if !matches!(*status, KmsServiceStatus::NotConfigured) { - *status = KmsServiceStatus::Configured; - } - } + let state = self.state.load_full(); + self.state.store(Arc::new(RuntimeState { + config: state.config.clone(), + status: if state.config.is_some() { + KmsServiceStatus::Configured + } else { + KmsServiceStatus::NotConfigured + }, + current_service: None, + })); debug!( event = EVENT_KMS_SERVICE_STATE, @@ -298,6 +352,22 @@ impl KmsServiceManager { /// This ensures zero downtime during reconfiguration, even for long-running /// operations like encrypting large files. pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> { + self.reconfigure_with_persistence(new_config, || async { Ok(()) }).await + } + + /// Reconfigure KMS after the candidate is healthy and persistence succeeds. + /// + /// The persistence callback runs under the lifecycle lock and must not call + /// another lifecycle method on this manager. + pub async fn reconfigure_with_persistence( + &self, + new_config: KmsConfig, + persist: Persist, + ) -> Result<()> + where + Persist: FnOnce() -> PersistFuture, + PersistFuture: Future>, + { let _guard = self.lifecycle_mutex.lock().await; debug!( @@ -308,33 +378,18 @@ impl KmsServiceManager { "KMS service reconfiguring" ); new_config.validate()?; - { - let config = self.config.read().await; - validate_local_transition(config.as_ref(), &new_config)?; - } + validate_local_transition(self.state.load().config.as_ref(), &new_config)?; // Create new service version without stopping old one // This allows existing operations to continue while new operations use new service - match self.create_service_version(&new_config).await { + match self.create_healthy_service_version(&new_config).await { Ok(new_service_version) => { // Get old version for logging (lock-free read) - let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version); + let old_version = self.state.load().current_service.as_ref().map(|sv| sv.version); - { - let mut config = self.config.write().await; - *config = Some(new_config); - } + persist().await?; - // Atomically switch to new service version (lock-free, instant CAS operation) - // This is a true atomic operation - no waiting for locks, instant switch - // Old service will be dropped when no more Arc references exist - self.current_service.store(Arc::new(Some(new_service_version.clone()))); - - // Update status - { - let mut status = self.status.write().await; - *status = KmsServiceStatus::Running; - } + self.publish_running(new_config, new_service_version.clone()); if let Some(old_ver) = old_version { info!( @@ -371,7 +426,7 @@ impl KmsServiceManager { /// Returns the manager from the current service version. /// Uses lock-free atomic load for optimal performance. pub async fn get_manager(&self) -> Option> { - self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone()) + self.state.load().current_service.as_ref().map(|sv| sv.manager.clone()) } /// Get encryption service (if running) @@ -381,7 +436,7 @@ impl KmsServiceManager { /// This ensures new operations always use the latest service version, /// while existing operations continue using their Arc references. pub async fn get_encryption_service(&self) -> Option> { - self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone()) + self.state.load().current_service.as_ref().map(|sv| sv.service.clone()) } /// Get current service version number @@ -389,14 +444,16 @@ impl KmsServiceManager { /// Useful for monitoring and debugging. /// Uses lock-free atomic load. pub async fn get_service_version(&self) -> Option { - self.current_service.load().as_ref().as_ref().map(|sv| sv.version) + self.state.load().current_service.as_ref().map(|sv| sv.version) } /// Health check for the KMS service pub async fn health_check(&self) -> Result { - let manager = self.get_manager().await; - match manager { - Some(manager) => { + let checked_state = self.state.load_full(); + match checked_state.current_service.as_ref() { + Some(service_version) => { + let manager = service_version.manager.clone(); + let checked_version = service_version.version; // Perform health check on the backend match manager.health_check().await { Ok(healthy) => { @@ -407,9 +464,8 @@ impl KmsServiceManager { } Err(e) => { error!("KMS health check error: {}", e); - // Update status to error - let mut status = self.status.write().await; - *status = KmsServiceStatus::Error(format!("Health check failed: {e}")); + let _guard = self.lifecycle_mutex.lock().await; + self.mark_health_error_if_current(checked_version, &e); Err(e) } } @@ -468,6 +524,33 @@ impl KmsServiceManager { manager: kms_manager, }) } + + async fn create_healthy_service_version(&self, config: &KmsConfig) -> Result { + let service_version = self.create_service_version(config).await?; + if !service_version.manager.health_check().await? { + return Err(KmsError::backend_error("KMS backend health check failed")); + } + Ok(service_version) + } + + fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) { + self.state.store(Arc::new(RuntimeState { + config: Some(config), + status: KmsServiceStatus::Running, + current_service: Some(service_version), + })); + } + + fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) { + let current = self.state.load_full(); + if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) { + self.state.store(Arc::new(RuntimeState { + config: current.config.clone(), + status: KmsServiceStatus::Error(format!("Health check failed: {error}")), + current_service: current.current_service.clone(), + })); + } + } } impl Default for KmsServiceManager { @@ -500,6 +583,11 @@ pub async fn get_global_encryption_service() -> Option KmsConfig { + KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32])) + } #[tokio::test] async fn configure_rejects_insecure_development_defaults_before_state_update() { @@ -517,8 +605,6 @@ mod tests { #[tokio::test] async fn redacted_config_omits_static_key_material() { - use base64::Engine as _; - let manager = KmsServiceManager::new(); let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]); manager @@ -533,6 +619,136 @@ mod tests { assert!(static_config.secret_key.is_empty()); } + #[tokio::test] + async fn configure_persistence_failure_leaves_state_unchanged() { + let manager = KmsServiceManager::new(); + + let result = manager + .configure_with_persistence(static_config("key-a", 0x11), || async { Err(KmsError::backend_error("persist failed")) }) + .await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured); + assert!(manager.get_config().await.is_none()); + assert!(manager.get_encryption_service().await.is_none()); + } + + #[tokio::test] + async fn configure_rejects_running_service_without_changing_snapshot() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let version = manager.get_service_version().await; + + let result = manager.configure(static_config("key-b", 0x22)).await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, version); + assert_eq!( + manager.get_config().await.and_then(|config| config.default_key_id), + Some("key-a".to_string()) + ); + } + + #[tokio::test] + async fn reconfigure_persistence_failure_keeps_old_running_snapshot() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await; + let old_service = manager.get_encryption_service().await.expect("old service"); + + let result = manager + .reconfigure_with_persistence(static_config("key-b", 0x22), || async { + Err(KmsError::backend_error("persist failed")) + }) + .await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, old_version); + assert_eq!( + manager.get_config().await.and_then(|config| config.default_key_id), + Some("key-a".to_string()) + ); + assert!(Arc::ptr_eq( + &old_service, + &manager.get_encryption_service().await.expect("old service remains") + )); + } + + #[tokio::test] + async fn reconfigure_candidate_failure_keeps_old_running_snapshot() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await; + let invalid_parent = tempfile::NamedTempFile::new().expect("temporary file"); + let invalid_config = KmsConfig::local(invalid_parent.path().join("keys")).with_insecure_development_defaults(); + + let result = manager.reconfigure(invalid_config).await; + + assert!(result.is_err()); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + assert_eq!(manager.get_service_version().await, old_version); + assert_eq!( + manager.get_config().await.and_then(|config| config.default_key_id), + Some("key-a".to_string()) + ); + } + + #[tokio::test] + async fn restart_never_unpublishes_the_running_service() { + let manager = Arc::new(KmsServiceManager::new()); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await.expect("old version"); + let restarting = { + let manager = manager.clone(); + tokio::spawn(async move { manager.restart().await }) + }; + + while !restarting.is_finished() { + assert!(manager.get_encryption_service().await.is_some()); + tokio::task::yield_now().await; + } + restarting.await.expect("restart task").expect("restart"); + + assert!(manager.get_encryption_service().await.is_some()); + assert!(manager.get_service_version().await.expect("new version") > old_version); + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + } + + #[tokio::test] + async fn start_or_restart_decides_under_the_lifecycle_lock() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + + assert_eq!(manager.start_or_restart(false).await.expect("initial start"), KmsStartOutcome::Started); + let first_version = manager.get_service_version().await.expect("first version"); + assert_eq!( + manager.start_or_restart(false).await.expect("already running"), + KmsStartOutcome::AlreadyRunning + ); + assert_eq!(manager.get_service_version().await, Some(first_version)); + assert_eq!(manager.start_or_restart(true).await.expect("forced restart"), KmsStartOutcome::Restarted); + assert!(manager.get_service_version().await.expect("restarted version") > first_version); + } + + #[tokio::test] + async fn stale_health_failure_cannot_poison_new_service_status() { + let manager = KmsServiceManager::new(); + manager.configure(static_config("key-a", 0x11)).await.expect("configure"); + manager.start().await.expect("start"); + let old_version = manager.get_service_version().await.expect("old version"); + manager.restart().await.expect("restart"); + + manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure")); + + assert_eq!(manager.get_status().await, KmsServiceStatus::Running); + } + #[tokio::test] async fn forbidden_local_master_key_change_preserves_running_config_and_service() { use crate::types::{CreateKeyRequest, KeyUsage}; diff --git a/crates/rio-v2/tests/minio_fixture_lab/README.md b/crates/rio-v2/tests/minio_fixture_lab/README.md index f20abe205..81ae85ef6 100644 --- a/crates/rio-v2/tests/minio_fixture_lab/README.md +++ b/crates/rio-v2/tests/minio_fixture_lab/README.md @@ -137,7 +137,7 @@ tests read): ./capture_via_docker.sh RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \ - cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored + cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored ``` This is exactly what the nightly `minio-interop` GitHub Actions workflow runs diff --git a/crates/utils/src/http/header_compat.rs b/crates/utils/src/http/header_compat.rs index 6dda300e6..d1bd38b82 100644 --- a/crates/utils/src/http/header_compat.rs +++ b/crates/utils/src/http/header_compat.rs @@ -25,6 +25,11 @@ const RUSTFS_PREFIX: &str = "x-rustfs-"; const MINIO_PREFIX: &str = "x-minio-"; const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-"; const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-"; +const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-"; +const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart"; +const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = "x-rustfs-encryption-original-size"; +const MINIO_ENCRYPTION_ORIGINAL_SIZE: &str = "x-minio-encryption-original-size"; +const SSEC_ORIGINAL_SIZE: &str = "x-amz-server-side-encryption-customer-original-size"; // Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header. pub const SUFFIX_FORCE_DELETE: &str = "force-delete"; @@ -40,11 +45,49 @@ pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request" pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check"; pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc"; -/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or -/// x-minio-encryption-*). Case-insensitive for metadata filtering. +/// Returns true if the key is object-encryption metadata understood by RustFS or MinIO. +/// Case-insensitive for metadata filtering. pub fn is_encryption_metadata_key(key: &str) -> bool { let lower = key.to_lowercase(); - lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX) + lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) + || lower.starts_with(MINIO_ENCRYPTION_PREFIX) + || lower.starts_with(MINIO_INTERNAL_ENCRYPTION_PREFIX) + || lower == MINIO_INTERNAL_ENCRYPTED_MULTIPART +} + +/// Returns true when a metadata key proves that object data is encrypted. +/// +/// Original-size metadata alone is not proof: older plaintext objects can +/// retain that compatibility field after metadata migration. +pub fn is_object_encryption_marker(key: &str) -> bool { + (is_encryption_metadata_key(key) + && !key.eq_ignore_ascii_case(RUSTFS_ENCRYPTION_ORIGINAL_SIZE) + && !key.eq_ignore_ascii_case(MINIO_ENCRYPTION_ORIGINAL_SIZE)) + || super::is_sse_header(key) +} + +/// Reads the logical object size recorded by encryption metadata. +pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap) -> std::io::Result> { + let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE); + let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE) + .or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE)) + .or(actual_size.as_deref()); + + let Some(size) = size.filter(|size| !size.is_empty()) else { + return Ok(None); + }; + size.parse::() + .map(Some) + .map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}"))) +} + +fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap, key: &str) -> Option<&'a str> { + metadata.get(key).map(String::as_str).or_else(|| { + metadata + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(key)) + .map(|(_, value)| value.as_str()) + }) } fn rustfs_key(suffix: &str) -> String { @@ -106,10 +149,37 @@ mod tests { assert!(is_encryption_metadata_key("x-rustfs-encryption-iv")); assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key")); assert!(is_encryption_metadata_key("x-minio-encryption-iv")); + assert!(is_encryption_metadata_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key")); + assert!(is_encryption_metadata_key("X-Minio-Internal-Encrypted-Multipart")); assert!(!is_encryption_metadata_key("x-amz-meta-custom")); assert!(!is_encryption_metadata_key("x-rustfs-internal-healing")); } + #[test] + fn object_encryption_marker_excludes_size_only_metadata() { + assert!(!is_object_encryption_marker(RUSTFS_ENCRYPTION_ORIGINAL_SIZE)); + assert!(is_object_encryption_marker("X-Minio-Internal-Server-Side-Encryption-Sealed-Key")); + assert!(is_object_encryption_marker("x-amz-server-side-encryption")); + } + + #[test] + fn object_encryption_original_size_is_case_insensitive() { + let metadata = std::collections::HashMap::from([( + "X-Amz-Server-Side-Encryption-Customer-Original-Size".to_string(), + "42".to_string(), + )]); + assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42)); + } + + #[test] + fn object_encryption_original_size_prefers_rustfs_metadata() { + let metadata = std::collections::HashMap::from([ + (SSEC_ORIGINAL_SIZE.to_string(), "21".to_string()), + (RUSTFS_ENCRYPTION_ORIGINAL_SIZE.to_string(), "42".to_string()), + ]); + assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42)); + } + #[test] fn test_get_header() { let mut headers = HeaderMap::new(); diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index 4eb3b41d5..62e95f340 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -358,7 +358,7 @@ Fixture-backed tests should run when the fixture path is present: ```bash cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture -cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture +cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture ``` ## Multi-Expert Adversarial Review Summary diff --git a/rustfs/src/admin/handlers/kms_dynamic.rs b/rustfs/src/admin/handlers/kms_dynamic.rs index e8fb0ee64..6a6da70d4 100644 --- a/rustfs/src/admin/handlers/kms_dynamic.rs +++ b/rustfs/src/admin/handlers/kms_dynamic.rs @@ -393,36 +393,27 @@ impl Operation for ConfigureKmsHandler { // Convert request to KmsConfig let kms_config = configure_request.to_kms_config(); - // Configure the service - let (success, message, status) = match service_manager.configure(kms_config.clone()).await { + let persisted_config = kms_config.clone(); + let (success, message, status) = match service_manager + .configure_with_persistence(kms_config, || async move { + save_kms_config(&persisted_config) + .await + .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}"))) + }) + .await + { Ok(()) => { - // Persist the configuration to cluster storage - if let Err(e) = save_kms_config(&kms_config).await { - let error_msg = format!("KMS configured in memory but failed to persist: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "configure", - state = "persist_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } else { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "configure", - state = "configured", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS configured successfully".to_string(), status) - } + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "configure", + state = "configured", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS configured successfully".to_string(), status) } Err(e) => { let error_msg = format!("Failed to configure KMS: {e}"); @@ -529,125 +520,61 @@ impl Operation for StartKmsHandler { ); let service_manager = kms_service_manager_from_context(); - - // Check if already running and force flag - let current_status = service_manager.get_status().await; - if matches!(current_status, KmsServiceStatus::Running) && !start_request.force.unwrap_or(false) { - warn!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "start", - state = "already_running", - "admin kms dynamic state" - ); - let response = StartKmsResponse { - success: false, - message: "KMS service is already running. Use force=true to restart.".to_string(), - status: current_status, - }; - let json_response = match serde_json::to_string(&response) { - Ok(json) => json, - Err(e) => { - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = EVENT_ADMIN_KMS_DYNAMIC_STATE, - operation = "start", - result = "response_serialize_failed", - error = %e, - "admin kms dynamic state" - ); - return Ok(S3Response::new(( - StatusCode::INTERNAL_SERVER_ERROR, - Body::from("Serialization error".to_string()), - ))); - } - }; - return Ok(S3Response::new((StatusCode::OK, Body::from(json_response)))); - } - - // Start the service (or restart if force=true) - let (success, message, status) = - if start_request.force.unwrap_or(false) && matches!(current_status, KmsServiceStatus::Running) { - // Force restart - match service_manager.stop().await { - Ok(()) => match service_manager.start().await { - Ok(()) => { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "restart", - state = "running", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS service restarted successfully".to_string(), status) - } - Err(e) => { - let error_msg = format!("Failed to restart KMS service: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "restart", - state = "start_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } - }, - Err(e) => { - let error_msg = format!("Failed to stop KMS service for restart: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "restart", - state = "stop_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } - } - } else { - // Normal start - match service_manager.start().await { - Ok(()) => { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "start", - state = "running", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS service started successfully".to_string(), status) - } - Err(e) => { - let error_msg = format!("Failed to start KMS service: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "start", - state = "start_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } - } - }; + let force = start_request.force.unwrap_or(false); + let (success, message, status) = match service_manager.start_or_restart(force).await { + Ok(rustfs_kms::KmsStartOutcome::Started) => { + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "start", + state = "running", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS service started successfully".to_string(), status) + } + Ok(rustfs_kms::KmsStartOutcome::Restarted) => { + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "restart", + state = "running", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS service restarted successfully".to_string(), status) + } + Ok(rustfs_kms::KmsStartOutcome::AlreadyRunning) => { + let status = service_manager.get_status().await; + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "start", + state = "already_running", + "admin kms dynamic state" + ); + (false, "KMS service is already running. Use force=true to restart.".to_string(), status) + } + Err(e) => { + let error_msg = format!("Failed to start or restart KMS service: {e}"); + error!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "start", + state = "start_failed", + error = %e, + "admin kms dynamic state" + ); + let status = service_manager.get_status().await; + (false, error_msg, status) + } + }; let response = StartKmsResponse { success, @@ -804,8 +731,7 @@ impl Operation for GetKmsStatusHandler { let service_manager = kms_service_manager_from_context(); - let status = service_manager.get_status().await; - let config = service_manager.get_redacted_config().await; + let (status, config) = service_manager.get_redacted_state().await; // Get backend type and health status let backend_type = config.as_ref().map(|c| c.backend.clone()); @@ -938,36 +864,27 @@ impl Operation for ReconfigureKmsHandler { // Convert request to KmsConfig let kms_config = configure_request.to_kms_config(); - // Reconfigure the service (stops, reconfigures, and starts) - let (success, message, status) = match service_manager.reconfigure(kms_config.clone()).await { + let persisted_config = kms_config.clone(); + let (success, message, status) = match service_manager + .reconfigure_with_persistence(kms_config, || async move { + save_kms_config(&persisted_config) + .await + .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}"))) + }) + .await + { Ok(()) => { - // Persist the configuration to cluster storage - if let Err(e) = save_kms_config(&kms_config).await { - let error_msg = format!("KMS reconfigured in memory but failed to persist: {e}"); - error!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "reconfigure", - state = "persist_failed", - error = %e, - "admin kms dynamic state" - ); - let status = service_manager.get_status().await; - (false, error_msg, status) - } else { - let status = service_manager.get_status().await; - info!( - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_KMS, - event = "kms_service_state", - operation = "reconfigure", - state = "reconfigured", - status = ?status, - "admin kms dynamic state" - ); - (true, "KMS reconfigured and restarted successfully".to_string(), status) - } + let status = service_manager.get_status().await; + info!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_KMS, + event = "kms_service_state", + operation = "reconfigure", + state = "reconfigured", + status = ?status, + "admin kms dynamic state" + ); + (true, "KMS reconfigured and restarted successfully".to_string(), status) } Err(e) => { let error_msg = format!("Failed to reconfigure KMS: {e}"); diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index bd1baf9a6..84118caba 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -18,13 +18,12 @@ use super::handles::{ IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface, default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface, default_endpoints_interface, default_expiry_state_interface, default_federated_identity_interface, - default_internode_metrics_interface, default_kms_runtime_interface, default_local_node_name_interface, - default_lock_client_interface, default_lock_clients_interface, default_notification_system_interface, - default_notify_interface, default_outbound_tls_runtime_interface, default_performance_metrics_interface, - default_region_interface, default_replication_pool_interface, default_replication_stats_interface, - default_runtime_port_interface, default_s3select_db_interface, default_scanner_metrics_interface, - default_server_config_interface, default_storage_class_interface, default_tier_config_interface, - default_transition_state_interface, + default_internode_metrics_interface, default_local_node_name_interface, default_lock_client_interface, + default_lock_clients_interface, default_notification_system_interface, default_notify_interface, + default_outbound_tls_runtime_interface, default_performance_metrics_interface, default_region_interface, + default_replication_pool_interface, default_replication_stats_interface, default_runtime_port_interface, + default_s3select_db_interface, default_scanner_metrics_interface, default_server_config_interface, + default_storage_class_interface, default_tier_config_interface, default_transition_state_interface, }; use super::interfaces::{ ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface, @@ -80,6 +79,7 @@ pub struct AppContext { impl AppContext { pub fn new(object_store: Arc, iam: Arc, kms: Arc) -> Self { let object_data_cache = ObjectDataCacheAdapter::from_env_or_disabled(); + let kms_runtime = Arc::new(crate::app::context::handles::KmsRuntimeHandle::new(kms.handle())); // Let ecstore probe this cache inside get_object_reader, after // metadata resolution but before the erasure data read (backlog#802). crate::app::object_data_cache::register_object_data_cache_body_hook(Arc::clone(&object_data_cache)); @@ -94,7 +94,7 @@ impl AppContext { iam, federated_identity: default_federated_identity_interface(), kms, - kms_runtime: default_kms_runtime_interface(), + kms_runtime, outbound_tls_runtime: default_outbound_tls_runtime_interface(), notify: default_notify_interface(), notification_system: default_notification_system_interface(), diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 7eb4e3f29..9e376e1a5 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -128,12 +128,19 @@ impl KmsInterface for KmsHandle { } /// Default KMS runtime interface adapter. -#[derive(Default)] -pub struct KmsRuntimeHandle; +pub struct KmsRuntimeHandle { + kms: Option>, +} + +impl KmsRuntimeHandle { + pub fn new(kms: Arc) -> Self { + Self { kms: Some(kms) } + } +} impl KmsRuntimeInterface for KmsRuntimeHandle { fn service_manager(&self) -> Option> { - runtime_sources::kms_service_manager() + self.kms.clone() } } @@ -485,7 +492,9 @@ pub fn default_notification_system_interface() -> Arc Arc { - Arc::new(KmsRuntimeHandle) + Arc::new(KmsRuntimeHandle { + kms: runtime_sources::kms_service_manager(), + }) } pub fn default_outbound_tls_runtime_interface() -> Arc { @@ -607,10 +616,10 @@ pub fn default_buffer_config_interface() -> Arc { #[cfg(test)] mod tests { use super::{ - ServerConfigHandle, default_federated_identity_interface, federated_identity_interface, - publish_default_federated_identity_service, runtime_sources, + KmsRuntimeHandle, KmsServiceManager, ServerConfigHandle, default_federated_identity_interface, + federated_identity_interface, publish_default_federated_identity_service, runtime_sources, }; - use crate::app::context::interfaces::ServerConfigInterface; + use crate::app::context::interfaces::{KmsRuntimeInterface, ServerConfigInterface}; use rustfs_config::server_config::Config; use rustfs_iam::{ federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter}, @@ -733,4 +742,19 @@ mod tests { "handle B must serve its own credentials" ); } + + #[test] + fn kms_runtime_handles_keep_injected_managers_isolated() { + let manager_a = Arc::new(KmsServiceManager::new()); + let manager_b = Arc::new(KmsServiceManager::new()); + let handle_a = KmsRuntimeHandle::new(manager_a.clone()); + let handle_b = KmsRuntimeHandle::new(manager_b.clone()); + + assert!(Arc::ptr_eq(&handle_a.service_manager().expect("manager A"), &manager_a)); + assert!(Arc::ptr_eq(&handle_b.service_manager().expect("manager B"), &manager_b)); + assert!(!Arc::ptr_eq( + &handle_a.service_manager().expect("manager A"), + &handle_b.service_manager().expect("manager B") + )); + } } diff --git a/crates/ecstore/tests/minio_generated_read_test.rs b/rustfs/src/storage/minio_generated_read_test.rs similarity index 96% rename from crates/ecstore/tests/minio_generated_read_test.rs rename to rustfs/src/storage/minio_generated_read_test.rs index 875aa9be9..36285c565 100644 --- a/crates/ecstore/tests/minio_generated_read_test.rs +++ b/rustfs/src/storage/minio_generated_read_test.rs @@ -4,14 +4,13 @@ use std::fs; use std::io::Cursor; use std::path::{Path, PathBuf}; -mod storage_api; - +use super::sse::SseObjectEncryptionResolver; +use super::storage_api::ecstore_test_support::{ + DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk, +}; use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info}; use serde::Deserialize; use sha2::{Digest, Sha256}; -use storage_api::minio_generated_read::{ - DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk, -}; use temp_env::async_with_vars; use tokio::io::{AsyncReadExt, AsyncWrite}; @@ -49,7 +48,7 @@ impl AsyncWrite for VecAsyncWriter { fn fixture_root() -> PathBuf { std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT") .map(PathBuf::from) - .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated")) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../crates/rio-v2/tests/fixtures/minio-generated")) } fn case_dir(case_id: &str) -> PathBuf { @@ -137,12 +136,14 @@ async fn read_fixture_plaintext(encrypted: Vec, object_info: ObjectInfo, kms ("RUSTFS_SSE_S3_MASTER_KEY", None::), ], async move { - let (mut reader, offset, length) = GetObjectReader::new( + let resolver = SseObjectEncryptionResolver; + let (mut reader, offset, length) = GetObjectReader::new_with_resolver( Box::new(Cursor::new(encrypted)), None, &object_info, &ObjectOptions::default(), &http::HeaderMap::new(), + Some(&resolver), ) .await .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?; @@ -219,11 +220,12 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil readers.push(reader); } - let erasure = Erasure::new( + let erasure = Erasure::try_new( file_info.erasure.data_blocks, file_info.erasure.parity_blocks, file_info.erasure.block_size, - ); + ) + .expect("fixture erasure geometry"); let mut writer = VecAsyncWriter::default(); let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await; if let Some(err) = err { diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 9033400c4..f427a3345 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -36,6 +36,8 @@ mod ecfs_extend; mod ecfs_test; pub(crate) mod head_prefix; #[cfg(test)] +mod minio_generated_read_test; +#[cfg(test)] mod multi_factor_scheduler_integration_test; pub(crate) mod runtime_sources; #[cfg(test)] diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 0cd426d6e..e06b9dea5 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -70,6 +70,10 @@ //! ``` use super::StorageError; +use super::storage_api::ecstore_object::{ + EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial, + ReadEncryptionMode, ReadEncryptionRequest, +}; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; #[cfg(feature = "rio-v2")] use aes_gcm::aead::Payload; @@ -155,6 +159,7 @@ use rustfs_utils::http::headers::{ }; use rustfs_utils::path::path_join_buf; use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId}; +use std::borrow::Cow; // ============================================================================ // High-Level SSE Configuration @@ -652,6 +657,23 @@ pub(crate) fn validate_sse_headers_for_read(metadata: &HashMap, } pub(crate) fn map_get_object_reader_error(err: StorageError) -> ApiError { + if let StorageError::Io(io_error) = &err + && let Some(resolution_error) = io_error + .get_ref() + .and_then(|source| source.downcast_ref::()) + { + let code = match resolution_error.kind() { + EncryptionResolutionErrorKind::InvalidRequest => S3ErrorCode::InvalidRequest, + EncryptionResolutionErrorKind::ServiceUnavailable => S3ErrorCode::ServiceUnavailable, + _ => S3ErrorCode::InternalError, + }; + return ApiError { + code, + message: resolution_error.to_string(), + source: Some(Box::new(err)), + }; + } + if let Some(message) = map_ssec_get_object_reader_error_message(&err) { return ApiError { code: S3ErrorCode::InvalidRequest, @@ -774,6 +796,117 @@ pub enum EncryptionKeyKind { Object, } +pub(crate) struct SseObjectEncryptionResolver; + +#[async_trait] +impl ObjectEncryptionResolver for SseObjectEncryptionResolver { + async fn resolve_read_material( + &self, + request: ReadEncryptionRequest<'_>, + ) -> Result, EncryptionResolutionError> { + let metadata = normalize_encryption_metadata_case(request.metadata)?; + let (customer_algorithm, customer_key, customer_key_md5) = + extract_ssec_params_from_headers(request.headers).map_err(map_encryption_resolution_error)?; + if let Some(stored_algorithm) = metadata.get("x-amz-server-side-encryption-customer-algorithm") { + let request_algorithm = customer_algorithm.as_ref().ok_or_else(|| { + map_encryption_resolution_error(ssec_invalid_request( + "The object was stored using a form of Server Side Encryption. \ + The correct parameters must be provided to retrieve the object.", + )) + })?; + if stored_algorithm != request_algorithm.as_str() { + return Err(map_encryption_resolution_error(ssec_invalid_request( + "The provided encryption parameters did not match the ones used originally to encrypt the object.", + ))); + } + } + let material = sse_decryption(DecryptionRequest { + bucket: request.bucket, + key: request.object, + metadata: &metadata, + sse_customer_key: customer_key.as_ref(), + sse_customer_key_md5: customer_key_md5.as_ref(), + }) + .await + .map_err(map_encryption_resolution_error)?; + + Ok(material.map(|material| ReadEncryptionMaterial { + key_bytes: material.key_bytes, + mode: match material.key_kind { + EncryptionKeyKind::Direct => ReadEncryptionMode::Direct { + base_nonce: material.base_nonce, + }, + EncryptionKeyKind::Object => ReadEncryptionMode::Object, + }, + })) + } +} + +fn normalize_encryption_metadata_case( + metadata: &HashMap, +) -> Result>, EncryptionResolutionError> { + const CANONICAL_KEYS: &[&str] = &[ + "x-amz-server-side-encryption", + "x-amz-server-side-encryption-aws-kms-key-id", + "x-amz-server-side-encryption-customer-algorithm", + "x-amz-server-side-encryption-customer-key-md5", + SSEC_ORIGINAL_SIZE_HEADER, + INTERNAL_ENCRYPTION_KEY_ID_HEADER, + INTERNAL_ENCRYPTION_KEY_HEADER, + INTERNAL_ENCRYPTION_ALGORITHM_HEADER, + INTERNAL_ENCRYPTION_IV_HEADER, + "x-rustfs-encryption-context", + "x-rustfs-encryption-tag", + INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, + MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, + MINIO_INTERNAL_ENCRYPTION_IV_HEADER, + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, + MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, + ]; + + let needs_normalization = metadata.keys().any(|key| { + CANONICAL_KEYS + .iter() + .any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical)) + }); + if !needs_normalization { + return Ok(Cow::Borrowed(metadata)); + } + + let mut normalized = metadata.clone(); + for canonical in CANONICAL_KEYS { + let mut matching_values = metadata + .iter() + .filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value)); + let Some(value) = matching_values.next() else { + continue; + }; + if matching_values.any(|candidate| candidate != value) { + return Err(EncryptionResolutionError::new( + EncryptionResolutionErrorKind::InvalidMetadata, + format!("conflicting object encryption metadata for {canonical}"), + )); + } + if !normalized.contains_key(*canonical) { + normalized.insert((*canonical).to_string(), value.clone()); + } + } + Ok(Cow::Owned(normalized)) +} + +fn map_encryption_resolution_error(error: ApiError) -> EncryptionResolutionError { + let kind = match error.code { + S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => EncryptionResolutionErrorKind::InvalidRequest, + S3ErrorCode::ServiceUnavailable => EncryptionResolutionErrorKind::ServiceUnavailable, + _ => EncryptionResolutionErrorKind::DecryptionFailed, + }; + EncryptionResolutionError::new(kind, error.message) +} + #[derive(Debug, Clone)] pub struct ManagedSealedKey { #[cfg(feature = "rio-v2")] @@ -2642,19 +2775,20 @@ fn ssec_invalid_request(message: &str) -> ApiError { mod tests { use super::{ ApiError, DataKey, DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest, - INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER, - INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, - MINIO_INTERNAL_ENCRYPTION_IV_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, - MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, - MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, - MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType, - SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material, - apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers, - extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse, - kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, md5_base64, - normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, - sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, - validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match, + EncryptionResolutionErrorKind, INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, + INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + ObjectEncryptionResolver, PrepareEncryptionRequest, ReadEncryptionMode, ReadEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, + SSEType, SseDekProvider, SseObjectEncryptionResolver, SsecParams, StorageError, TestSseDekProvider, + apply_managed_decryption_material, apply_managed_encryption_material, encryption_material_to_metadata, + extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, + generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, + md5_base64, normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, + sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, + validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match, }; #[cfg(feature = "rio-v2")] use super::{ @@ -2709,11 +2843,154 @@ mod tests { use tokio::sync::Mutex; static SSE_TEST_LOCK: OnceLock> = OnceLock::new(); + static SSE_TEST_KMS_KEY_DIR: OnceLock = OnceLock::new(); async fn lock_sse_test_state() -> tokio::sync::MutexGuard<'static, ()> { SSE_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await } + async fn configure_test_global_local_kms() -> Arc { + let key_dir = SSE_TEST_KMS_KEY_DIR.get_or_init(|| tempfile::TempDir::new().expect("create KMS key directory")); + let manager = rustfs_kms::init_global_kms_service_manager(); + manager + .reconfigure(rustfs_kms::KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) + .await + .expect("configure test KMS service"); + manager + } + + #[tokio::test] + async fn object_encryption_resolver_returns_ssec_read_material() { + let key = [0x31; 32]; + let key_b64 = BASE64_STANDARD.encode(key); + let key_md5 = md5_base64(key); + let nonce = [0x42; 12]; + let metadata = HashMap::from([ + ("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()), + ("X-Amz-Server-Side-Encryption-Customer-Key-Md5".to_string(), key_md5.clone()), + ("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode(nonce)), + ]); + let mut headers = HeaderMap::new(); + headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256")); + headers.insert( + "x-amz-server-side-encryption-customer-key", + HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"), + ); + headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"), + ); + + let material = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &headers, + }) + .await + .expect("SSE-C material should resolve") + .expect("SSE-C metadata should produce material"); + + assert_eq!(material.key_bytes, key); + assert_eq!(material.mode, ReadEncryptionMode::Direct { base_nonce: nonce }); + } + + #[tokio::test] + async fn object_encryption_resolver_rejects_missing_or_invalid_ssec_algorithm() { + let key = [0x31; 32]; + let key_b64 = BASE64_STANDARD.encode(key); + let key_md5 = md5_base64(key); + let metadata = HashMap::from([ + ("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()), + ("x-amz-server-side-encryption-customer-key-md5".to_string(), key_md5.clone()), + ]); + + for algorithm in [None, Some("AES128")] { + let mut headers = HeaderMap::new(); + if let Some(algorithm) = algorithm { + headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static(algorithm)); + } + headers.insert( + "x-amz-server-side-encryption-customer-key", + HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"), + ); + headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"), + ); + + let result = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &headers, + }) + .await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("missing or invalid SSE-C algorithm must fail closed"), + }; + + assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest); + } + } + + #[tokio::test] + async fn object_encryption_resolver_classifies_missing_ssec_key_as_invalid_request() { + let metadata = HashMap::from([("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string())]); + let result = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &HeaderMap::new(), + }) + .await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("missing SSE-C key must fail closed"), + }; + + assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest); + } + + #[tokio::test] + async fn object_encryption_resolver_rejects_conflicting_metadata_case_variants() { + let metadata = HashMap::from([ + ("x-rustfs-encryption-key".to_string(), "first".to_string()), + ("X-Rustfs-Encryption-Key".to_string(), "second".to_string()), + ]); + let result = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &HeaderMap::new(), + }) + .await; + let error = match result { + Err(error) => error, + Ok(_) => panic!("conflicting metadata aliases must fail closed"), + }; + + assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidMetadata); + } + + #[test] + fn normalize_encryption_metadata_case_accepts_lowercase_minio_internal_keys() { + let lowercase_key = MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase(); + let metadata = HashMap::from([(lowercase_key, "sealed-key".to_string())]); + + let normalized = super::normalize_encryption_metadata_case(&metadata).expect("metadata aliases should normalize"); + + assert_eq!( + normalized.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER), + Some(&"sealed-key".to_string()) + ); + } + struct UnavailableSseDekProvider; #[async_trait::async_trait] @@ -3433,18 +3710,11 @@ mod tests { #[cfg(feature = "rio-v2")] #[tokio::test] async fn test_sse_kms_roundtrip_persists_and_uses_minio_context() { - use rustfs_kms::config::KmsConfig; use rustfs_kms::types::{CreateKeyRequest, KeyUsage}; - use tempfile::TempDir; let _guard = lock_sse_test_state().await; reset_sse_dek_provider(); - let manager = rustfs_kms::init_global_kms_service_manager(); - let temp_dir = TempDir::new().expect("temp dir"); - manager - .reconfigure(KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("kms reconfigure should succeed"); + let manager = configure_test_global_local_kms().await; manager .get_encryption_service() .await @@ -3736,6 +4006,19 @@ mod tests { assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object); assert_eq!(decrypted.key_bytes, material.key_bytes); + + let resolved = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &HeaderMap::new(), + }) + .await + .expect("managed resolver") + .expect("managed material"); + assert_eq!(resolved.mode, ReadEncryptionMode::Object); + assert_eq!(resolved.key_bytes, material.key_bytes); }, ) .await; @@ -3796,6 +4079,29 @@ mod tests { assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object); assert_eq!(decrypted.key_bytes, material.key_bytes); + + let mut headers = HeaderMap::new(); + headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256")); + headers.insert( + "x-amz-server-side-encryption-customer-key", + HeaderValue::from_str(&customer_key).expect("customer key header"), + ); + headers.insert( + "x-amz-server-side-encryption-customer-key-md5", + HeaderValue::from_str(&customer_key_md5).expect("customer key MD5 header"), + ); + let resolved = SseObjectEncryptionResolver + .resolve_read_material(ReadEncryptionRequest { + bucket: "bucket", + object: "object", + metadata: &metadata, + headers: &headers, + }) + .await + .expect("SSE-C resolver") + .expect("SSE-C material"); + assert_eq!(resolved.mode, ReadEncryptionMode::Object); + assert_eq!(resolved.key_bytes, material.key_bytes); } #[cfg(feature = "rio-v2")] @@ -4270,17 +4576,9 @@ mod tests { #[tokio::test] async fn test_managed_decryption_selects_provider_from_persisted_dek() { - use rustfs_kms::config::KmsConfig; - use tempfile::TempDir; - let _guard = lock_sse_test_state().await; reset_sse_dek_provider(); - let manager = rustfs_kms::init_global_kms_service_manager(); - let key_dir = TempDir::new().expect("create KMS key directory"); - manager - .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("start test KMS service"); + let manager = configure_test_global_local_kms().await; let local_master_key = [7u8; 32]; let local_provider = TestSseDekProvider::new_with_key(local_master_key); @@ -4348,9 +4646,6 @@ mod tests { /// the local-provider cache. #[tokio::test] async fn test_kms_envelope_never_routes_to_cached_local_provider() { - use rustfs_kms::config::KmsConfig; - use tempfile::TempDir; - let _guard = lock_sse_test_state().await; reset_sse_dek_provider(); @@ -4362,12 +4657,7 @@ mod tests { .expect("write local provider into local cache") = Some(Arc::new(TestSseDekProvider::new_with_key(local_master_key))); // 2. Start a KMS service (dynamic enable). - let manager = rustfs_kms::init_global_kms_service_manager(); - let key_dir = TempDir::new().expect("create KMS key directory"); - manager - .reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults()) - .await - .expect("start test KMS service"); + let manager = configure_test_global_local_kms().await; // 3. Construct a KMS JSON envelope — the persisted format of a KMS-wrapped DEK. // is_data_key_envelope() will return true for this payload. @@ -4458,7 +4748,7 @@ mod tests { use rustfs_kms::config::KmsConfig; let _guard = lock_sse_test_state().await; - let manager = rustfs_kms::init_global_kms_service_manager(); + let manager = Arc::new(rustfs_kms::KmsServiceManager::new()); manager .reconfigure(KmsConfig::static_kms("first-key".to_string(), BASE64_STANDARD.encode([0x11; 32]))) @@ -4692,6 +4982,15 @@ mod tests { ); } + #[test] + fn test_map_get_object_reader_error_preserves_typed_service_unavailable() { + let resolution_error = + super::EncryptionResolutionError::new(EncryptionResolutionErrorKind::ServiceUnavailable, "KMS unavailable"); + let err = map_get_object_reader_error(StorageError::other(resolution_error)); + assert_eq!(err.code, S3ErrorCode::ServiceUnavailable); + assert_eq!(err.message, "KMS unavailable"); + } + #[test] fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() { let err = map_get_object_reader_error(StorageError::other("plain io failure")); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 5cbf1554f..de8c9d9c3 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -509,12 +509,21 @@ pub(crate) mod ecstore_object { #[cfg(test)] pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource; pub(crate) use rustfs_ecstore::api::object::{ - GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len, - lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook, - unregister_get_object_body_cache_hook, unregister_object_mutation_hook, + EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, + ObjectEncryptionResolver, ObjectMutationHook, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, + get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, + register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook, }; } +#[cfg(all(test, feature = "rio-v2"))] +pub(crate) mod ecstore_test_support { + pub(crate) use rustfs_ecstore::api::bitrot::create_bitrot_reader; + pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk}; + pub(crate) use rustfs_ecstore::api::erasure::Erasure; + pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions}; +} + pub(crate) mod ecstore_set_disk { pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class}; } @@ -945,13 +954,21 @@ pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Res /// The process-level bootstrap instance context that single-instance startup /// threads through the storage foundation (Phase 5 follow-up, backlog#1052). pub(crate) fn bootstrap_instance_ctx() -> Arc { - ecstore_runtime::bootstrap_ctx() + let context = ecstore_runtime::bootstrap_ctx(); + configure_object_encryption_resolver(&context); + context } /// Construct a fresh per-server instance context (backlog#1052 S5): a second /// embedded server owns its own erasure/region/endpoint/deployment id cells. pub(crate) fn new_instance_ctx() -> Arc { - Arc::new(InstanceContext::new()) + let context = Arc::new(InstanceContext::new()); + configure_object_encryption_resolver(&context); + context +} + +fn configure_object_encryption_resolver(context: &InstanceContext) { + let _ = context.set_object_encryption_resolver(Arc::new(super::sse::SseObjectEncryptionResolver)); } pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) { @@ -1746,7 +1763,7 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc&2 exit 1