gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 15:46:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(sse): decouple ecstore and harden KMS lifecycle (#5435)

Browse Source 
* refactor(sse): decouple encryption from ecstore

* feat(kms): enhance KMS service manager with runtime state and persistence support

* feat(kms): add local key export functionality for SSE-S3 migration tests

* fix(kms): keep local key export narrowly scoped

* fix(sse): validate copy source customer algorithm

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
唐小鸭
2026-07-30 12:39:25 +08:00
committed by GitHub
parent 6e6b38ad8e
commit ad7663afd1
30 changed files with 1486 additions and 1951 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/Cargo.toml
-3
View File
@@ -57,7 +57,6 @@ rustfs-policy.workspace = true
rustfs-protos.workspace = true
rustfs-replication.workspace = true
rustfs-lifecycle.workspace = true
rustfs-kms.workspace = true
rustfs-s3-types = { workspace = true }
rustfs-data-usage.workspace = true
rustfs-object-capacity.workspace = true
@@ -125,8 +124,6 @@ libc.workspace = true
rustix = { workspace = true, features = ["process", "fs"] }
rustfs-madmin.workspace = true
reqwest = { workspace = true }
aes-gcm = { workspace = true, features = ["rand_core"] }
chacha20poly1305.workspace = true
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
urlencoding = { workspace = true }
smallvec = { workspace = true, features = ["serde"] }
crates/ecstore/src/api/mod.rs
+6 -4
View File
@@ -391,10 +391,12 @@ pub mod notification {
pub mod object {
pub use crate::object_api::{
BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo,
ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode,
ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
unregister_object_mutation_hook,
};
pub use crate::store::PreparedGetObjectReader;
}
crates/ecstore/src/client/utils.rs
+1 -15
View File
@@ -46,16 +46,6 @@ lazy_static! {
m.insert("x-amz-replication-status".to_string(), true);
m
};
static ref SSE_HEADERS: HashMap<String, bool> = {
let mut m = HashMap::new();
m.insert("x-amz-server-side-encryption".to_string(), true);
m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
m.insert("x-amz-server-side-encryption-context".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
m
};
}
pub fn is_standard_query_value(qs_key: &str) -> bool {
@@ -70,16 +60,12 @@ pub fn is_standard_header(header_key: &str) -> bool {
*SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_sse_header(header_key: &str) -> bool {
*SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
}
pub fn is_amz_header(header_key: &str) -> bool {
let key = header_key.to_lowercase();
key.starts_with("x-amz-meta-")
|| key.starts_with("x-amz-grant-")
|| key == "x-amz-acl"
|| is_sse_header(header_key)
|| rustfs_utils::http::is_sse_header(header_key)
|| key.starts_with("x-amz-checksum-")
}
crates/ecstore/src/object_api/encryption.rs
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use http::{HeaderMap, HeaderValue};
use std::collections::HashMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadEncryptionMode {
Direct { base_nonce: [u8; 12] },
Object,
}
pub struct ReadEncryptionMaterial {
pub key_bytes: [u8; 32],
pub mode: ReadEncryptionMode,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EncryptionResolutionErrorKind {
InvalidRequest,
InvalidMetadata,
ServiceUnavailable,
DecryptionFailed,
}
#[derive(Debug)]
pub struct EncryptionResolutionError {
kind: EncryptionResolutionErrorKind,
message: String,
}
impl EncryptionResolutionError {
pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
}
}
pub fn kind(&self) -> EncryptionResolutionErrorKind {
self.kind
}
}
impl Display for EncryptionResolutionError {
fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl Error for EncryptionResolutionError {}
pub struct ReadEncryptionRequest<'a> {
pub bucket: &'a str,
pub object: &'a str,
pub metadata: &'a HashMap<String, String>,
pub headers: &'a HeaderMap<HeaderValue>,
}
#[async_trait]
pub trait ObjectEncryptionResolver: Send + Sync {
async fn resolve_read_material(
&self,
request: ReadEncryptionRequest<'_>,
) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError>;
}
crates/ecstore/src/object_api/mod.rs
+5
View File
@@ -84,6 +84,7 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool {
}
mod body_cache_hook;
mod encryption;
mod hook_slot;
mod object_mutation_hook;
mod readers;
@@ -98,6 +99,10 @@ pub use body_cache_hook::{
pub(crate) use body_cache_hook::{
get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
};
pub use encryption::{
EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
ReadEncryptionMode, ReadEncryptionRequest,
};
pub(crate) use object_mutation_hook::notify_object_mutation;
pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
pub use readers::*;
crates/ecstore/src/object_api/readers.rs
+137 -1509
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/object_api/types.rs
+17 -46
View File
@@ -273,29 +273,9 @@ impl ObjectInfo {
}
pub fn is_encrypted(&self) -> bool {
// Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
self.user_defined.keys().any(|key| {
let lower = key.to_ascii_lowercase();
lower.starts_with("x-minio-encryption-")
|| lower.starts_with("x-minio-internal-server-side-encryption-")
|| matches!(
lower.as_str(),
"x-minio-internal-encrypted-multipart"
| "x-rustfs-encryption-key"
| "x-rustfs-encryption-algorithm"
| "x-rustfs-encryption-iv"
| "x-rustfs-encryption-key-id"
| "x-rustfs-encryption-context"
| "x-rustfs-encryption-tag"
| "x-amz-server-side-encryption-aws-kms-key-id"
| SSEC_ALGORITHM_HEADER
| SSEC_KEY_HEADER
| SSEC_KEY_MD5_HEADER
| "x-amz-server-side-encryption"
)
})
self.user_defined
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
}
/// Maximum inline size for non-versioned objects (128 KiB).
@@ -339,26 +319,7 @@ impl ObjectInfo {
}
pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
if let Some(size_str) = self
.user_defined
.get("x-rustfs-encryption-original-size")
.map(String::as_str)
.or_else(|| {
self.user_defined
.get("x-amz-server-side-encryption-customer-original-size")
.map(String::as_str)
})
.or(actual_size.as_deref())
&& !size_str.is_empty()
{
let size = size_str
.parse::<i64>()
.map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?;
return Ok(Some(size));
}
Ok(None)
rustfs_utils::http::get_object_encryption_original_size(&self.user_defined)
}
pub fn decrypted_size(&self) -> std::io::Result<i64> {
@@ -388,9 +349,6 @@ impl ObjectInfo {
return Ok(actual_size);
}
// Check if object is encrypted
// Managed SSE stores original size in x-rustfs-encryption-original-size metadata
// SSE-C stores original size in x-amz-server-side-encryption-customer-original-size
if let Some(size) = self.encryption_original_size()? {
return Ok(size);
}
@@ -881,6 +839,19 @@ mod tests {
assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back");
}
#[test]
fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() {
let object = ObjectInfo {
user_defined: Arc::new(HashMap::from([(
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(),
"sealed".to_string(),
)])),
..Default::default()
};
assert!(object.is_encrypted());
}
#[test]
fn versions_after_marker_handles_null_version_marker() {
let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap();
crates/ecstore/src/runtime/instance.rs
+17
View File
@@ -46,6 +46,7 @@ use crate::bucket::metadata_sys::BucketMetadataSys;
use crate::bucket::replication::{DynReplicationPool, ReplicationStats};
use crate::disk::DiskStore;
use crate::layout::endpoints::{EndpointServerPools, SetupType};
use crate::object_api::ObjectEncryptionResolver;
use crate::services::event_notification::EventNotifier;
use crate::services::tier::tier::TierConfigMgr;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
@@ -159,6 +160,8 @@ pub struct InstanceContext {
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
/// Replaces the process-global cancel-token static.
background_cancel_token: OnceLock<CancellationToken>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
#[cfg(test)]
@@ -197,6 +200,7 @@ impl InstanceContext {
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
bucket_metadata_sys: std::sync::Mutex::new(None),
background_cancel_token: OnceLock::new(),
object_encryption_resolver: OnceLock::new(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
#[cfg(test)]
@@ -209,6 +213,19 @@ impl InstanceContext {
self.lock_manager.clone()
}
/// Install the application-owned object-encryption resolver once.
pub fn set_object_encryption_resolver(
&self,
resolver: Arc<dyn ObjectEncryptionResolver>,
) -> Result<(), Arc<dyn ObjectEncryptionResolver>> {
self.object_encryption_resolver.set(resolver)
}
/// Return the configured object-encryption resolver, if startup installed one.
pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> {
self.object_encryption_resolver.get().map(Arc::as_ref)
}
/// Set this instance's S3 region.
///
/// Write-once: panics on a second write, preserving the startup fail-fast
crates/ecstore/src/runtime/sources.rs
-5
View File
@@ -46,7 +46,6 @@ use crate::{
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
use rustfs_lock::client::LockClient;
use s3s::dto::BucketLifecycleConfiguration;
use s3s::region::Region;
@@ -105,10 +104,6 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_
global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
}
pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
get_global_encryption_service().await
}
pub fn object_store_handle() -> Option<Arc<ECStore>> {
resolve_object_store_handle()
}
crates/ecstore/src/set_disk/metadata.rs
+1 -3
View File
@@ -505,9 +505,7 @@ impl SetDisks {
}
fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool {
meta.metadata
.keys()
.any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name))
meta.metadata.keys().any(|name| http::is_object_encryption_marker(name))
}
fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {
crates/ecstore/src/set_disk/mod.rs
+6 -7
View File
@@ -147,15 +147,17 @@ use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_s3_types::EventName;
#[cfg(test)]
use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
};
use rustfs_utils::http::{
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str,
get_header_map, get_str, insert_str, is_encryption_metadata_key, remove_header_map,
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker,
remove_header_map,
};
use rustfs_utils::{
HashAlgorithm,
@@ -670,10 +672,7 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, S
}
fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
metadata.keys().any(|key| is_encryption_metadata_key(key))
|| metadata.contains_key(SSEC_ALGORITHM_HEADER)
|| metadata.contains_key(SSEC_KEY_HEADER)
|| metadata.contains_key(SSEC_KEY_MD5_HEADER)
metadata.keys().any(|key| is_object_encryption_marker(key))
}
/// Per-set memoized capacity dirty scope.
crates/ecstore/src/set_disk/ops/object.rs
+71 -2
View File
@@ -42,6 +42,7 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
use crate::store::ECStore;
use futures::FutureExt as _;
use http::HeaderValue;
use std::future::Future;
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
@@ -49,6 +50,17 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Er
.map_err(Error::from)
}
async fn get_object_reader_with_context(
ctx: &InstanceContext,
reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
range: Option<HTTPRangeSpec>,
object_info: &ObjectInfo,
opts: &ObjectOptions,
headers: &HeaderMap<HeaderValue>,
) -> Result<(GetObjectReader, usize, i64)> {
GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await
}
/// Length of the full plaintext body when — and only when — this read's output
/// is exactly the object's complete plaintext, so the app-layer body cache may
/// serve it in place of the erasure read.
@@ -713,7 +725,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
size_bucket,
);
record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
let (mut reader, _offset, _length) =
get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?;
// Carry the hook probe result so the app layer skips its
// now-redundant lookup on the streaming miss path (ODC-16).
reader.body_source = body_source;
@@ -745,7 +758,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?;
let (mut reader, offset, length) =
get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?;
// Carry the hook probe result so the app layer skips its now-redundant
// lookup on the streaming miss path (ODC-16).
reader.body_source = body_source;
@@ -4536,6 +4550,61 @@ mod erasure_construction_tests {
}
}
#[cfg(test)]
mod object_encryption_resolver_wiring_tests {
use super::*;
use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest};
use std::io::Cursor;
use std::sync::atomic::{AtomicUsize, Ordering};
struct CountingResolver {
calls: AtomicUsize,
}
#[async_trait::async_trait]
impl ObjectEncryptionResolver for CountingResolver {
async fn resolve_read_material(
&self,
_request: ReadEncryptionRequest<'_>,
) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
self.calls.fetch_add(1, Ordering::Relaxed);
Ok(None)
}
}
#[tokio::test]
async fn get_object_reader_forwards_instance_resolver() {
let resolver = Arc::new(CountingResolver {
calls: AtomicUsize::new(0),
});
let ctx = InstanceContext::new();
assert!(
ctx.set_object_encryption_resolver(resolver.clone()).is_ok(),
"fresh context should accept resolver"
);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 1,
user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])),
..Default::default()
};
let result = get_object_reader_with_context(
&ctx,
Box::new(Cursor::new(Vec::<u8>::new())),
None,
&object_info,
&ObjectOptions::default(),
&HeaderMap::new(),
)
.await;
assert!(result.is_err(), "resolver returning no material must fail closed");
assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
}
}
#[cfg(test)]
pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
//! Shared hermetic `SetDisks` construction for the ops tests below: the
crates/ecstore/tests/README.md
+2 -2
View File
@@ -2,7 +2,7 @@
## MinIO-generated encrypted fixtures
`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`.\rustfs\scripts\minio_fixture_lab\lab.py`.
It currently covers multipart fixtures for:
@@ -20,5 +20,5 @@ Example:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
```
crates/ecstore/tests/minio_generated_read_test.rs
-299
View File
@@ -1,299 +0,0 @@
#![cfg(feature = "rio-v2")]
use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
mod storage_api;
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use storage_api::minio_generated_read::{
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
};
use temp_env::async_with_vars;
use tokio::io::{AsyncReadExt, AsyncWrite};
#[derive(Debug, Deserialize)]
struct ManifestRecord {
bucket: String,
object: String,
backend_files: Vec<String>,
}
#[derive(Default)]
struct VecAsyncWriter {
bytes: Vec<u8>,
}
impl AsyncWrite for VecAsyncWriter {
fn poll_write(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
self.bytes.extend_from_slice(buf);
std::task::Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
}
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
}
fn case_dir(case_id: &str) -> PathBuf {
fixture_root().join("cases").join(case_id)
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
}
fn require_fixture_case(case_id: &str) -> PathBuf {
let path = case_dir(case_id);
assert!(
path.is_dir(),
"fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
path.display()
);
path
}
fn read_plaintext_sha256(case_dir: &Path) -> String {
fs::read_to_string(case_dir.join("plaintext.sha256"))
.unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
.trim()
.to_string()
}
fn minio_static_kms_key_b64() -> String {
std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
.unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
}
fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
let expected = format!("disk1/{}/{}/xl.meta", manifest.bucket, manifest.object);
let relative = manifest
.backend_files
.iter()
.find(|entry| entry.as_str() == expected)
.unwrap_or_else(|| panic!("object xl.meta missing from manifest backend_files: {expected}"));
case_dir.join("backend").join(relative)
}
fn load_file_info(case_dir: &Path, manifest: &ManifestRecord) -> FileInfo {
let xl_meta_path = object_xl_meta_path(case_dir, manifest);
let xl_meta = fs::read(&xl_meta_path).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta_path.display()));
get_file_info(
&xl_meta,
&manifest.bucket,
&manifest.object,
"",
FileInfoOpts {
data: true,
include_free_versions: true,
},
)
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta_path.display()))
}
fn load_object_info(file_info: &FileInfo, manifest: &ManifestRecord) -> ObjectInfo {
ObjectInfo::from_file_info(file_info, &manifest.bucket, &manifest.object, false)
}
fn sha256_hex(bytes: &[u8]) -> String {
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
}
async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String) {
let case_dir = require_fixture_case(case_id);
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
let expected_sha256 = read_plaintext_sha256(&case_dir);
let file_info = load_file_info(&case_dir, &manifest);
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
let object_info = load_object_info(&file_info, &manifest);
(object_info, encrypted, expected_sha256)
}
async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms_key_b64: String) -> Result<Vec<u8>, String> {
let object_size = object_info.size;
async_with_vars(
[
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
],
async move {
let (mut reader, offset, length) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&http::HeaderMap::new(),
)
.await
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
if offset != 0 || length != object_size {
return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}"));
}
let mut plaintext = Vec::new();
reader
.read_to_end(&mut plaintext)
.await
.map_err(|err| format!("read plaintext from MinIO raw fixture: {err}"))?;
Ok(plaintext)
},
)
.await
}
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
let mut disks = Vec::with_capacity(file_info.erasure.distribution.len());
for disk_number in 1..=file_info.erasure.distribution.len() {
let disk_root = case_dir.join("backend").join(format!("disk{disk_number}"));
let disk_root_str = disk_root
.to_str()
.unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_number - 1);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.unwrap_or_else(|err| panic!("open fixture disk {disk_number}: {err}"));
disks.push(disk);
}
let mut disk_order = vec![None; disks.len()];
for (idx, disk) in disks.iter().enumerate() {
let block_index = file_info.erasure.distribution[idx];
disk_order[block_index - 1] = Some(disk);
}
let data_dir = file_info
.data_dir
.as_ref()
.unwrap_or_else(|| panic!("fixture {} is missing data_dir", manifest.object));
let mut encrypted = Vec::new();
for part in &file_info.parts {
let checksum_info = file_info.erasure.get_checksum_info(part.number);
let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
let shard_read_len = file_info.erasure.shard_file_size(part.size as i64);
let mut readers = Vec::with_capacity(disks.len());
for (idx, disk) in disk_order.iter().enumerate() {
let reader = create_bitrot_reader(
None,
*disk,
&manifest.bucket,
&path,
0,
shard_read_len as usize,
file_info.erasure.shard_size(),
checksum_info.algorithm.clone(),
false,
false,
)
.await
.unwrap_or_else(|err| panic!("create bitrot reader for disk{} {path}: {err:?}", idx + 1));
readers.push(reader);
}
let erasure = Erasure::new(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
);
let mut writer = VecAsyncWriter::default();
let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
if let Some(err) = err {
panic!("decode erasure shards for {path}: {err}");
}
assert_eq!(written, part.size, "decoded part size should match xl.meta part size");
encrypted.extend_from_slice(&writer.bytes);
}
for disk in disks {
disk.close().await.expect("close fixture disk");
}
encrypted
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_s3_multipart_fixture() {
assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_kms_multipart_fixture() {
assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
let (object_info, encrypted, _) = load_fixture_reader_input("sse-s3-multipart-8m").await;
let wrong_key_b64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
let result = read_fixture_plaintext(encrypted, object_info, wrong_key_b64).await;
assert!(result.is_err(), "wrong KMS key must fail closed");
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {
let (object_info, mut encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await;
encrypted.truncate(encrypted.len() / 2);
let result = read_fixture_plaintext(encrypted, object_info, minio_static_kms_key_b64()).await;
if let Ok(plaintext) = result {
assert_ne!(
sha256_hex(&plaintext),
expected_sha256,
"truncated ciphertext must not restore the original plaintext"
);
}
}
async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
// `ObjectInfo.size` is the on-disk size. For SSE objects that is the
// DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is
// deliberately larger than the logical object size. The size a client sees
// (and what MinIO records via `x-*-internal-actual-size`) comes from
// `decrypted_size()`/`get_actual_size()`, so assert against that — the raw
// `size` field would never equal the plaintext length for encrypted objects.
let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata");
let kms_key_b64 = minio_static_kms_key_b64();
let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
.await
.expect("fixture must restore with the configured KMS key");
assert_eq!(decrypted_size, expected_size);
assert_eq!(plaintext.len(), expected_size as usize);
assert_eq!(sha256_hex(&plaintext), expected_sha256);
}
crates/kms/AGENTS.md
+22
View File
@@ -25,3 +25,25 @@ For local KMS end-to-end tests, keep proxy bypass settings:
NO_PROXY=127.0.0.1,localhost HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \
cargo test --package e2e_test test_local_kms_end_to_end -- --nocapture --test-threads=1
```
## Local Key Export for SSE-S3 Migration Tests
Use the read-only `local_kms_key_decrypt` example to export an AES-256 Local
KMS key as the base64 value expected by `RUSTFS_SSE_S3_MASTER_KEY`:
```bash
export RUSTFS_KMS_LOCAL_MASTER_KEY='<local-kms-at-rest-master-key>'
export RUSTFS_SSE_S3_MASTER_KEY="$(
cargo run -q -p rustfs-kms --example local_kms_key_decrypt -- \
/absolute/path/to/<key-id>.key
)"
```
For a `plaintext-dev-only` Local KMS key file,
`RUSTFS_KMS_LOCAL_MASTER_KEY` is not required.
The example writes only the base64-encoded 32-byte key to stdout. Diagnostics
go to stderr. Never paste its output into logs, shell history, issue comments,
or committed configuration. The export path must remain read-only and must
reuse `LocalKmsClient` decoding so current Argon2id and legacy key-file
compatibility stay aligned with the backend.
crates/kms/examples/local_kms_key_decrypt.rs
+112
View File
@@ -0,0 +1,112 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
use rustfs_kms::{LocalConfig, backends::local::LocalKmsClient};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use zeroize::Zeroizing;
const LOCAL_KMS_MASTER_KEY_ENV: &str = "RUSTFS_KMS_LOCAL_MASTER_KEY";
fn usage(program: &str) -> String {
format!(
"Usage: {program} <local-kms-key-file>\n\
Reads {LOCAL_KMS_MASTER_KEY_ENV} when the key file is encrypted.\n\
Writes only the base64-encoded 32-byte key to stdout."
)
}
fn resolve_key_file(path: &Path) -> Result<(PathBuf, String), String> {
let canonical = std::fs::canonicalize(path).map_err(|error| format!("cannot open Local KMS key file: {error}"))?;
if canonical.extension().and_then(|extension| extension.to_str()) != Some("key") {
return Err("Local KMS key file must have a .key extension".to_string());
}
let key_dir = canonical
.parent()
.ok_or_else(|| "Local KMS key file must have a parent directory".to_string())?
.to_path_buf();
let key_id = canonical
.file_stem()
.and_then(|stem| stem.to_str())
.filter(|stem| !stem.is_empty())
.ok_or_else(|| "Local KMS key file name must contain a valid UTF-8 key ID".to_string())?
.to_string();
Ok((key_dir, key_id))
}
async fn run() -> Result<(), String> {
let mut args = std::env::args();
let program = args.next().unwrap_or_else(|| "local_kms_key_decrypt".to_string());
let Some(key_file) = args.next() else {
return Err(usage(&program));
};
if args.next().is_some() {
return Err(usage(&program));
}
let (key_dir, key_id) = resolve_key_file(Path::new(&key_file))?;
let master_key = std::env::var(LOCAL_KMS_MASTER_KEY_ENV).ok().filter(|value| !value.is_empty());
let client = LocalKmsClient::new_for_key_export(LocalConfig {
key_dir,
master_key,
file_permissions: Some(0o600),
})
.await
.map_err(|error| error.to_string())?;
let key_material = client
.decrypt_key_material_for_export(&key_id)
.await
.map_err(|error| error.to_string())?;
let encoded = Zeroizing::new(BASE64_STANDARD.encode(key_material.as_ref()));
let mut stdout = io::stdout().lock();
writeln!(stdout, "{}", encoded.as_str()).map_err(|error| format!("failed to write decrypted key: {error}"))
}
#[tokio::main]
async fn main() {
if let Err(error) = run().await {
let _ = writeln!(io::stderr().lock(), "local_kms_key_decrypt: {error}");
std::process::exit(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resolve_key_file_extracts_directory_and_key_id() {
let directory = tempfile::tempdir().expect("create temporary directory");
let key_file = directory.path().join("migration-key.key");
std::fs::write(&key_file, b"{}").expect("create key file");
let (key_dir, key_id) = resolve_key_file(&key_file).expect("resolve key file");
assert_eq!(key_dir, directory.path().canonicalize().expect("canonical directory"));
assert_eq!(key_id, "migration-key");
}
#[test]
fn resolve_key_file_rejects_non_key_extension() {
let directory = tempfile::tempdir().expect("create temporary directory");
let key_file = directory.path().join("migration-key.json");
std::fs::write(&key_file, b"{}").expect("create key file");
let error = resolve_key_file(&key_file).expect_err("non-key file must be rejected");
assert!(error.contains(".key"));
}
}
crates/kms/src/backends/local.rs
+100
View File
@@ -36,6 +36,7 @@ use std::path::{Component, Path, PathBuf};
use std::time::Duration;
use tokio::fs;
use tracing::{debug, warn};
use zeroize::Zeroizing;
/// Reject key identifiers that would not name a single file directly inside the key
/// directory.
@@ -146,6 +147,45 @@ impl LocalKmsClient {
Ok(client)
}
/// Open a Local KMS key directory without creating or modifying any files.
///
/// This constructor is restricted to explicit key-export tooling. Normal
/// backend operation must use [`Self::new`].
pub async fn new_for_key_export(config: LocalConfig) -> Result<Self> {
if !fs::try_exists(&config.key_dir).await? {
return Err(KmsError::configuration_error("Local KMS key directory does not exist"));
}
let (master_cipher, legacy_master_cipher) = if let Some(ref master_key) = config.master_key {
let legacy_key = Self::derive_legacy_master_key(master_key)?;
let legacy_master_cipher = Aes256Gcm::new(&legacy_key);
let salt_path = Self::master_key_salt_path(&config);
let master_cipher = if fs::try_exists(&salt_path).await? {
let salt = fs::read(&salt_path).await?;
let salt: [u8; LOCAL_KMS_MASTER_KEY_SALT_LEN] = salt.try_into().map_err(|_| {
KmsError::configuration_error(format!(
"Local KMS master key salt at {} must be exactly {} bytes",
salt_path.display(),
LOCAL_KMS_MASTER_KEY_SALT_LEN
))
})?;
Aes256Gcm::new(&Self::derive_master_key(master_key, &salt)?)
} else {
Aes256Gcm::new(&legacy_key)
};
(Some(master_cipher), Some(legacy_master_cipher))
} else {
(None, None)
};
Ok(Self {
config,
master_cipher,
legacy_master_cipher,
dek_crypto: AesDekCrypto::new(),
})
}
/// Derive a 256-bit key from the master key string using a persistent Argon2id salt.
fn derive_master_key(master_key: &str, salt: &[u8]) -> Result<Key<Aes256Gcm>> {
let params = Params::new(
@@ -435,6 +475,20 @@ impl LocalKmsClient {
Ok(key_material)
}
/// Decrypt an AES-256 Local KMS key for explicit migration tooling.
///
/// The returned buffer is zeroized on drop. Callers must treat the value as
/// plaintext key material and avoid logging or persisting it.
pub async fn decrypt_key_material_for_export(&self, key_id: &str) -> Result<Zeroizing<[u8; 32]>> {
let (stored_key, key_material) = self.decode_stored_key(key_id).await?;
if stored_key.algorithm != "AES_256" {
return Err(KmsError::unsupported_algorithm(stored_key.algorithm));
}
let actual = key_material.len();
let key_material = key_material.try_into().map_err(|_| KmsError::invalid_key_size(32, actual))?;
Ok(Zeroizing::new(key_material))
}
async fn validate_existing_keys(&self) -> Result<()> {
let mut entries = fs::read_dir(&self.config.key_dir).await?;
while let Some(entry) = entries.next_entry().await? {
@@ -1208,6 +1262,52 @@ mod tests {
assert!(matches!(wrong_master_error, KmsError::CryptographicError { .. }));
}
#[tokio::test]
async fn key_export_uses_existing_local_decryption_path_without_writing_files() {
let (client, _temp_dir) = create_test_client().await;
let key_id = "export-key";
client
.create_key(key_id, "AES_256", None)
.await
.expect("create encrypted key");
let expected = client.get_key_material(key_id).await.expect("load expected key material");
let salt_path = LocalKmsClient::master_key_salt_path(&client.config);
let salt_before = fs::read(&salt_path).await.expect("read existing salt");
let export_client = LocalKmsClient::new_for_key_export(client.config.clone())
.await
.expect("open read-only export client");
let exported = export_client
.decrypt_key_material_for_export(key_id)
.await
.expect("decrypt key for export");
assert_eq!(exported.as_ref(), expected.as_slice());
assert_eq!(fs::read(&salt_path).await.expect("read unchanged salt"), salt_before);
}
#[tokio::test]
async fn key_export_accepts_plaintext_dev_only_key_without_master_key() {
let (client, _temp_dir) = create_dev_mode_client().await;
let key_id = "plaintext-export-key";
client
.create_key(key_id, "AES_256", None)
.await
.expect("create plaintext-dev-only key");
let expected = client.get_key_material(key_id).await.expect("load expected key material");
let export_client = LocalKmsClient::new_for_key_export(client.config.clone())
.await
.expect("open read-only export client");
let exported = export_client
.decrypt_key_material_for_export(key_id)
.await
.expect("export plaintext-dev-only key");
assert_eq!(exported.as_ref(), expected.as_slice());
assert!(!LocalKmsClient::master_key_salt_path(&client.config).exists());
}
#[tokio::test]
async fn test_plaintext_dev_only_storage_is_explicit_and_loadable() {
let (client, _temp_dir) = create_dev_mode_client().await;
crates/kms/src/lib.rs
+1 -1
View File
@@ -89,7 +89,7 @@ pub use error::{KmsError, KmsUnavailableError, Result};
pub use manager::KmsManager;
pub use service::{DataKey, ObjectEncryptionService};
pub use service_manager::{
KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager,
KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,
init_global_kms_service_manager,
};
pub use types::*;
crates/kms/src/service_manager.rs
+311 -95
View File
@@ -21,12 +21,13 @@ use crate::manager::KmsManager;
use crate::service::ObjectEncryptionService;
use arc_swap::ArcSwap;
use sha2::{Digest, Sha256};
use std::future::Future;
use std::sync::{
Arc, OnceLock,
atomic::{AtomicU64, Ordering},
};
use subtle::ConstantTimeEq;
use tokio::sync::{Mutex, RwLock};
use tokio::sync::Mutex;
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_KMS: &str = "kms";
@@ -93,6 +94,13 @@ pub enum KmsServiceStatus {
Error(String),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KmsStartOutcome {
Started,
Restarted,
AlreadyRunning,
}
/// Service version information for zero-downtime reconfiguration
#[derive(Clone)]
struct ServiceVersion {
@@ -104,16 +112,17 @@ struct ServiceVersion {
manager: Arc<KmsManager>,
}
#[derive(Clone)]
struct RuntimeState {
config: Option<KmsConfig>,
status: KmsServiceStatus,
current_service: Option<ServiceVersion>,
}
/// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration
pub struct KmsServiceManager {
/// Current service version (if running)
/// Uses ArcSwap for atomic, lock-free service switching
/// This allows instant atomic updates without blocking readers
current_service: ArcSwap<Option<ServiceVersion>>,
/// Current configuration
config: Arc<RwLock<Option<KmsConfig>>>,
/// Current status
status: Arc<RwLock<KmsServiceStatus>>,
/// Atomically published configuration, status, and current service.
state: ArcSwap<RuntimeState>,
/// Version counter (monotonically increasing)
version_counter: Arc<AtomicU64>,
/// Mutex to protect lifecycle operations (start, stop, reconfigure)
@@ -125,9 +134,11 @@ impl KmsServiceManager {
/// Create a new KMS service manager (not configured)
pub fn new() -> Self {
Self {
current_service: ArcSwap::from_pointee(None),
config: Arc::new(RwLock::new(None)),
status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)),
state: ArcSwap::from_pointee(RuntimeState {
config: None,
status: KmsServiceStatus::NotConfigured,
current_service: None,
}),
version_counter: Arc::new(AtomicU64::new(0)),
lifecycle_mutex: Arc::new(Mutex::new(())),
}
@@ -135,44 +146,67 @@ impl KmsServiceManager {
/// Get current service status
pub async fn get_status(&self) -> KmsServiceStatus {
self.status.read().await.clone()
self.state.load().status.clone()
}
/// Get current configuration (if any)
pub async fn get_config(&self) -> Option<KmsConfig> {
self.config.read().await.clone()
self.state.load().config.clone()
}
/// Get configuration for status and management responses without static key material.
pub async fn get_redacted_config(&self) -> Option<KmsConfig> {
let mut config = self.config.read().await.clone()?;
let mut config = self.state.load().config.clone()?;
Self::redact_config(&mut config);
Some(config)
}
/// Get status and redacted configuration from the same published snapshot.
pub async fn get_redacted_state(&self) -> (KmsServiceStatus, Option<KmsConfig>) {
let state = self.state.load();
let mut config = state.config.clone();
if let Some(config) = &mut config {
Self::redact_config(config);
}
(state.status.clone(), config)
}
fn redact_config(config: &mut KmsConfig) {
if let BackendConfig::Static(static_config) = &mut config.backend_config {
use zeroize::Zeroize;
static_config.secret_key.zeroize();
}
Some(config)
}
/// Configure KMS with new configuration
pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
let _guard = self.lifecycle_mutex.lock().await;
self.configure_with_persistence(new_config, || async { Ok(()) }).await
}
/// Configure KMS and publish the in-memory state only after persistence succeeds.
///
/// The persistence callback runs under the lifecycle lock and must not call
/// another lifecycle method on this manager.
pub async fn configure_with_persistence<Persist, PersistFuture>(&self, new_config: KmsConfig, persist: Persist) -> Result<()>
where
Persist: FnOnce() -> PersistFuture,
PersistFuture: Future<Output = Result<()>>,
{
new_config.validate()?;
{
let config = self.config.read().await;
validate_local_transition(config.as_ref(), &new_config)?;
}
// Update configuration
{
let mut config = self.config.write().await;
*config = Some(new_config.clone());
}
// Update status
{
let mut status = self.status.write().await;
*status = KmsServiceStatus::Configured;
let _guard = self.lifecycle_mutex.lock().await;
let current = self.state.load_full();
validate_local_transition(current.config.as_ref(), &new_config)?;
if current.current_service.is_some() {
return Err(KmsError::configuration_error(
"Cannot configure KMS while it is running; use reconfigure instead",
));
}
persist().await?;
self.state.store(Arc::new(RuntimeState {
config: Some(new_config),
status: KmsServiceStatus::Configured,
current_service: None,
}));
debug!(
event = EVENT_KMS_SERVICE_STATE,
@@ -190,19 +224,35 @@ impl KmsServiceManager {
self.start_internal().await
}
/// Start or restart KMS with the running-state decision serialized with the lifecycle action.
pub async fn start_or_restart(&self, force: bool) -> Result<KmsStartOutcome> {
let _guard = self.lifecycle_mutex.lock().await;
let running = self.state.load().current_service.is_some();
if running && !force {
return Ok(KmsStartOutcome::AlreadyRunning);
}
self.start_internal().await?;
Ok(if running {
KmsStartOutcome::Restarted
} else {
KmsStartOutcome::Started
})
}
/// Internal start implementation (called within lifecycle mutex)
async fn start_internal(&self) -> Result<()> {
let config = {
let config_guard = self.config.read().await;
match config_guard.as_ref() {
Some(config) => config.clone(),
None => {
let err_msg = "Cannot start KMS: no configuration provided";
error!("{}", err_msg);
let mut status = self.status.write().await;
*status = KmsServiceStatus::Error(err_msg.to_string());
return Err(KmsError::configuration_error(err_msg));
}
let state = self.state.load_full();
let config = match state.config.as_ref() {
Some(config) => config.clone(),
None => {
let err_msg = "Cannot start KMS: no configuration provided";
error!("{}", err_msg);
self.state.store(Arc::new(RuntimeState {
config: None,
status: KmsServiceStatus::Error(err_msg.to_string()),
current_service: None,
}));
return Err(KmsError::configuration_error(err_msg));
}
};
@@ -215,17 +265,9 @@ impl KmsServiceManager {
"KMS service starting"
);
match self.create_service_version(&config).await {
match self.create_healthy_service_version(&config).await {
Ok(service_version) => {
// Atomically update to new service version (lock-free, instant)
// ArcSwap::store() is a true atomic operation using CAS
self.current_service.store(Arc::new(Some(service_version)));
// Update status
{
let mut status = self.status.write().await;
*status = KmsServiceStatus::Running;
}
self.publish_running(config, service_version);
debug!(
event = EVENT_KMS_SERVICE_STATE,
@@ -239,13 +281,24 @@ impl KmsServiceManager {
Err(e) => {
let err_msg = format!("Failed to create KMS backend: {e}");
error!("{}", err_msg);
let mut status = self.status.write().await;
*status = KmsServiceStatus::Error(err_msg.clone());
if state.current_service.is_none() {
self.state.store(Arc::new(RuntimeState {
config: state.config.clone(),
status: KmsServiceStatus::Error(err_msg.clone()),
current_service: None,
}));
}
Err(KmsError::backend_error(&err_msg))
}
}
}
/// Replace the running service without exposing a stopped interval.
pub async fn restart(&self) -> Result<()> {
let _guard = self.lifecycle_mutex.lock().await;
self.start_internal().await
}
/// Stop KMS service
///
/// Note: This stops accepting new operations, but existing operations using
@@ -267,15 +320,16 @@ impl KmsServiceManager {
// Atomically clear current service version (lock-free, instant)
// Note: Existing Arc references will keep the service alive until operations complete
self.current_service.store(Arc::new(None));
// Update status (keep configuration)
{
let mut status = self.status.write().await;
if !matches!(*status, KmsServiceStatus::NotConfigured) {
*status = KmsServiceStatus::Configured;
}
}
let state = self.state.load_full();
self.state.store(Arc::new(RuntimeState {
config: state.config.clone(),
status: if state.config.is_some() {
KmsServiceStatus::Configured
} else {
KmsServiceStatus::NotConfigured
},
current_service: None,
}));
debug!(
event = EVENT_KMS_SERVICE_STATE,
@@ -298,6 +352,22 @@ impl KmsServiceManager {
/// This ensures zero downtime during reconfiguration, even for long-running
/// operations like encrypting large files.
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
self.reconfigure_with_persistence(new_config, || async { Ok(()) }).await
}
/// Reconfigure KMS after the candidate is healthy and persistence succeeds.
///
/// The persistence callback runs under the lifecycle lock and must not call
/// another lifecycle method on this manager.
pub async fn reconfigure_with_persistence<Persist, PersistFuture>(
&self,
new_config: KmsConfig,
persist: Persist,
) -> Result<()>
where
Persist: FnOnce() -> PersistFuture,
PersistFuture: Future<Output = Result<()>>,
{
let _guard = self.lifecycle_mutex.lock().await;
debug!(
@@ -308,33 +378,18 @@ impl KmsServiceManager {
"KMS service reconfiguring"
);
new_config.validate()?;
{
let config = self.config.read().await;
validate_local_transition(config.as_ref(), &new_config)?;
}
validate_local_transition(self.state.load().config.as_ref(), &new_config)?;
// Create new service version without stopping old one
// This allows existing operations to continue while new operations use new service
match self.create_service_version(&new_config).await {
match self.create_healthy_service_version(&new_config).await {
Ok(new_service_version) => {
// Get old version for logging (lock-free read)
let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
let old_version = self.state.load().current_service.as_ref().map(|sv| sv.version);
{
let mut config = self.config.write().await;
*config = Some(new_config);
}
persist().await?;
// Atomically switch to new service version (lock-free, instant CAS operation)
// This is a true atomic operation - no waiting for locks, instant switch
// Old service will be dropped when no more Arc references exist
self.current_service.store(Arc::new(Some(new_service_version.clone())));
// Update status
{
let mut status = self.status.write().await;
*status = KmsServiceStatus::Running;
}
self.publish_running(new_config, new_service_version.clone());
if let Some(old_ver) = old_version {
info!(
@@ -371,7 +426,7 @@ impl KmsServiceManager {
/// Returns the manager from the current service version.
/// Uses lock-free atomic load for optimal performance.
pub async fn get_manager(&self) -> Option<Arc<KmsManager>> {
self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone())
self.state.load().current_service.as_ref().map(|sv| sv.manager.clone())
}
/// Get encryption service (if running)
@@ -381,7 +436,7 @@ impl KmsServiceManager {
/// This ensures new operations always use the latest service version,
/// while existing operations continue using their Arc references.
pub async fn get_encryption_service(&self) -> Option<Arc<ObjectEncryptionService>> {
self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone())
self.state.load().current_service.as_ref().map(|sv| sv.service.clone())
}
/// Get current service version number
@@ -389,14 +444,16 @@ impl KmsServiceManager {
/// Useful for monitoring and debugging.
/// Uses lock-free atomic load.
pub async fn get_service_version(&self) -> Option<u64> {
self.current_service.load().as_ref().as_ref().map(|sv| sv.version)
self.state.load().current_service.as_ref().map(|sv| sv.version)
}
/// Health check for the KMS service
pub async fn health_check(&self) -> Result<bool> {
let manager = self.get_manager().await;
match manager {
Some(manager) => {
let checked_state = self.state.load_full();
match checked_state.current_service.as_ref() {
Some(service_version) => {
let manager = service_version.manager.clone();
let checked_version = service_version.version;
// Perform health check on the backend
match manager.health_check().await {
Ok(healthy) => {
@@ -407,9 +464,8 @@ impl KmsServiceManager {
}
Err(e) => {
error!("KMS health check error: {}", e);
// Update status to error
let mut status = self.status.write().await;
*status = KmsServiceStatus::Error(format!("Health check failed: {e}"));
let _guard = self.lifecycle_mutex.lock().await;
self.mark_health_error_if_current(checked_version, &e);
Err(e)
}
}
@@ -468,6 +524,33 @@ impl KmsServiceManager {
manager: kms_manager,
})
}
async fn create_healthy_service_version(&self, config: &KmsConfig) -> Result<ServiceVersion> {
let service_version = self.create_service_version(config).await?;
if !service_version.manager.health_check().await? {
return Err(KmsError::backend_error("KMS backend health check failed"));
}
Ok(service_version)
}
fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) {
self.state.store(Arc::new(RuntimeState {
config: Some(config),
status: KmsServiceStatus::Running,
current_service: Some(service_version),
}));
}
fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
let current = self.state.load_full();
if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
self.state.store(Arc::new(RuntimeState {
config: current.config.clone(),
status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
current_service: current.current_service.clone(),
}));
}
}
}
impl Default for KmsServiceManager {
@@ -500,6 +583,11 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
#[cfg(test)]
mod tests {
use super::*;
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
fn static_config(key_id: &str, fill: u8) -> KmsConfig {
KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
}
#[tokio::test]
async fn configure_rejects_insecure_development_defaults_before_state_update() {
@@ -517,8 +605,6 @@ mod tests {
#[tokio::test]
async fn redacted_config_omits_static_key_material() {
use base64::Engine as _;
let manager = KmsServiceManager::new();
let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
manager
@@ -533,6 +619,136 @@ mod tests {
assert!(static_config.secret_key.is_empty());
}
#[tokio::test]
async fn configure_persistence_failure_leaves_state_unchanged() {
let manager = KmsServiceManager::new();
let result = manager
.configure_with_persistence(static_config("key-a", 0x11), || async { Err(KmsError::backend_error("persist failed")) })
.await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
assert!(manager.get_config().await.is_none());
assert!(manager.get_encryption_service().await.is_none());
}
#[tokio::test]
async fn configure_rejects_running_service_without_changing_snapshot() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let version = manager.get_service_version().await;
let result = manager.configure(static_config("key-b", 0x22)).await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
assert_eq!(manager.get_service_version().await, version);
assert_eq!(
manager.get_config().await.and_then(|config| config.default_key_id),
Some("key-a".to_string())
);
}
#[tokio::test]
async fn reconfigure_persistence_failure_keeps_old_running_snapshot() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await;
let old_service = manager.get_encryption_service().await.expect("old service");
let result = manager
.reconfigure_with_persistence(static_config("key-b", 0x22), || async {
Err(KmsError::backend_error("persist failed"))
})
.await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
assert_eq!(manager.get_service_version().await, old_version);
assert_eq!(
manager.get_config().await.and_then(|config| config.default_key_id),
Some("key-a".to_string())
);
assert!(Arc::ptr_eq(
&old_service,
&manager.get_encryption_service().await.expect("old service remains")
));
}
#[tokio::test]
async fn reconfigure_candidate_failure_keeps_old_running_snapshot() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await;
let invalid_parent = tempfile::NamedTempFile::new().expect("temporary file");
let invalid_config = KmsConfig::local(invalid_parent.path().join("keys")).with_insecure_development_defaults();
let result = manager.reconfigure(invalid_config).await;
assert!(result.is_err());
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
assert_eq!(manager.get_service_version().await, old_version);
assert_eq!(
manager.get_config().await.and_then(|config| config.default_key_id),
Some("key-a".to_string())
);
}
#[tokio::test]
async fn restart_never_unpublishes_the_running_service() {
let manager = Arc::new(KmsServiceManager::new());
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await.expect("old version");
let restarting = {
let manager = manager.clone();
tokio::spawn(async move { manager.restart().await })
};
while !restarting.is_finished() {
assert!(manager.get_encryption_service().await.is_some());
tokio::task::yield_now().await;
}
restarting.await.expect("restart task").expect("restart");
assert!(manager.get_encryption_service().await.is_some());
assert!(manager.get_service_version().await.expect("new version") > old_version);
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
}
#[tokio::test]
async fn start_or_restart_decides_under_the_lifecycle_lock() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
assert_eq!(manager.start_or_restart(false).await.expect("initial start"), KmsStartOutcome::Started);
let first_version = manager.get_service_version().await.expect("first version");
assert_eq!(
manager.start_or_restart(false).await.expect("already running"),
KmsStartOutcome::AlreadyRunning
);
assert_eq!(manager.get_service_version().await, Some(first_version));
assert_eq!(manager.start_or_restart(true).await.expect("forced restart"), KmsStartOutcome::Restarted);
assert!(manager.get_service_version().await.expect("restarted version") > first_version);
}
#[tokio::test]
async fn stale_health_failure_cannot_poison_new_service_status() {
let manager = KmsServiceManager::new();
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
manager.start().await.expect("start");
let old_version = manager.get_service_version().await.expect("old version");
manager.restart().await.expect("restart");
manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
}
#[tokio::test]
async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
use crate::types::{CreateKeyRequest, KeyUsage};
crates/rio-v2/tests/minio_fixture_lab/README.md
+1 -1
View File
@@ -137,7 +137,7 @@ tests read):
./capture_via_docker.sh
RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
```
This is exactly what the nightly `minio-interop` GitHub Actions workflow runs
crates/utils/src/http/header_compat.rs
+73 -3
View File
@@ -25,6 +25,11 @@ const RUSTFS_PREFIX: &str = "x-rustfs-";
const MINIO_PREFIX: &str = "x-minio-";
const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-";
const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart";
const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = "x-rustfs-encryption-original-size";
const MINIO_ENCRYPTION_ORIGINAL_SIZE: &str = "x-minio-encryption-original-size";
const SSEC_ORIGINAL_SIZE: &str = "x-amz-server-side-encryption-customer-original-size";
// Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
@@ -40,11 +45,49 @@ pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request"
pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check";
pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc";
/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or
/// x-minio-encryption-*). Case-insensitive for metadata filtering.
/// Returns true if the key is object-encryption metadata understood by RustFS or MinIO.
/// Case-insensitive for metadata filtering.
pub fn is_encryption_metadata_key(key: &str) -> bool {
let lower = key.to_lowercase();
lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
lower.starts_with(RUSTFS_ENCRYPTION_PREFIX)
|| lower.starts_with(MINIO_ENCRYPTION_PREFIX)
|| lower.starts_with(MINIO_INTERNAL_ENCRYPTION_PREFIX)
|| lower == MINIO_INTERNAL_ENCRYPTED_MULTIPART
}
/// Returns true when a metadata key proves that object data is encrypted.
///
/// Original-size metadata alone is not proof: older plaintext objects can
/// retain that compatibility field after metadata migration.
pub fn is_object_encryption_marker(key: &str) -> bool {
(is_encryption_metadata_key(key)
&& !key.eq_ignore_ascii_case(RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
&& !key.eq_ignore_ascii_case(MINIO_ENCRYPTION_ORIGINAL_SIZE))
|| super::is_sse_header(key)
}
/// Reads the logical object size recorded by encryption metadata.
pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap<String, String>) -> std::io::Result<Option<i64>> {
let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE);
let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
.or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE))
.or(actual_size.as_deref());
let Some(size) = size.filter(|size| !size.is_empty()) else {
return Ok(None);
};
size.parse::<i64>()
.map(Some)
.map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}")))
}
fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap<String, String>, key: &str) -> Option<&'a str> {
metadata.get(key).map(String::as_str).or_else(|| {
metadata
.iter()
.find(|(candidate, _)| candidate.eq_ignore_ascii_case(key))
.map(|(_, value)| value.as_str())
})
}
fn rustfs_key(suffix: &str) -> String {
@@ -106,10 +149,37 @@ mod tests {
assert!(is_encryption_metadata_key("x-rustfs-encryption-iv"));
assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key"));
assert!(is_encryption_metadata_key("x-minio-encryption-iv"));
assert!(is_encryption_metadata_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
assert!(is_encryption_metadata_key("X-Minio-Internal-Encrypted-Multipart"));
assert!(!is_encryption_metadata_key("x-amz-meta-custom"));
assert!(!is_encryption_metadata_key("x-rustfs-internal-healing"));
}
#[test]
fn object_encryption_marker_excludes_size_only_metadata() {
assert!(!is_object_encryption_marker(RUSTFS_ENCRYPTION_ORIGINAL_SIZE));
assert!(is_object_encryption_marker("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
assert!(is_object_encryption_marker("x-amz-server-side-encryption"));
}
#[test]
fn object_encryption_original_size_is_case_insensitive() {
let metadata = std::collections::HashMap::from([(
"X-Amz-Server-Side-Encryption-Customer-Original-Size".to_string(),
"42".to_string(),
)]);
assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
}
#[test]
fn object_encryption_original_size_prefers_rustfs_metadata() {
let metadata = std::collections::HashMap::from([
(SSEC_ORIGINAL_SIZE.to_string(), "21".to_string()),
(RUSTFS_ENCRYPTION_ORIGINAL_SIZE.to_string(), "42".to_string()),
]);
assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
}
#[test]
fn test_get_header() {
let mut headers = HeaderMap::new();