feat(kms): enhance KMS service manager with runtime state and persistence support

Cargo.lock

@@ -9106,7 +9106,6 @@ dependencies = [
 name = "rustfs-ecstore"
 version = "1.0.0-beta.11"
 dependencies = [
- "aes-gcm",
  "arc-swap",
  "async-channel",
  "async-recursion",
@@ -9122,7 +9121,6 @@ dependencies = [
  "byteorder",
  "bytes",
  "bytesize",
- "chacha20poly1305",
  "chrono",
  "criterion",
  "enumset",
@@ -9175,7 +9173,6 @@ dependencies = [
  "rustfs-erasure-codec",
  "rustfs-filemeta",
  "rustfs-io-metrics",
- "rustfs-kms",
  "rustfs-lifecycle",
  "rustfs-lock",
  "rustfs-madmin",

crates/ecstore/Cargo.toml

@@ -57,7 +57,6 @@ rustfs-policy.workspace = true
 rustfs-protos.workspace = true
 rustfs-replication.workspace = true
 rustfs-lifecycle.workspace = true
-rustfs-kms.workspace = true
 rustfs-s3-types = { workspace = true }
 rustfs-data-usage.workspace = true
 rustfs-object-capacity.workspace = true
@@ -124,8 +123,6 @@ libc.workspace = true
 rustix = { workspace = true, features = ["process", "fs"] }
 rustfs-madmin.workspace = true
 reqwest = { workspace = true }
-aes-gcm = { workspace = true, features = ["rand_core"] }
-chacha20poly1305.workspace = true
 aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
 urlencoding = { workspace = true }
 smallvec = { workspace = true, features = ["serde"] }

crates/ecstore/src/api/mod.rs

@@ -381,10 +381,12 @@ pub mod notification {
 
 pub mod object {
     pub use crate::object_api::{
-        BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
-        GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
-        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
-        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
+        GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, ObjectEncryptionResolver, ObjectInfo,
+        ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode,
+        ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
+        register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
+        unregister_object_mutation_hook,
     };
     pub use crate::store::PreparedGetObjectReader;
 }

crates/ecstore/src/client/utils.rs

@@ -46,16 +46,6 @@ lazy_static! {
         m.insert("x-amz-replication-status".to_string(), true);
         m
     };
-    static ref SSE_HEADERS: HashMap<String, bool> = {
-        let mut m = HashMap::new();
-        m.insert("x-amz-server-side-encryption".to_string(), true);
-        m.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), true);
-        m.insert("x-amz-server-side-encryption-context".to_string(), true);
-        m.insert("x-amz-server-side-encryption-customer-algorithm".to_string(), true);
-        m.insert("x-amz-server-side-encryption-customer-key".to_string(), true);
-        m.insert("x-amz-server-side-encryption-customer-key-md5".to_string(), true);
-        m
-    };
 }
 
 pub fn is_standard_query_value(qs_key: &str) -> bool {
@@ -70,16 +60,12 @@ pub fn is_standard_header(header_key: &str) -> bool {
     *SUPPORTED_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
 }
 
-pub fn is_sse_header(header_key: &str) -> bool {
-    *SSE_HEADERS.get(&header_key.to_lowercase()).unwrap_or(&false)
-}
-
 pub fn is_amz_header(header_key: &str) -> bool {
     let key = header_key.to_lowercase();
     key.starts_with("x-amz-meta-")
         || key.starts_with("x-amz-grant-")
         || key == "x-amz-acl"
-        || is_sse_header(header_key)
+        || rustfs_utils::http::is_sse_header(header_key)
         || key.starts_with("x-amz-checksum-")
 }
 

crates/ecstore/src/object_api/encryption.rs

@@ -0,0 +1,80 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+use http::{HeaderMap, HeaderValue};
+use std::collections::HashMap;
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ReadEncryptionMode {
+    Direct { base_nonce: [u8; 12] },
+    Object,
+}
+
+pub struct ReadEncryptionMaterial {
+    pub key_bytes: [u8; 32],
+    pub mode: ReadEncryptionMode,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EncryptionResolutionErrorKind {
+    InvalidRequest,
+    InvalidMetadata,
+    ServiceUnavailable,
+    DecryptionFailed,
+}
+
+#[derive(Debug)]
+pub struct EncryptionResolutionError {
+    kind: EncryptionResolutionErrorKind,
+    message: String,
+}
+
+impl EncryptionResolutionError {
+    pub fn new(kind: EncryptionResolutionErrorKind, message: impl Into<String>) -> Self {
+        Self {
+            kind,
+            message: message.into(),
+        }
+    }
+
+    pub fn kind(&self) -> EncryptionResolutionErrorKind {
+        self.kind
+    }
+}
+
+impl Display for EncryptionResolutionError {
+    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(&self.message)
+    }
+}
+
+impl Error for EncryptionResolutionError {}
+
+pub struct ReadEncryptionRequest<'a> {
+    pub bucket: &'a str,
+    pub object: &'a str,
+    pub metadata: &'a HashMap<String, String>,
+    pub headers: &'a HeaderMap<HeaderValue>,
+}
+
+#[async_trait]
+pub trait ObjectEncryptionResolver: Send + Sync {
+    async fn resolve_read_material(
+        &self,
+        request: ReadEncryptionRequest<'_>,
+    ) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError>;
+}

crates/ecstore/src/object_api/mod.rs

@@ -84,6 +84,7 @@ pub(crate) fn legacy_encrypted_range_seek_enabled() -> bool {
 }
 
 mod body_cache_hook;
+mod encryption;
 mod hook_slot;
 mod object_mutation_hook;
 mod readers;
@@ -98,6 +99,10 @@ pub use body_cache_hook::{
 pub(crate) use body_cache_hook::{
     get_object_body_cache_hook, get_object_body_cache_hook_suppressed, without_get_object_body_cache_hook,
 };
+pub use encryption::{
+    EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
+    ReadEncryptionMode, ReadEncryptionRequest,
+};
 pub(crate) use object_mutation_hook::notify_object_mutation;
 pub use object_mutation_hook::{ObjectMutationHook, register_object_mutation_hook, unregister_object_mutation_hook};
 pub use readers::*;

crates/ecstore/src/object_api/types.rs

@@ -271,29 +271,9 @@ impl ObjectInfo {
     }
 
     pub fn is_encrypted(&self) -> bool {
-        // Corresponding to the logic in rustfs/src/sse.rs/encryption_material_to_metadata function
-        use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
-
-        self.user_defined.keys().any(|key| {
-            let lower = key.to_ascii_lowercase();
-            lower.starts_with("x-minio-encryption-")
-                || lower.starts_with("x-minio-internal-server-side-encryption-")
-                || matches!(
-                    lower.as_str(),
-                    "x-minio-internal-encrypted-multipart"
-                        | "x-rustfs-encryption-key"
-                        | "x-rustfs-encryption-algorithm"
-                        | "x-rustfs-encryption-iv"
-                        | "x-rustfs-encryption-key-id"
-                        | "x-rustfs-encryption-context"
-                        | "x-rustfs-encryption-tag"
-                        | "x-amz-server-side-encryption-aws-kms-key-id"
-                        | SSEC_ALGORITHM_HEADER
-                        | SSEC_KEY_HEADER
-                        | SSEC_KEY_MD5_HEADER
-                        | "x-amz-server-side-encryption"
-                )
-        })
+        self.user_defined
+            .keys()
+            .any(|key| rustfs_utils::http::is_object_encryption_marker(key))
     }
 
     /// Maximum inline size for non-versioned objects (128 KiB).
@@ -337,26 +317,7 @@ impl ObjectInfo {
     }
 
     pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
-        let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
-        if let Some(size_str) = self
-            .user_defined
-            .get("x-rustfs-encryption-original-size")
-            .map(String::as_str)
-            .or_else(|| {
-                self.user_defined
-                    .get("x-amz-server-side-encryption-customer-original-size")
-                    .map(String::as_str)
-            })
-            .or(actual_size.as_deref())
-            && !size_str.is_empty()
-        {
-            let size = size_str
-                .parse::<i64>()
-                .map_err(|e| std::io::Error::other(format!("Failed to parse encryption original size: {e}")))?;
-            return Ok(Some(size));
-        }
-
-        Ok(None)
+        rustfs_utils::http::get_object_encryption_original_size(&self.user_defined)
     }
 
     pub fn decrypted_size(&self) -> std::io::Result<i64> {
@@ -386,9 +347,6 @@ impl ObjectInfo {
             return Ok(actual_size);
         }
 
-        // Check if object is encrypted
-        // Managed SSE stores original size in x-rustfs-encryption-original-size metadata
-        // SSE-C stores original size in x-amz-server-side-encryption-customer-original-size
         if let Some(size) = self.encryption_original_size()? {
             return Ok(size);
         }
@@ -878,6 +836,19 @@ mod tests {
         assert!(!object.is_inline_fast_path_eligible(), "transitioned objects must fall back");
     }
 
+    #[test]
+    fn minio_internal_encryption_metadata_is_not_treated_as_plaintext() {
+        let object = ObjectInfo {
+            user_defined: Arc::new(HashMap::from([(
+                "X-Minio-Internal-Server-Side-Encryption-Sealed-Key".to_string(),
+                "sealed".to_string(),
+            )])),
+            ..Default::default()
+        };
+
+        assert!(object.is_encrypted());
+    }
+
     #[test]
     fn versions_after_marker_handles_null_version_marker() {
         let first_version = Uuid::parse_str("11111111-2222-3333-4444-555555555555").unwrap();

crates/ecstore/src/runtime/instance.rs

@@ -46,6 +46,7 @@ use crate::bucket::metadata_sys::BucketMetadataSys;
 use crate::bucket::replication::{DynReplicationPool, ReplicationStats};
 use crate::disk::DiskStore;
 use crate::layout::endpoints::{EndpointServerPools, SetupType};
+use crate::object_api::ObjectEncryptionResolver;
 use crate::services::event_notification::EventNotifier;
 use crate::services::tier::tier::TierConfigMgr;
 use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
@@ -159,6 +160,8 @@ pub struct InstanceContext {
     /// workers (scanner/heal/tier/lifecycle) without touching another instance.
     /// Replaces the process-global cancel-token static.
     background_cancel_token: OnceLock<CancellationToken>,
+    /// Resolves object-encryption material at the application boundary.
+    object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
     tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
     #[cfg(test)]
@@ -197,6 +200,7 @@ impl InstanceContext {
             local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
             bucket_metadata_sys: std::sync::Mutex::new(None),
             background_cancel_token: OnceLock::new(),
+            object_encryption_resolver: OnceLock::new(),
             tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
             #[cfg(test)]
@@ -209,6 +213,19 @@ impl InstanceContext {
         self.lock_manager.clone()
     }
 
+    /// Install the application-owned object-encryption resolver once.
+    pub fn set_object_encryption_resolver(
+        &self,
+        resolver: Arc<dyn ObjectEncryptionResolver>,
+    ) -> Result<(), Arc<dyn ObjectEncryptionResolver>> {
+        self.object_encryption_resolver.set(resolver)
+    }
+
+    /// Return the configured object-encryption resolver, if startup installed one.
+    pub fn object_encryption_resolver(&self) -> Option<&dyn ObjectEncryptionResolver> {
+        self.object_encryption_resolver.get().map(Arc::as_ref)
+    }
+
     /// Set this instance's S3 region.
     ///
     /// Write-once: panics on a second write, preserving the startup fail-fast

crates/ecstore/src/runtime/sources.rs

@@ -46,7 +46,6 @@ use crate::{
 use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
 use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
-use rustfs_kms::{ObjectEncryptionService, get_global_encryption_service};
 use rustfs_lock::client::LockClient;
 use s3s::dto::BucketLifecycleConfiguration;
 use s3s::region::Region;
@@ -105,10 +104,6 @@ pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_
     global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
 }
 
-pub(crate) async fn object_encryption_service() -> Option<Arc<ObjectEncryptionService>> {
-    get_global_encryption_service().await
-}
-
 pub fn object_store_handle() -> Option<Arc<ECStore>> {
     resolve_object_store_handle()
 }

crates/ecstore/src/set_disk/metadata.rs

@@ -505,9 +505,7 @@ impl SetDisks {
     }
 
     fn file_info_has_encryption_metadata(meta: &FileInfo) -> bool {
-        meta.metadata
-            .keys()
-            .any(|name| http::is_encryption_metadata_key(name) || http::is_sse_header(name))
+        meta.metadata.keys().any(|name| http::is_object_encryption_marker(name))
     }
 
     fn starts_with_ignore_ascii_case(value: &str, prefix: &str) -> bool {

crates/ecstore/src/set_disk/mod.rs

@@ -143,15 +143,17 @@ use rustfs_object_capacity::capacity_scope::{
     CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
 };
 use rustfs_s3_types::EventName;
+#[cfg(test)]
+use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
 use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
 use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
 use rustfs_utils::http::headers::{
     CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
 };
 use rustfs_utils::http::{
-    SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE,
-    SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, contains_key_str,
-    get_header_map, get_str, insert_str, is_encryption_metadata_key, remove_header_map,
+    SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
+    SUFFIX_RESTORE_OPERATION_ID, contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker,
+    remove_header_map,
 };
 use rustfs_utils::{
     HashAlgorithm,
@@ -407,10 +409,7 @@ pub(crate) fn strip_internal_multipart_metadata(metadata: &mut HashMap<String, S
 }
 
 fn should_persist_encryption_original_size(metadata: &HashMap<String, String>) -> bool {
-    metadata.keys().any(|key| is_encryption_metadata_key(key))
-        || metadata.contains_key(SSEC_ALGORITHM_HEADER)
-        || metadata.contains_key(SSEC_KEY_HEADER)
-        || metadata.contains_key(SSEC_KEY_MD5_HEADER)
+    metadata.keys().any(|key| is_object_encryption_marker(key))
 }
 
 /// Per-set memoized capacity dirty scope.

crates/ecstore/src/set_disk/ops/object.rs

@@ -39,6 +39,7 @@ use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppress
 use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
 use crate::store::ECStore;
 use futures::FutureExt as _;
+use http::HeaderValue;
 use std::future::Future;
 
 fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
@@ -46,6 +47,17 @@ fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Er
         .map_err(Error::from)
 }
 
+async fn get_object_reader_with_context(
+    ctx: &InstanceContext,
+    reader: Box<dyn AsyncRead + Unpin + Send + Sync>,
+    range: Option<HTTPRangeSpec>,
+    object_info: &ObjectInfo,
+    opts: &ObjectOptions,
+    headers: &HeaderMap<HeaderValue>,
+) -> Result<(GetObjectReader, usize, i64)> {
+    GetObjectReader::new_with_resolver(reader, range, object_info, opts, headers, ctx.object_encryption_resolver()).await
+}
+
 /// Length of the full plaintext body when — and only when — this read's output
 /// is exactly the object's complete plaintext, so the app-layer body cache may
 /// serve it in place of the erasure read.
@@ -704,7 +716,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
                             size_bucket,
                         );
                         record_get_object_reader_path_observation(GET_OBJECT_PATH_CODEC_STREAMING, object_class, size_bucket);
-                        let (mut reader, _offset, _length) = GetObjectReader::new(stream, range, &object_info, opts, &h).await?;
+                        let (mut reader, _offset, _length) =
+                            get_object_reader_with_context(&self.ctx, stream, range, &object_info, opts, &h).await?;
                         // Carry the hook probe result so the app layer skips its
                         // now-redundant lookup on the streaming miss path (ODC-16).
                         reader.body_source = body_source;
@@ -742,7 +755,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
         let (rd, wd) = tokio::io::duplex(duplex_buffer_size);
         debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer");
 
-        let (mut reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h).await?;
+        let (mut reader, offset, length) =
+            get_object_reader_with_context(&self.ctx, Box::new(rd), range, &object_info, opts, &h).await?;
         // Carry the hook probe result so the app layer skips its now-redundant
         // lookup on the streaming miss path (ODC-16).
         reader.body_source = body_source;
@@ -4275,6 +4289,61 @@ mod erasure_construction_tests {
     }
 }
 
+#[cfg(test)]
+mod object_encryption_resolver_wiring_tests {
+    use super::*;
+    use crate::object_api::{EncryptionResolutionError, ObjectEncryptionResolver, ReadEncryptionMaterial, ReadEncryptionRequest};
+    use std::io::Cursor;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    struct CountingResolver {
+        calls: AtomicUsize,
+    }
+
+    #[async_trait::async_trait]
+    impl ObjectEncryptionResolver for CountingResolver {
+        async fn resolve_read_material(
+            &self,
+            _request: ReadEncryptionRequest<'_>,
+        ) -> std::result::Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
+            self.calls.fetch_add(1, Ordering::Relaxed);
+            Ok(None)
+        }
+    }
+
+    #[tokio::test]
+    async fn get_object_reader_forwards_instance_resolver() {
+        let resolver = Arc::new(CountingResolver {
+            calls: AtomicUsize::new(0),
+        });
+        let ctx = InstanceContext::new();
+        assert!(
+            ctx.set_object_encryption_resolver(resolver.clone()).is_ok(),
+            "fresh context should accept resolver"
+        );
+        let object_info = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            size: 1,
+            user_defined: Arc::new(HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])),
+            ..Default::default()
+        };
+
+        let result = get_object_reader_with_context(
+            &ctx,
+            Box::new(Cursor::new(Vec::<u8>::new())),
+            None,
+            &object_info,
+            &ObjectOptions::default(),
+            &HeaderMap::new(),
+        )
+        .await;
+
+        assert!(result.is_err(), "resolver returning no material must fail closed");
+        assert_eq!(resolver.calls.load(Ordering::Relaxed), 1);
+    }
+}
+
 #[cfg(test)]
 pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
     //! Shared hermetic `SetDisks` construction for the ops tests below: the

crates/ecstore/tests/README.md

@@ -2,7 +2,7 @@
 
 ## MinIO-generated encrypted fixtures
 
-`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
+`rustfs/src/storage/minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
 `.\rustfs\scripts\minio_fixture_lab\lab.py`.
 
 It currently covers multipart fixtures for:
@@ -20,5 +20,5 @@ Example:
 ```powershell
 $env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
 $env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
-cargo +1.97.1 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
+cargo +1.97.1 test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
 ```

crates/kms/src/lib.rs

@@ -89,7 +89,7 @@ pub use error::{KmsError, KmsUnavailableError, Result};
 pub use manager::KmsManager;
 pub use service::{DataKey, ObjectEncryptionService};
 pub use service_manager::{
-    KmsServiceManager, KmsServiceStatus, get_global_encryption_service, get_global_kms_service_manager,
+    KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,
     init_global_kms_service_manager,
 };
 pub use types::*;

crates/kms/src/service_manager.rs

@@ -20,11 +20,12 @@ use crate::error::{KmsError, Result};
 use crate::manager::KmsManager;
 use crate::service::ObjectEncryptionService;
 use arc_swap::ArcSwap;
+use std::future::Future;
 use std::sync::{
     Arc, OnceLock,
     atomic::{AtomicU64, Ordering},
 };
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::Mutex;
 use tracing::{debug, error, info, warn};
 
 const LOG_COMPONENT_KMS: &str = "kms";
@@ -44,6 +45,13 @@ pub enum KmsServiceStatus {
     Error(String),
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum KmsStartOutcome {
+    Started,
+    Restarted,
+    AlreadyRunning,
+}
+
 /// Service version information for zero-downtime reconfiguration
 #[derive(Clone)]
 struct ServiceVersion {
@@ -55,16 +63,17 @@ struct ServiceVersion {
     manager: Arc<KmsManager>,
 }
 
+#[derive(Clone)]
+struct RuntimeState {
+    config: Option<KmsConfig>,
+    status: KmsServiceStatus,
+    current_service: Option<ServiceVersion>,
+}
+
 /// Dynamic KMS service manager with versioned services for zero-downtime reconfiguration
 pub struct KmsServiceManager {
-    /// Current service version (if running)
-    /// Uses ArcSwap for atomic, lock-free service switching
-    /// This allows instant atomic updates without blocking readers
-    current_service: ArcSwap<Option<ServiceVersion>>,
-    /// Current configuration
-    config: Arc<RwLock<Option<KmsConfig>>>,
-    /// Current status
-    status: Arc<RwLock<KmsServiceStatus>>,
+    /// Atomically published configuration, status, and current service.
+    state: ArcSwap<RuntimeState>,
     /// Version counter (monotonically increasing)
     version_counter: Arc<AtomicU64>,
     /// Mutex to protect lifecycle operations (start, stop, reconfigure)
@@ -76,9 +85,11 @@ impl KmsServiceManager {
     /// Create a new KMS service manager (not configured)
     pub fn new() -> Self {
         Self {
-            current_service: ArcSwap::from_pointee(None),
-            config: Arc::new(RwLock::new(None)),
-            status: Arc::new(RwLock::new(KmsServiceStatus::NotConfigured)),
+            state: ArcSwap::from_pointee(RuntimeState {
+                config: None,
+                status: KmsServiceStatus::NotConfigured,
+                current_service: None,
+            }),
             version_counter: Arc::new(AtomicU64::new(0)),
             lifecycle_mutex: Arc::new(Mutex::new(())),
         }
@@ -86,39 +97,65 @@ impl KmsServiceManager {
 
     /// Get current service status
     pub async fn get_status(&self) -> KmsServiceStatus {
-        self.status.read().await.clone()
+        self.state.load().status.clone()
     }
 
     /// Get current configuration (if any)
     pub async fn get_config(&self) -> Option<KmsConfig> {
-        self.config.read().await.clone()
+        self.state.load().config.clone()
     }
 
     /// Get configuration for status and management responses without static key material.
     pub async fn get_redacted_config(&self) -> Option<KmsConfig> {
-        let mut config = self.config.read().await.clone()?;
+        let mut config = self.state.load().config.clone()?;
+        Self::redact_config(&mut config);
+        Some(config)
+    }
+
+    /// Get status and redacted configuration from the same published snapshot.
+    pub async fn get_redacted_state(&self) -> (KmsServiceStatus, Option<KmsConfig>) {
+        let state = self.state.load();
+        let mut config = state.config.clone();
+        if let Some(config) = &mut config {
+            Self::redact_config(config);
+        }
+        (state.status.clone(), config)
+    }
+
+    fn redact_config(config: &mut KmsConfig) {
         if let BackendConfig::Static(static_config) = &mut config.backend_config {
             use zeroize::Zeroize;
             static_config.secret_key.zeroize();
         }
-        Some(config)
     }
 
     /// Configure KMS with new configuration
     pub async fn configure(&self, new_config: KmsConfig) -> Result<()> {
+        self.configure_with_persistence(new_config, || async { Ok(()) }).await
+    }
+
+    /// Configure KMS and publish the in-memory state only after persistence succeeds.
+    ///
+    /// The persistence callback runs under the lifecycle lock and must not call
+    /// another lifecycle method on this manager.
+    pub async fn configure_with_persistence<Persist, PersistFuture>(&self, new_config: KmsConfig, persist: Persist) -> Result<()>
+    where
+        Persist: FnOnce() -> PersistFuture,
+        PersistFuture: Future<Output = Result<()>>,
+    {
         new_config.validate()?;
-
-        // Update configuration
-        {
-            let mut config = self.config.write().await;
-            *config = Some(new_config.clone());
-        }
-
-        // Update status
-        {
-            let mut status = self.status.write().await;
-            *status = KmsServiceStatus::Configured;
+        let _guard = self.lifecycle_mutex.lock().await;
+        if self.state.load().current_service.is_some() {
+            return Err(KmsError::configuration_error(
+                "Cannot configure KMS while it is running; use reconfigure instead",
+            ));
         }
+        persist().await?;
+        self.state.store(Arc::new(RuntimeState {
+            config: Some(new_config),
+            status: KmsServiceStatus::Configured,
+            current_service: None,
+        }));
 
         debug!(
             event = EVENT_KMS_SERVICE_STATE,
@@ -136,19 +173,35 @@ impl KmsServiceManager {
         self.start_internal().await
     }
 
+    /// Start or restart KMS with the running-state decision serialized with the lifecycle action.
+    pub async fn start_or_restart(&self, force: bool) -> Result<KmsStartOutcome> {
+        let _guard = self.lifecycle_mutex.lock().await;
+        let running = self.state.load().current_service.is_some();
+        if running && !force {
+            return Ok(KmsStartOutcome::AlreadyRunning);
+        }
+        self.start_internal().await?;
+        Ok(if running {
+            KmsStartOutcome::Restarted
+        } else {
+            KmsStartOutcome::Started
+        })
+    }
+
     /// Internal start implementation (called within lifecycle mutex)
     async fn start_internal(&self) -> Result<()> {
-        let config = {
-            let config_guard = self.config.read().await;
-            match config_guard.as_ref() {
-                Some(config) => config.clone(),
-                None => {
-                    let err_msg = "Cannot start KMS: no configuration provided";
-                    error!("{}", err_msg);
-                    let mut status = self.status.write().await;
-                    *status = KmsServiceStatus::Error(err_msg.to_string());
-                    return Err(KmsError::configuration_error(err_msg));
-                }
+        let state = self.state.load_full();
+        let config = match state.config.as_ref() {
+            Some(config) => config.clone(),
+            None => {
+                let err_msg = "Cannot start KMS: no configuration provided";
+                error!("{}", err_msg);
+                self.state.store(Arc::new(RuntimeState {
+                    config: None,
+                    status: KmsServiceStatus::Error(err_msg.to_string()),
+                    current_service: None,
+                }));
+                return Err(KmsError::configuration_error(err_msg));
             }
         };
 
@@ -161,17 +214,9 @@ impl KmsServiceManager {
             "KMS service starting"
         );
 
-        match self.create_service_version(&config).await {
+        match self.create_healthy_service_version(&config).await {
             Ok(service_version) => {
-                // Atomically update to new service version (lock-free, instant)
-                // ArcSwap::store() is a true atomic operation using CAS
-                self.current_service.store(Arc::new(Some(service_version)));
-
-                // Update status
-                {
-                    let mut status = self.status.write().await;
-                    *status = KmsServiceStatus::Running;
-                }
+                self.publish_running(config, service_version);
 
                 debug!(
                     event = EVENT_KMS_SERVICE_STATE,
@@ -185,13 +230,24 @@ impl KmsServiceManager {
             Err(e) => {
                 let err_msg = format!("Failed to create KMS backend: {e}");
                 error!("{}", err_msg);
-                let mut status = self.status.write().await;
-                *status = KmsServiceStatus::Error(err_msg.clone());
+                if state.current_service.is_none() {
+                    self.state.store(Arc::new(RuntimeState {
+                        config: state.config.clone(),
+                        status: KmsServiceStatus::Error(err_msg.clone()),
+                        current_service: None,
+                    }));
+                }
                 Err(KmsError::backend_error(&err_msg))
             }
         }
     }
 
+    /// Replace the running service without exposing a stopped interval.
+    pub async fn restart(&self) -> Result<()> {
+        let _guard = self.lifecycle_mutex.lock().await;
+        self.start_internal().await
+    }
+
     /// Stop KMS service
     ///
     /// Note: This stops accepting new operations, but existing operations using
@@ -213,15 +269,16 @@ impl KmsServiceManager {
 
         // Atomically clear current service version (lock-free, instant)
         // Note: Existing Arc references will keep the service alive until operations complete
-        self.current_service.store(Arc::new(None));
-
-        // Update status (keep configuration)
-        {
-            let mut status = self.status.write().await;
-            if !matches!(*status, KmsServiceStatus::NotConfigured) {
-                *status = KmsServiceStatus::Configured;
-            }
-        }
+        let state = self.state.load_full();
+        self.state.store(Arc::new(RuntimeState {
+            config: state.config.clone(),
+            status: if state.config.is_some() {
+                KmsServiceStatus::Configured
+            } else {
+                KmsServiceStatus::NotConfigured
+            },
+            current_service: None,
+        }));
 
         debug!(
             event = EVENT_KMS_SERVICE_STATE,
@@ -244,6 +301,22 @@ impl KmsServiceManager {
     /// This ensures zero downtime during reconfiguration, even for long-running
     /// operations like encrypting large files.
     pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
+        self.reconfigure_with_persistence(new_config, || async { Ok(()) }).await
+    }
+
+    /// Reconfigure KMS after the candidate is healthy and persistence succeeds.
+    ///
+    /// The persistence callback runs under the lifecycle lock and must not call
+    /// another lifecycle method on this manager.
+    pub async fn reconfigure_with_persistence<Persist, PersistFuture>(
+        &self,
+        new_config: KmsConfig,
+        persist: Persist,
+    ) -> Result<()>
+    where
+        Persist: FnOnce() -> PersistFuture,
+        PersistFuture: Future<Output = Result<()>>,
+    {
         let _guard = self.lifecycle_mutex.lock().await;
 
         debug!(
@@ -255,29 +328,16 @@ impl KmsServiceManager {
         );
         new_config.validate()?;
 
-        // Configure with new config
-        {
-            let mut config = self.config.write().await;
-            *config = Some(new_config.clone());
-        }
-
         // Create new service version without stopping old one
         // This allows existing operations to continue while new operations use new service
-        match self.create_service_version(&new_config).await {
+        match self.create_healthy_service_version(&new_config).await {
             Ok(new_service_version) => {
                 // Get old version for logging (lock-free read)
-                let old_version = self.current_service.load().as_ref().as_ref().map(|sv| sv.version);
+                let old_version = self.state.load().current_service.as_ref().map(|sv| sv.version);
 
-                // Atomically switch to new service version (lock-free, instant CAS operation)
-                // This is a true atomic operation - no waiting for locks, instant switch
-                // Old service will be dropped when no more Arc references exist
-                self.current_service.store(Arc::new(Some(new_service_version.clone())));
+                persist().await?;
 
-                // Update status
-                {
-                    let mut status = self.status.write().await;
-                    *status = KmsServiceStatus::Running;
-                }
+                self.publish_running(new_config, new_service_version.clone());
 
                 if let Some(old_ver) = old_version {
                     info!(
@@ -304,8 +364,6 @@ impl KmsServiceManager {
             Err(e) => {
                 let err_msg = format!("Failed to reconfigure KMS: {e}");
                 error!("{}", err_msg);
-                let mut status = self.status.write().await;
-                *status = KmsServiceStatus::Error(err_msg.clone());
                 Err(KmsError::backend_error(&err_msg))
             }
         }
@@ -316,7 +374,7 @@ impl KmsServiceManager {
     /// Returns the manager from the current service version.
     /// Uses lock-free atomic load for optimal performance.
     pub async fn get_manager(&self) -> Option<Arc<KmsManager>> {
-        self.current_service.load().as_ref().as_ref().map(|sv| sv.manager.clone())
+        self.state.load().current_service.as_ref().map(|sv| sv.manager.clone())
     }
 
     /// Get encryption service (if running)
@@ -326,7 +384,7 @@ impl KmsServiceManager {
     /// This ensures new operations always use the latest service version,
     /// while existing operations continue using their Arc references.
     pub async fn get_encryption_service(&self) -> Option<Arc<ObjectEncryptionService>> {
-        self.current_service.load().as_ref().as_ref().map(|sv| sv.service.clone())
+        self.state.load().current_service.as_ref().map(|sv| sv.service.clone())
     }
 
     /// Get current service version number
@@ -334,14 +392,16 @@ impl KmsServiceManager {
     /// Useful for monitoring and debugging.
     /// Uses lock-free atomic load.
     pub async fn get_service_version(&self) -> Option<u64> {
-        self.current_service.load().as_ref().as_ref().map(|sv| sv.version)
+        self.state.load().current_service.as_ref().map(|sv| sv.version)
     }
 
     /// Health check for the KMS service
     pub async fn health_check(&self) -> Result<bool> {
-        let manager = self.get_manager().await;
-        match manager {
-            Some(manager) => {
+        let checked_state = self.state.load_full();
+        match checked_state.current_service.as_ref() {
+            Some(service_version) => {
+                let manager = service_version.manager.clone();
+                let checked_version = service_version.version;
                 // Perform health check on the backend
                 match manager.health_check().await {
                     Ok(healthy) => {
@@ -352,9 +412,8 @@ impl KmsServiceManager {
                     }
                     Err(e) => {
                         error!("KMS health check error: {}", e);
-                        // Update status to error
-                        let mut status = self.status.write().await;
-                        *status = KmsServiceStatus::Error(format!("Health check failed: {e}"));
+                        let _guard = self.lifecycle_mutex.lock().await;
+                        self.mark_health_error_if_current(checked_version, &e);
                         Err(e)
                     }
                 }
@@ -413,6 +472,33 @@ impl KmsServiceManager {
             manager: kms_manager,
         })
     }
+
+    async fn create_healthy_service_version(&self, config: &KmsConfig) -> Result<ServiceVersion> {
+        let service_version = self.create_service_version(config).await?;
+        if !service_version.manager.health_check().await? {
+            return Err(KmsError::backend_error("KMS backend health check failed"));
+        }
+        Ok(service_version)
+    }
+
+    fn publish_running(&self, config: KmsConfig, service_version: ServiceVersion) {
+        self.state.store(Arc::new(RuntimeState {
+            config: Some(config),
+            status: KmsServiceStatus::Running,
+            current_service: Some(service_version),
+        }));
+    }
+
+    fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
+        let current = self.state.load_full();
+        if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
+            self.state.store(Arc::new(RuntimeState {
+                config: current.config.clone(),
+                status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
+                current_service: current.current_service.clone(),
+            }));
+        }
+    }
 }
 
 impl Default for KmsServiceManager {
@@ -445,6 +531,11 @@ pub async fn get_global_encryption_service() -> Option<Arc<ObjectEncryptionServi
 #[cfg(test)]
 mod tests {
     use super::*;
+    use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
+
+    fn static_config(key_id: &str, fill: u8) -> KmsConfig {
+        KmsConfig::static_kms(key_id.to_string(), BASE64_STANDARD.encode([fill; 32]))
+    }
 
     #[tokio::test]
     async fn configure_rejects_insecure_development_defaults_before_state_update() {
@@ -462,8 +553,6 @@ mod tests {
 
     #[tokio::test]
     async fn redacted_config_omits_static_key_material() {
-        use base64::Engine as _;
-
         let manager = KmsServiceManager::new();
         let encoded_key = base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]);
         manager
@@ -477,4 +566,134 @@ mod tests {
         };
         assert!(static_config.secret_key.is_empty());
     }
+
+    #[tokio::test]
+    async fn configure_persistence_failure_leaves_state_unchanged() {
+        let manager = KmsServiceManager::new();
+
+        let result = manager
+            .configure_with_persistence(static_config("key-a", 0x11), || async { Err(KmsError::backend_error("persist failed")) })
+            .await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::NotConfigured);
+        assert!(manager.get_config().await.is_none());
+        assert!(manager.get_encryption_service().await.is_none());
+    }
+
+    #[tokio::test]
+    async fn configure_rejects_running_service_without_changing_snapshot() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let version = manager.get_service_version().await;
+
+        let result = manager.configure(static_config("key-b", 0x22)).await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert_eq!(manager.get_service_version().await, version);
+        assert_eq!(
+            manager.get_config().await.and_then(|config| config.default_key_id),
+            Some("key-a".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn reconfigure_persistence_failure_keeps_old_running_snapshot() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await;
+        let old_service = manager.get_encryption_service().await.expect("old service");
+
+        let result = manager
+            .reconfigure_with_persistence(static_config("key-b", 0x22), || async {
+                Err(KmsError::backend_error("persist failed"))
+            })
+            .await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert_eq!(manager.get_service_version().await, old_version);
+        assert_eq!(
+            manager.get_config().await.and_then(|config| config.default_key_id),
+            Some("key-a".to_string())
+        );
+        assert!(Arc::ptr_eq(
+            &old_service,
+            &manager.get_encryption_service().await.expect("old service remains")
+        ));
+    }
+
+    #[tokio::test]
+    async fn reconfigure_candidate_failure_keeps_old_running_snapshot() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await;
+        let invalid_parent = tempfile::NamedTempFile::new().expect("temporary file");
+        let invalid_config = KmsConfig::local(invalid_parent.path().join("keys")).with_insecure_development_defaults();
+
+        let result = manager.reconfigure(invalid_config).await;
+
+        assert!(result.is_err());
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+        assert_eq!(manager.get_service_version().await, old_version);
+        assert_eq!(
+            manager.get_config().await.and_then(|config| config.default_key_id),
+            Some("key-a".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn restart_never_unpublishes_the_running_service() {
+        let manager = Arc::new(KmsServiceManager::new());
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await.expect("old version");
+        let restarting = {
+            let manager = manager.clone();
+            tokio::spawn(async move { manager.restart().await })
+        };
+
+        while !restarting.is_finished() {
+            assert!(manager.get_encryption_service().await.is_some());
+            tokio::task::yield_now().await;
+        }
+        restarting.await.expect("restart task").expect("restart");
+
+        assert!(manager.get_encryption_service().await.is_some());
+        assert!(manager.get_service_version().await.expect("new version") > old_version);
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+    }
+
+    #[tokio::test]
+    async fn start_or_restart_decides_under_the_lifecycle_lock() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+
+        assert_eq!(manager.start_or_restart(false).await.expect("initial start"), KmsStartOutcome::Started);
+        let first_version = manager.get_service_version().await.expect("first version");
+        assert_eq!(
+            manager.start_or_restart(false).await.expect("already running"),
+            KmsStartOutcome::AlreadyRunning
+        );
+        assert_eq!(manager.get_service_version().await, Some(first_version));
+        assert_eq!(manager.start_or_restart(true).await.expect("forced restart"), KmsStartOutcome::Restarted);
+        assert!(manager.get_service_version().await.expect("restarted version") > first_version);
+    }
+
+    #[tokio::test]
+    async fn stale_health_failure_cannot_poison_new_service_status() {
+        let manager = KmsServiceManager::new();
+        manager.configure(static_config("key-a", 0x11)).await.expect("configure");
+        manager.start().await.expect("start");
+        let old_version = manager.get_service_version().await.expect("old version");
+        manager.restart().await.expect("restart");
+
+        manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
+
+        assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
+    }
 }

crates/rio-v2/tests/minio_fixture_lab/README.md

@@ -137,7 +137,7 @@ tests read):
 ./capture_via_docker.sh
 
 RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
-  cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
+  cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored
 ```
 
 This is exactly what the nightly `minio-interop` GitHub Actions workflow runs

crates/utils/src/http/header_compat.rs

@@ -25,6 +25,11 @@ const RUSTFS_PREFIX: &str = "x-rustfs-";
 const MINIO_PREFIX: &str = "x-minio-";
 const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
 const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
+const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-";
+const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart";
+const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = "x-rustfs-encryption-original-size";
+const MINIO_ENCRYPTION_ORIGINAL_SIZE: &str = "x-minio-encryption-original-size";
+const SSEC_ORIGINAL_SIZE: &str = "x-amz-server-side-encryption-customer-original-size";
 
 // Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
 pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
@@ -40,11 +45,49 @@ pub const SUFFIX_SOURCE_REPLICATION_REQUEST: &str = "source-replication-request"
 pub const SUFFIX_SOURCE_REPLICATION_CHECK: &str = "source-replication-check";
 pub const SUFFIX_REPLICATION_SSEC_CRC: &str = "replication-ssec-crc";
 
-/// Returns true if the key is an internal encryption metadata key (x-rustfs-encryption-* or
-/// x-minio-encryption-*). Case-insensitive for metadata filtering.
+/// Returns true if the key is object-encryption metadata understood by RustFS or MinIO.
+/// Case-insensitive for metadata filtering.
 pub fn is_encryption_metadata_key(key: &str) -> bool {
     let lower = key.to_lowercase();
-    lower.starts_with(RUSTFS_ENCRYPTION_PREFIX) || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
+    lower.starts_with(RUSTFS_ENCRYPTION_PREFIX)
+        || lower.starts_with(MINIO_ENCRYPTION_PREFIX)
+        || lower.starts_with(MINIO_INTERNAL_ENCRYPTION_PREFIX)
+        || lower == MINIO_INTERNAL_ENCRYPTED_MULTIPART
+}
+
+/// Returns true when a metadata key proves that object data is encrypted.
+///
+/// Original-size metadata alone is not proof: older plaintext objects can
+/// retain that compatibility field after metadata migration.
+pub fn is_object_encryption_marker(key: &str) -> bool {
+    (is_encryption_metadata_key(key)
+        && !key.eq_ignore_ascii_case(RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
+        && !key.eq_ignore_ascii_case(MINIO_ENCRYPTION_ORIGINAL_SIZE))
+        || super::is_sse_header(key)
+}
+
+/// Reads the logical object size recorded by encryption metadata.
+pub fn get_object_encryption_original_size(metadata: &std::collections::HashMap<String, String>) -> std::io::Result<Option<i64>> {
+    let actual_size = super::get_str(metadata, super::SUFFIX_ACTUAL_SIZE);
+    let size = get_case_insensitive(metadata, RUSTFS_ENCRYPTION_ORIGINAL_SIZE)
+        .or_else(|| get_case_insensitive(metadata, SSEC_ORIGINAL_SIZE))
+        .or(actual_size.as_deref());
+
+    let Some(size) = size.filter(|size| !size.is_empty()) else {
+        return Ok(None);
+    };
+    size.parse::<i64>()
+        .map(Some)
+        .map_err(|error| std::io::Error::other(format!("Failed to parse encryption original size: {error}")))
+}
+
+fn get_case_insensitive<'a>(metadata: &'a std::collections::HashMap<String, String>, key: &str) -> Option<&'a str> {
+    metadata.get(key).map(String::as_str).or_else(|| {
+        metadata
+            .iter()
+            .find(|(candidate, _)| candidate.eq_ignore_ascii_case(key))
+            .map(|(_, value)| value.as_str())
+    })
 }
 
 fn rustfs_key(suffix: &str) -> String {
@@ -106,10 +149,37 @@ mod tests {
         assert!(is_encryption_metadata_key("x-rustfs-encryption-iv"));
         assert!(is_encryption_metadata_key("X-Rustfs-Encryption-Key"));
         assert!(is_encryption_metadata_key("x-minio-encryption-iv"));
+        assert!(is_encryption_metadata_key("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
+        assert!(is_encryption_metadata_key("X-Minio-Internal-Encrypted-Multipart"));
         assert!(!is_encryption_metadata_key("x-amz-meta-custom"));
         assert!(!is_encryption_metadata_key("x-rustfs-internal-healing"));
     }
 
+    #[test]
+    fn object_encryption_marker_excludes_size_only_metadata() {
+        assert!(!is_object_encryption_marker(RUSTFS_ENCRYPTION_ORIGINAL_SIZE));
+        assert!(is_object_encryption_marker("X-Minio-Internal-Server-Side-Encryption-Sealed-Key"));
+        assert!(is_object_encryption_marker("x-amz-server-side-encryption"));
+    }
+
+    #[test]
+    fn object_encryption_original_size_is_case_insensitive() {
+        let metadata = std::collections::HashMap::from([(
+            "X-Amz-Server-Side-Encryption-Customer-Original-Size".to_string(),
+            "42".to_string(),
+        )]);
+        assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
+    }
+
+    #[test]
+    fn object_encryption_original_size_prefers_rustfs_metadata() {
+        let metadata = std::collections::HashMap::from([
+            (SSEC_ORIGINAL_SIZE.to_string(), "21".to_string()),
+            (RUSTFS_ENCRYPTION_ORIGINAL_SIZE.to_string(), "42".to_string()),
+        ]);
+        assert_eq!(get_object_encryption_original_size(&metadata).expect("valid size"), Some(42));
+    }
+
     #[test]
     fn test_get_header() {
         let mut headers = HeaderMap::new();

docs/testing/ecstore-validation-suite-design.md

@@ -358,7 +358,7 @@ Fixture-backed tests should run when the fixture path is present:
 
 ```bash
 cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture
-cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
+cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture
 ```
 
 ## Multi-Expert Adversarial Review Summary

rustfs/src/admin/handlers/kms_dynamic.rs

@@ -363,36 +363,27 @@ impl Operation for ConfigureKmsHandler {
         // Convert request to KmsConfig
         let kms_config = configure_request.to_kms_config();
 
-        // Configure the service
-        let (success, message, status) = match service_manager.configure(kms_config.clone()).await {
+        let persisted_config = kms_config.clone();
+        let (success, message, status) = match service_manager
+            .configure_with_persistence(kms_config, || async move {
+                save_kms_config(&persisted_config)
+                    .await
+                    .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
+            })
+            .await
+        {
             Ok(()) => {
-                // Persist the configuration to cluster storage
-                if let Err(e) = save_kms_config(&kms_config).await {
-                    let error_msg = format!("KMS configured in memory but failed to persist: {e}");
-                    error!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "configure",
-                        state = "persist_failed",
-                        error = %e,
-                        "admin kms dynamic state"
-                    );
-                    let status = service_manager.get_status().await;
-                    (false, error_msg, status)
-                } else {
-                    let status = service_manager.get_status().await;
-                    info!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "configure",
-                        state = "configured",
-                        status = ?status,
-                        "admin kms dynamic state"
-                    );
-                    (true, "KMS configured successfully".to_string(), status)
-                }
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "configure",
+                    state = "configured",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS configured successfully".to_string(), status)
             }
             Err(e) => {
                 let error_msg = format!("Failed to configure KMS: {e}");
@@ -499,125 +490,61 @@ impl Operation for StartKmsHandler {
         );
 
         let service_manager = kms_service_manager_from_context();
-
-        // Check if already running and force flag
-        let current_status = service_manager.get_status().await;
-        if matches!(current_status, KmsServiceStatus::Running) && !start_request.force.unwrap_or(false) {
-            warn!(
-                component = LOG_COMPONENT_ADMIN,
-                subsystem = LOG_SUBSYSTEM_KMS,
-                event = "kms_service_state",
-                operation = "start",
-                state = "already_running",
-                "admin kms dynamic state"
-            );
-            let response = StartKmsResponse {
-                success: false,
-                message: "KMS service is already running. Use force=true to restart.".to_string(),
-                status: current_status,
-            };
-            let json_response = match serde_json::to_string(&response) {
-                Ok(json) => json,
-                Err(e) => {
-                    error!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
-                        operation = "start",
-                        result = "response_serialize_failed",
-                        error = %e,
-                        "admin kms dynamic state"
-                    );
-                    return Ok(S3Response::new((
-                        StatusCode::INTERNAL_SERVER_ERROR,
-                        Body::from("Serialization error".to_string()),
-                    )));
-                }
-            };
-            return Ok(S3Response::new((StatusCode::OK, Body::from(json_response))));
-        }
-
-        // Start the service (or restart if force=true)
-        let (success, message, status) =
-            if start_request.force.unwrap_or(false) && matches!(current_status, KmsServiceStatus::Running) {
-                // Force restart
-                match service_manager.stop().await {
-                    Ok(()) => match service_manager.start().await {
-                        Ok(()) => {
-                            let status = service_manager.get_status().await;
-                            info!(
-                                component = LOG_COMPONENT_ADMIN,
-                                subsystem = LOG_SUBSYSTEM_KMS,
-                                event = "kms_service_state",
-                                operation = "restart",
-                                state = "running",
-                                status = ?status,
-                                "admin kms dynamic state"
-                            );
-                            (true, "KMS service restarted successfully".to_string(), status)
-                        }
-                        Err(e) => {
-                            let error_msg = format!("Failed to restart KMS service: {e}");
-                            error!(
-                                component = LOG_COMPONENT_ADMIN,
-                                subsystem = LOG_SUBSYSTEM_KMS,
-                                event = "kms_service_state",
-                                operation = "restart",
-                                state = "start_failed",
-                                error = %e,
-                                "admin kms dynamic state"
-                            );
-                            let status = service_manager.get_status().await;
-                            (false, error_msg, status)
-                        }
-                    },
-                    Err(e) => {
-                        let error_msg = format!("Failed to stop KMS service for restart: {e}");
-                        error!(
-                            component = LOG_COMPONENT_ADMIN,
-                            subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_service_state",
-                            operation = "restart",
-                            state = "stop_failed",
-                            error = %e,
-                            "admin kms dynamic state"
-                        );
-                        let status = service_manager.get_status().await;
-                        (false, error_msg, status)
-                    }
-                }
-            } else {
-                // Normal start
-                match service_manager.start().await {
-                    Ok(()) => {
-                        let status = service_manager.get_status().await;
-                        info!(
-                            component = LOG_COMPONENT_ADMIN,
-                            subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_service_state",
-                            operation = "start",
-                            state = "running",
-                            status = ?status,
-                            "admin kms dynamic state"
-                        );
-                        (true, "KMS service started successfully".to_string(), status)
-                    }
-                    Err(e) => {
-                        let error_msg = format!("Failed to start KMS service: {e}");
-                        error!(
-                            component = LOG_COMPONENT_ADMIN,
-                            subsystem = LOG_SUBSYSTEM_KMS,
-                            event = "kms_service_state",
-                            operation = "start",
-                            state = "start_failed",
-                            error = %e,
-                            "admin kms dynamic state"
-                        );
-                        let status = service_manager.get_status().await;
-                        (false, error_msg, status)
-                    }
-                }
-            };
+        let force = start_request.force.unwrap_or(false);
+        let (success, message, status) = match service_manager.start_or_restart(force).await {
+            Ok(rustfs_kms::KmsStartOutcome::Started) => {
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "start",
+                    state = "running",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS service started successfully".to_string(), status)
+            }
+            Ok(rustfs_kms::KmsStartOutcome::Restarted) => {
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "restart",
+                    state = "running",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS service restarted successfully".to_string(), status)
+            }
+            Ok(rustfs_kms::KmsStartOutcome::AlreadyRunning) => {
+                let status = service_manager.get_status().await;
+                warn!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "start",
+                    state = "already_running",
+                    "admin kms dynamic state"
+                );
+                (false, "KMS service is already running. Use force=true to restart.".to_string(), status)
+            }
+            Err(e) => {
+                let error_msg = format!("Failed to start or restart KMS service: {e}");
+                error!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "start",
+                    state = "start_failed",
+                    error = %e,
+                    "admin kms dynamic state"
+                );
+                let status = service_manager.get_status().await;
+                (false, error_msg, status)
+            }
+        };
 
         let response = StartKmsResponse {
             success,
@@ -774,8 +701,7 @@ impl Operation for GetKmsStatusHandler {
 
         let service_manager = kms_service_manager_from_context();
 
-        let status = service_manager.get_status().await;
-        let config = service_manager.get_redacted_config().await;
+        let (status, config) = service_manager.get_redacted_state().await;
 
         // Get backend type and health status
         let backend_type = config.as_ref().map(|c| c.backend.clone());
@@ -908,36 +834,27 @@ impl Operation for ReconfigureKmsHandler {
         // Convert request to KmsConfig
         let kms_config = configure_request.to_kms_config();
 
-        // Reconfigure the service (stops, reconfigures, and starts)
-        let (success, message, status) = match service_manager.reconfigure(kms_config.clone()).await {
+        let persisted_config = kms_config.clone();
+        let (success, message, status) = match service_manager
+            .reconfigure_with_persistence(kms_config, || async move {
+                save_kms_config(&persisted_config)
+                    .await
+                    .map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
+            })
+            .await
+        {
             Ok(()) => {
-                // Persist the configuration to cluster storage
-                if let Err(e) = save_kms_config(&kms_config).await {
-                    let error_msg = format!("KMS reconfigured in memory but failed to persist: {e}");
-                    error!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "reconfigure",
-                        state = "persist_failed",
-                        error = %e,
-                        "admin kms dynamic state"
-                    );
-                    let status = service_manager.get_status().await;
-                    (false, error_msg, status)
-                } else {
-                    let status = service_manager.get_status().await;
-                    info!(
-                        component = LOG_COMPONENT_ADMIN,
-                        subsystem = LOG_SUBSYSTEM_KMS,
-                        event = "kms_service_state",
-                        operation = "reconfigure",
-                        state = "reconfigured",
-                        status = ?status,
-                        "admin kms dynamic state"
-                    );
-                    (true, "KMS reconfigured and restarted successfully".to_string(), status)
-                }
+                let status = service_manager.get_status().await;
+                info!(
+                    component = LOG_COMPONENT_ADMIN,
+                    subsystem = LOG_SUBSYSTEM_KMS,
+                    event = "kms_service_state",
+                    operation = "reconfigure",
+                    state = "reconfigured",
+                    status = ?status,
+                    "admin kms dynamic state"
+                );
+                (true, "KMS reconfigured and restarted successfully".to_string(), status)
             }
             Err(e) => {
                 let error_msg = format!("Failed to reconfigure KMS: {e}");

rustfs/src/app/context/global.rs

@@ -18,13 +18,12 @@ use super::handles::{
     IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface,
     default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface,
     default_endpoints_interface, default_expiry_state_interface, default_federated_identity_interface,
-    default_internode_metrics_interface, default_kms_runtime_interface, default_local_node_name_interface,
-    default_lock_client_interface, default_lock_clients_interface, default_notification_system_interface,
-    default_notify_interface, default_outbound_tls_runtime_interface, default_performance_metrics_interface,
-    default_region_interface, default_replication_pool_interface, default_replication_stats_interface,
-    default_runtime_port_interface, default_s3select_db_interface, default_scanner_metrics_interface,
-    default_server_config_interface, default_storage_class_interface, default_tier_config_interface,
-    default_transition_state_interface,
+    default_internode_metrics_interface, default_local_node_name_interface, default_lock_client_interface,
+    default_lock_clients_interface, default_notification_system_interface, default_notify_interface,
+    default_outbound_tls_runtime_interface, default_performance_metrics_interface, default_region_interface,
+    default_replication_pool_interface, default_replication_stats_interface, default_runtime_port_interface,
+    default_s3select_db_interface, default_scanner_metrics_interface, default_server_config_interface,
+    default_storage_class_interface, default_tier_config_interface, default_transition_state_interface,
 };
 use super::interfaces::{
     ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface,
@@ -80,6 +79,7 @@ pub struct AppContext {
 impl AppContext {
     pub fn new(object_store: Arc<ECStore>, iam: Arc<dyn IamInterface>, kms: Arc<dyn KmsInterface>) -> Self {
         let object_data_cache = ObjectDataCacheAdapter::from_env_or_disabled();
+        let kms_runtime = Arc::new(crate::app::context::handles::KmsRuntimeHandle::new(kms.handle()));
         // Let ecstore probe this cache inside get_object_reader, after
         // metadata resolution but before the erasure data read (backlog#802).
         crate::app::object_data_cache::register_object_data_cache_body_hook(Arc::clone(&object_data_cache));
@@ -94,7 +94,7 @@ impl AppContext {
             iam,
             federated_identity: default_federated_identity_interface(),
             kms,
-            kms_runtime: default_kms_runtime_interface(),
+            kms_runtime,
             outbound_tls_runtime: default_outbound_tls_runtime_interface(),
             notify: default_notify_interface(),
             notification_system: default_notification_system_interface(),

rustfs/src/app/context/handles.rs

@@ -128,12 +128,19 @@ impl KmsInterface for KmsHandle {
 }
 
 /// Default KMS runtime interface adapter.
-#[derive(Default)]
-pub struct KmsRuntimeHandle;
+pub struct KmsRuntimeHandle {
+    kms: Option<Arc<KmsServiceManager>>,
+}
+
+impl KmsRuntimeHandle {
+    pub fn new(kms: Arc<KmsServiceManager>) -> Self {
+        Self { kms: Some(kms) }
+    }
+}
 
 impl KmsRuntimeInterface for KmsRuntimeHandle {
     fn service_manager(&self) -> Option<Arc<KmsServiceManager>> {
-        runtime_sources::kms_service_manager()
+        self.kms.clone()
     }
 }
 
@@ -485,7 +492,9 @@ pub fn default_notification_system_interface() -> Arc<dyn NotificationSystemInte
 }
 
 pub fn default_kms_runtime_interface() -> Arc<dyn KmsRuntimeInterface> {
-    Arc::new(KmsRuntimeHandle)
+    Arc::new(KmsRuntimeHandle {
+        kms: runtime_sources::kms_service_manager(),
+    })
 }
 
 pub fn default_outbound_tls_runtime_interface() -> Arc<dyn OutboundTlsRuntimeInterface> {
@@ -607,10 +616,10 @@ pub fn default_buffer_config_interface() -> Arc<dyn BufferConfigInterface> {
 #[cfg(test)]
 mod tests {
     use super::{
-        ServerConfigHandle, default_federated_identity_interface, federated_identity_interface,
-        publish_default_federated_identity_service, runtime_sources,
+        KmsRuntimeHandle, KmsServiceManager, ServerConfigHandle, default_federated_identity_interface,
+        federated_identity_interface, publish_default_federated_identity_service, runtime_sources,
     };
-    use crate::app::context::interfaces::ServerConfigInterface;
+    use crate::app::context::interfaces::{KmsRuntimeInterface, ServerConfigInterface};
     use rustfs_config::server_config::Config;
     use rustfs_iam::{
         federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter},
@@ -733,4 +742,19 @@ mod tests {
             "handle B must serve its own credentials"
         );
     }
+
+    #[test]
+    fn kms_runtime_handles_keep_injected_managers_isolated() {
+        let manager_a = Arc::new(KmsServiceManager::new());
+        let manager_b = Arc::new(KmsServiceManager::new());
+        let handle_a = KmsRuntimeHandle::new(manager_a.clone());
+        let handle_b = KmsRuntimeHandle::new(manager_b.clone());
+
+        assert!(Arc::ptr_eq(&handle_a.service_manager().expect("manager A"), &manager_a));
+        assert!(Arc::ptr_eq(&handle_b.service_manager().expect("manager B"), &manager_b));
+        assert!(!Arc::ptr_eq(
+            &handle_a.service_manager().expect("manager A"),
+            &handle_b.service_manager().expect("manager B")
+        ));
+    }
 }

rustfs/src/storage/minio_generated_read_test.rs

@@ -4,14 +4,13 @@ use std::fs;
 use std::io::Cursor;
 use std::path::{Path, PathBuf};
 
-mod storage_api;
-
+use super::sse::SseObjectEncryptionResolver;
+use super::storage_api::ecstore_test_support::{
+    DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
+};
 use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
 use serde::Deserialize;
 use sha2::{Digest, Sha256};
-use storage_api::minio_generated_read::{
-    DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
-};
 use temp_env::async_with_vars;
 use tokio::io::{AsyncReadExt, AsyncWrite};
 
@@ -49,7 +48,7 @@ impl AsyncWrite for VecAsyncWriter {
 fn fixture_root() -> PathBuf {
     std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
         .map(PathBuf::from)
-        .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
+        .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../crates/rio-v2/tests/fixtures/minio-generated"))
 }
 
 fn case_dir(case_id: &str) -> PathBuf {
@@ -137,12 +136,14 @@ async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms
             ("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
         ],
         async move {
-            let (mut reader, offset, length) = GetObjectReader::new(
+            let resolver = SseObjectEncryptionResolver;
+            let (mut reader, offset, length) = GetObjectReader::new_with_resolver(
                 Box::new(Cursor::new(encrypted)),
                 None,
                 &object_info,
                 &ObjectOptions::default(),
                 &http::HeaderMap::new(),
+                Some(&resolver),
             )
             .await
             .map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
@@ -219,11 +220,12 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
             readers.push(reader);
         }
 
-        let erasure = Erasure::new(
+        let erasure = Erasure::try_new(
             file_info.erasure.data_blocks,
             file_info.erasure.parity_blocks,
             file_info.erasure.block_size,
-        );
+        )
+        .expect("fixture erasure geometry");
         let mut writer = VecAsyncWriter::default();
         let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
         if let Some(err) = err {

rustfs/src/storage/mod.rs

@@ -36,6 +36,8 @@ mod ecfs_extend;
 mod ecfs_test;
 pub(crate) mod head_prefix;
 #[cfg(test)]
+mod minio_generated_read_test;
+#[cfg(test)]
 mod multi_factor_scheduler_integration_test;
 pub(crate) mod runtime_sources;
 #[cfg(test)]

rustfs/src/storage/sse.rs

@@ -70,6 +70,10 @@
 //! ```
 
 use super::StorageError;
+use super::storage_api::ecstore_object::{
+    EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
+    ReadEncryptionMode, ReadEncryptionRequest,
+};
 use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
 #[cfg(feature = "rio-v2")]
 use aes_gcm::aead::Payload;
@@ -144,6 +148,7 @@ use rustfs_utils::http::headers::{
 };
 use rustfs_utils::path::path_join_buf;
 use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId};
+use std::borrow::Cow;
 
 // ============================================================================
 // High-Level SSE Configuration
@@ -641,6 +646,23 @@ pub(crate) fn validate_sse_headers_for_read(metadata: &HashMap<String, String>,
 }
 
 pub(crate) fn map_get_object_reader_error(err: StorageError) -> ApiError {
+    if let StorageError::Io(io_error) = &err
+        && let Some(resolution_error) = io_error
+            .get_ref()
+            .and_then(|source| source.downcast_ref::<EncryptionResolutionError>())
+    {
+        let code = match resolution_error.kind() {
+            EncryptionResolutionErrorKind::InvalidRequest => S3ErrorCode::InvalidRequest,
+            EncryptionResolutionErrorKind::ServiceUnavailable => S3ErrorCode::ServiceUnavailable,
+            _ => S3ErrorCode::InternalError,
+        };
+        return ApiError {
+            code,
+            message: resolution_error.to_string(),
+            source: Some(Box::new(err)),
+        };
+    }
+
     if let Some(message) = map_ssec_get_object_reader_error_message(&err) {
         return ApiError {
             code: S3ErrorCode::InvalidRequest,
@@ -763,6 +785,104 @@ pub enum EncryptionKeyKind {
     Object,
 }
 
+pub(crate) struct SseObjectEncryptionResolver;
+
+#[async_trait]
+impl ObjectEncryptionResolver for SseObjectEncryptionResolver {
+    async fn resolve_read_material(
+        &self,
+        request: ReadEncryptionRequest<'_>,
+    ) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
+        let metadata = normalize_encryption_metadata_case(request.metadata)?;
+        let (_, customer_key, customer_key_md5) =
+            extract_ssec_params_from_headers(request.headers).map_err(map_encryption_resolution_error)?;
+        let material = sse_decryption(DecryptionRequest {
+            bucket: request.bucket,
+            key: request.object,
+            metadata: &metadata,
+            sse_customer_key: customer_key.as_ref(),
+            sse_customer_key_md5: customer_key_md5.as_ref(),
+        })
+        .await
+        .map_err(map_encryption_resolution_error)?;
+
+        Ok(material.map(|material| ReadEncryptionMaterial {
+            key_bytes: material.key_bytes,
+            mode: match material.key_kind {
+                EncryptionKeyKind::Direct => ReadEncryptionMode::Direct {
+                    base_nonce: material.base_nonce,
+                },
+                EncryptionKeyKind::Object => ReadEncryptionMode::Object,
+            },
+        }))
+    }
+}
+
+fn normalize_encryption_metadata_case(
+    metadata: &HashMap<String, String>,
+) -> Result<Cow<'_, HashMap<String, String>>, EncryptionResolutionError> {
+    const CANONICAL_KEYS: &[&str] = &[
+        "x-amz-server-side-encryption",
+        "x-amz-server-side-encryption-aws-kms-key-id",
+        "x-amz-server-side-encryption-customer-algorithm",
+        "x-amz-server-side-encryption-customer-key-md5",
+        SSEC_ORIGINAL_SIZE_HEADER,
+        INTERNAL_ENCRYPTION_KEY_ID_HEADER,
+        INTERNAL_ENCRYPTION_KEY_HEADER,
+        INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
+        INTERNAL_ENCRYPTION_IV_HEADER,
+        "x-rustfs-encryption-context",
+        "x-rustfs-encryption-tag",
+        INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
+    ];
+
+    let needs_normalization = metadata.keys().any(|key| {
+        CANONICAL_KEYS
+            .iter()
+            .any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical))
+    });
+    if !needs_normalization {
+        return Ok(Cow::Borrowed(metadata));
+    }
+
+    let mut normalized = metadata.clone();
+    for canonical in CANONICAL_KEYS {
+        let mut matching_values = metadata
+            .iter()
+            .filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value));
+        let Some(value) = matching_values.next() else {
+            continue;
+        };
+        if matching_values.any(|candidate| candidate != value) {
+            return Err(EncryptionResolutionError::new(
+                EncryptionResolutionErrorKind::InvalidMetadata,
+                format!("conflicting object encryption metadata for {canonical}"),
+            ));
+        }
+        if !normalized.contains_key(*canonical) {
+            normalized.insert((*canonical).to_string(), value.clone());
+        }
+    }
+    Ok(Cow::Owned(normalized))
+}
+
+fn map_encryption_resolution_error(error: ApiError) -> EncryptionResolutionError {
+    let kind = match error.code {
+        S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => EncryptionResolutionErrorKind::InvalidRequest,
+        S3ErrorCode::ServiceUnavailable => EncryptionResolutionErrorKind::ServiceUnavailable,
+        _ => EncryptionResolutionErrorKind::DecryptionFailed,
+    };
+    EncryptionResolutionError::new(kind, error.message)
+}
+
 #[derive(Debug, Clone)]
 pub struct ManagedSealedKey {
     #[cfg(feature = "rio-v2")]
@@ -2631,19 +2751,20 @@ fn ssec_invalid_request(message: &str) -> ApiError {
 mod tests {
     use super::{
         ApiError, DataKey, DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest,
-        INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
-        INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_IV_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
-        MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType,
-        SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material,
-        apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
-        extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse,
-        kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata,
-        reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption,
-        strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
-        validate_ssec_params, verify_ssec_key_match,
+        EncryptionResolutionErrorKind, INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
+        INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError,
+        MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
+        MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
+        ObjectEncryptionResolver, PrepareEncryptionRequest, ReadEncryptionMode, ReadEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER,
+        SSEType, SseDekProvider, SseObjectEncryptionResolver, SsecParams, StorageError, TestSseDekProvider,
+        apply_managed_decryption_material, apply_managed_encryption_material, encryption_material_to_metadata,
+        extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
+        generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata,
+        normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption,
+        sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write,
+        validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match,
     };
     #[cfg(feature = "rio-v2")]
     use super::{
@@ -2703,6 +2824,97 @@ mod tests {
         SSE_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await
     }
 
+    #[tokio::test]
+    async fn object_encryption_resolver_returns_ssec_read_material() {
+        let key = [0x31; 32];
+        let key_b64 = BASE64_STANDARD.encode(key);
+        let key_md5 = BASE64_STANDARD.encode(md5::compute(key).0);
+        let nonce = [0x42; 12];
+        let metadata = HashMap::from([
+            ("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()),
+            ("X-Amz-Server-Side-Encryption-Customer-Key-Md5".to_string(), key_md5.clone()),
+            ("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode(nonce)),
+        ]);
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key",
+            HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"),
+        );
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key-md5",
+            HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"),
+        );
+
+        let material = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &headers,
+            })
+            .await
+            .expect("SSE-C material should resolve")
+            .expect("SSE-C metadata should produce material");
+
+        assert_eq!(material.key_bytes, key);
+        assert_eq!(material.mode, ReadEncryptionMode::Direct { base_nonce: nonce });
+    }
+
+    #[tokio::test]
+    async fn object_encryption_resolver_classifies_missing_ssec_key_as_invalid_request() {
+        let metadata = HashMap::from([("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string())]);
+        let result = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &HeaderMap::new(),
+            })
+            .await;
+        let error = match result {
+            Err(error) => error,
+            Ok(_) => panic!("missing SSE-C key must fail closed"),
+        };
+
+        assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest);
+    }
+
+    #[tokio::test]
+    async fn object_encryption_resolver_rejects_conflicting_metadata_case_variants() {
+        let metadata = HashMap::from([
+            ("x-rustfs-encryption-key".to_string(), "first".to_string()),
+            ("X-Rustfs-Encryption-Key".to_string(), "second".to_string()),
+        ]);
+        let result = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &HeaderMap::new(),
+            })
+            .await;
+        let error = match result {
+            Err(error) => error,
+            Ok(_) => panic!("conflicting metadata aliases must fail closed"),
+        };
+
+        assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidMetadata);
+    }
+
+    #[test]
+    fn normalize_encryption_metadata_case_accepts_lowercase_minio_internal_keys() {
+        let lowercase_key = MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase();
+        let metadata = HashMap::from([(lowercase_key, "sealed-key".to_string())]);
+
+        let normalized = super::normalize_encryption_metadata_case(&metadata).expect("metadata aliases should normalize");
+
+        assert_eq!(
+            normalized.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER),
+            Some(&"sealed-key".to_string())
+        );
+    }
+
     struct UnavailableSseDekProvider;
 
     #[async_trait::async_trait]
@@ -3725,6 +3937,19 @@ mod tests {
 
                 assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
                 assert_eq!(decrypted.key_bytes, material.key_bytes);
+
+                let resolved = SseObjectEncryptionResolver
+                    .resolve_read_material(ReadEncryptionRequest {
+                        bucket: "bucket",
+                        object: "object",
+                        metadata: &metadata,
+                        headers: &HeaderMap::new(),
+                    })
+                    .await
+                    .expect("managed resolver")
+                    .expect("managed material");
+                assert_eq!(resolved.mode, ReadEncryptionMode::Object);
+                assert_eq!(resolved.key_bytes, material.key_bytes);
             },
         )
         .await;
@@ -3785,6 +4010,29 @@ mod tests {
 
         assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
         assert_eq!(decrypted.key_bytes, material.key_bytes);
+
+        let mut headers = HeaderMap::new();
+        headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key",
+            HeaderValue::from_str(&customer_key).expect("customer key header"),
+        );
+        headers.insert(
+            "x-amz-server-side-encryption-customer-key-md5",
+            HeaderValue::from_str(&customer_key_md5).expect("customer key MD5 header"),
+        );
+        let resolved = SseObjectEncryptionResolver
+            .resolve_read_material(ReadEncryptionRequest {
+                bucket: "bucket",
+                object: "object",
+                metadata: &metadata,
+                headers: &headers,
+            })
+            .await
+            .expect("SSE-C resolver")
+            .expect("SSE-C material");
+        assert_eq!(resolved.mode, ReadEncryptionMode::Object);
+        assert_eq!(resolved.key_bytes, material.key_bytes);
     }
 
     #[cfg(feature = "rio-v2")]
@@ -4715,6 +4963,15 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_map_get_object_reader_error_preserves_typed_service_unavailable() {
+        let resolution_error =
+            super::EncryptionResolutionError::new(EncryptionResolutionErrorKind::ServiceUnavailable, "KMS unavailable");
+        let err = map_get_object_reader_error(StorageError::other(resolution_error));
+        assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
+        assert_eq!(err.message, "KMS unavailable");
+    }
+
     #[test]
     fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() {
         let err = map_get_object_reader_error(StorageError::other("plain io failure"));

rustfs/src/storage/storage_api.rs

@@ -510,12 +510,21 @@ pub(crate) mod ecstore_object {
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource;
     pub(crate) use rustfs_ecstore::api::object::{
-        GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len,
-        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
-        unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
+        EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup,
+        ObjectEncryptionResolver, ObjectMutationHook, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
+        get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
+        register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
     };
 }
 
+#[cfg(all(test, feature = "rio-v2"))]
+pub(crate) mod ecstore_test_support {
+    pub(crate) use rustfs_ecstore::api::bitrot::create_bitrot_reader;
+    pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
+    pub(crate) use rustfs_ecstore::api::erasure::Erasure;
+    pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions};
+}
+
 pub(crate) mod ecstore_set_disk {
     pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class};
 }
@@ -945,13 +954,21 @@ pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Res
 /// The process-level bootstrap instance context that single-instance startup
 /// threads through the storage foundation (Phase 5 follow-up, backlog#1052).
 pub(crate) fn bootstrap_instance_ctx() -> Arc<InstanceContext> {
-    ecstore_runtime::bootstrap_ctx()
+    let context = ecstore_runtime::bootstrap_ctx();
+    configure_object_encryption_resolver(&context);
+    context
 }
 
 /// Construct a fresh per-server instance context (backlog#1052 S5): a second
 /// embedded server owns its own erasure/region/endpoint/deployment id cells.
 pub(crate) fn new_instance_ctx() -> Arc<InstanceContext> {
-    Arc::new(InstanceContext::new())
+    let context = Arc::new(InstanceContext::new());
+    configure_object_encryption_resolver(&context);
+    context
+}
+
+fn configure_object_encryption_resolver(context: &InstanceContext) {
+    let _ = context.set_object_encryption_resolver(Arc::new(super::sse::SseObjectEncryptionResolver));
 }
 
 pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) {
@@ -1713,7 +1730,7 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc<ECStor
 mod tests {
     use super::{
         apply_active_resync_intents, bucket_targets_metadata_lock_shard, ecstore_bucket, lock_bucket_targets_metadata,
-        scanner_maintenance_config_file,
+        new_instance_ctx, scanner_maintenance_config_file,
     };
     use std::time::Duration;
 
@@ -1744,6 +1761,16 @@ mod tests {
         );
     }
 
+    #[test]
+    fn fresh_instance_context_installs_object_encryption_resolver() {
+        assert!(new_instance_ctx().object_encryption_resolver().is_some());
+    }
+
+    #[test]
+    fn bootstrap_instance_context_installs_object_encryption_resolver() {
+        assert!(super::bootstrap_instance_ctx().object_encryption_resolver().is_some());
+    }
+
     #[test]
     fn scanner_maintenance_config_only_includes_scanner_owned_work() {
         assert!(scanner_maintenance_config_file(ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG));

scripts/run_ecstore_validation_suite.sh

@@ -317,7 +317,7 @@ write_blackbox_matrix() {
 	    printf 'quick\theal degraded erasure disk rebuild\tblack-box\tcargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status"
 	    printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tcargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status"
 	    printf 'full\tlegacy bitrot read fixture restore\tfixture\tcargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture\tRUSTFS_LEGACY_TEST_ROOT,RUSTFS_LEGACY_TEST_DISK\t%s\n' "$legacy_status"
-	    printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
+	    printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
 	    printf 'full\tS3 multipart range versioning delete subset\tblack-box\tenv TESTEXPR=\"multipart or range or versioning or delete\" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh\tnone\t%s\n' "$s3_status"
 	    printf 'destructive\tdistributed cluster concurrency\tblack-box\tcargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status"
 	    printf 'destructive\tstale multipart cleanup cluster\tblack-box\tcargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status"
@@ -377,7 +377,7 @@ run_fixture_steps() {
 
 	  if fixture_available; then
 	    run_step "ecstore-minio-generated-read-fixture" \
-	      cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored --nocapture
+	      cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture
 	  elif [[ "$REQUIRE_FIXTURES" == "true" ]]; then
 	    echo "ERROR: $(minio_fixture_missing_reason)" >&2
 	    exit 1