mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-06 21:33:14 +00:00
refactor(sse): decouple ecstore and harden KMS lifecycle (#5435)
* refactor(sse): decouple encryption from ecstore * feat(kms): enhance KMS service manager with runtime state and persistence support * feat(kms): add local key export functionality for SSE-S3 migration tests * fix(kms): keep local key export narrowly scoped * fix(sse): validate copy source customer algorithm --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
@@ -393,36 +393,27 @@ impl Operation for ConfigureKmsHandler {
|
||||
// Convert request to KmsConfig
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
// Configure the service
|
||||
let (success, message, status) = match service_manager.configure(kms_config.clone()).await {
|
||||
let persisted_config = kms_config.clone();
|
||||
let (success, message, status) = match service_manager
|
||||
.configure_with_persistence(kms_config, || async move {
|
||||
save_kms_config(&persisted_config)
|
||||
.await
|
||||
.map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
// Persist the configuration to cluster storage
|
||||
if let Err(e) = save_kms_config(&kms_config).await {
|
||||
let error_msg = format!("KMS configured in memory but failed to persist: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "configure",
|
||||
state = "persist_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
} else {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "configure",
|
||||
state = "configured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS configured successfully".to_string(), status)
|
||||
}
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "configure",
|
||||
state = "configured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS configured successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to configure KMS: {e}");
|
||||
@@ -529,125 +520,61 @@ impl Operation for StartKmsHandler {
|
||||
);
|
||||
|
||||
let service_manager = kms_service_manager_from_context();
|
||||
|
||||
// Check if already running and force flag
|
||||
let current_status = service_manager.get_status().await;
|
||||
if matches!(current_status, KmsServiceStatus::Running) && !start_request.force.unwrap_or(false) {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "already_running",
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let response = StartKmsResponse {
|
||||
success: false,
|
||||
message: "KMS service is already running. Use force=true to restart.".to_string(),
|
||||
status: current_status,
|
||||
};
|
||||
let json_response = match serde_json::to_string(&response) {
|
||||
Ok(json) => json,
|
||||
Err(e) => {
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
|
||||
operation = "start",
|
||||
result = "response_serialize_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
return Ok(S3Response::new((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Body::from("Serialization error".to_string()),
|
||||
)));
|
||||
}
|
||||
};
|
||||
return Ok(S3Response::new((StatusCode::OK, Body::from(json_response))));
|
||||
}
|
||||
|
||||
// Start the service (or restart if force=true)
|
||||
let (success, message, status) =
|
||||
if start_request.force.unwrap_or(false) && matches!(current_status, KmsServiceStatus::Running) {
|
||||
// Force restart
|
||||
match service_manager.stop().await {
|
||||
Ok(()) => match service_manager.start().await {
|
||||
Ok(()) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service restarted successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to restart KMS service: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to stop KMS service for restart: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "stop_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Normal start
|
||||
match service_manager.start().await {
|
||||
Ok(()) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service started successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to start KMS service: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
}
|
||||
};
|
||||
let force = start_request.force.unwrap_or(false);
|
||||
let (success, message, status) = match service_manager.start_or_restart(force).await {
|
||||
Ok(rustfs_kms::KmsStartOutcome::Started) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service started successfully".to_string(), status)
|
||||
}
|
||||
Ok(rustfs_kms::KmsStartOutcome::Restarted) => {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "restart",
|
||||
state = "running",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS service restarted successfully".to_string(), status)
|
||||
}
|
||||
Ok(rustfs_kms::KmsStartOutcome::AlreadyRunning) => {
|
||||
let status = service_manager.get_status().await;
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "already_running",
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(false, "KMS service is already running. Use force=true to restart.".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to start or restart KMS service: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "start",
|
||||
state = "start_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
}
|
||||
};
|
||||
|
||||
let response = StartKmsResponse {
|
||||
success,
|
||||
@@ -804,8 +731,7 @@ impl Operation for GetKmsStatusHandler {
|
||||
|
||||
let service_manager = kms_service_manager_from_context();
|
||||
|
||||
let status = service_manager.get_status().await;
|
||||
let config = service_manager.get_redacted_config().await;
|
||||
let (status, config) = service_manager.get_redacted_state().await;
|
||||
|
||||
// Get backend type and health status
|
||||
let backend_type = config.as_ref().map(|c| c.backend.clone());
|
||||
@@ -938,36 +864,27 @@ impl Operation for ReconfigureKmsHandler {
|
||||
// Convert request to KmsConfig
|
||||
let kms_config = configure_request.to_kms_config();
|
||||
|
||||
// Reconfigure the service (stops, reconfigures, and starts)
|
||||
let (success, message, status) = match service_manager.reconfigure(kms_config.clone()).await {
|
||||
let persisted_config = kms_config.clone();
|
||||
let (success, message, status) = match service_manager
|
||||
.reconfigure_with_persistence(kms_config, || async move {
|
||||
save_kms_config(&persisted_config)
|
||||
.await
|
||||
.map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
// Persist the configuration to cluster storage
|
||||
if let Err(e) = save_kms_config(&kms_config).await {
|
||||
let error_msg = format!("KMS reconfigured in memory but failed to persist: {e}");
|
||||
error!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "reconfigure",
|
||||
state = "persist_failed",
|
||||
error = %e,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
let status = service_manager.get_status().await;
|
||||
(false, error_msg, status)
|
||||
} else {
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "reconfigure",
|
||||
state = "reconfigured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS reconfigured and restarted successfully".to_string(), status)
|
||||
}
|
||||
let status = service_manager.get_status().await;
|
||||
info!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_KMS,
|
||||
event = "kms_service_state",
|
||||
operation = "reconfigure",
|
||||
state = "reconfigured",
|
||||
status = ?status,
|
||||
"admin kms dynamic state"
|
||||
);
|
||||
(true, "KMS reconfigured and restarted successfully".to_string(), status)
|
||||
}
|
||||
Err(e) => {
|
||||
let error_msg = format!("Failed to reconfigure KMS: {e}");
|
||||
|
||||
@@ -18,13 +18,12 @@ use super::handles::{
|
||||
IamHandle, KmsHandle, default_action_credential_interface, default_boot_time_interface, default_bucket_metadata_interface,
|
||||
default_bucket_monitor_interface, default_buffer_config_interface, default_deployment_id_interface,
|
||||
default_endpoints_interface, default_expiry_state_interface, default_federated_identity_interface,
|
||||
default_internode_metrics_interface, default_kms_runtime_interface, default_local_node_name_interface,
|
||||
default_lock_client_interface, default_lock_clients_interface, default_notification_system_interface,
|
||||
default_notify_interface, default_outbound_tls_runtime_interface, default_performance_metrics_interface,
|
||||
default_region_interface, default_replication_pool_interface, default_replication_stats_interface,
|
||||
default_runtime_port_interface, default_s3select_db_interface, default_scanner_metrics_interface,
|
||||
default_server_config_interface, default_storage_class_interface, default_tier_config_interface,
|
||||
default_transition_state_interface,
|
||||
default_internode_metrics_interface, default_local_node_name_interface, default_lock_client_interface,
|
||||
default_lock_clients_interface, default_notification_system_interface, default_notify_interface,
|
||||
default_outbound_tls_runtime_interface, default_performance_metrics_interface, default_region_interface,
|
||||
default_replication_pool_interface, default_replication_stats_interface, default_runtime_port_interface,
|
||||
default_s3select_db_interface, default_scanner_metrics_interface, default_server_config_interface,
|
||||
default_storage_class_interface, default_tier_config_interface, default_transition_state_interface,
|
||||
};
|
||||
use super::interfaces::{
|
||||
ActionCredentialInterface, BootTimeInterface, BucketMetadataInterface, BucketMonitorInterface, BufferConfigInterface,
|
||||
@@ -80,6 +79,7 @@ pub struct AppContext {
|
||||
impl AppContext {
|
||||
pub fn new(object_store: Arc<ECStore>, iam: Arc<dyn IamInterface>, kms: Arc<dyn KmsInterface>) -> Self {
|
||||
let object_data_cache = ObjectDataCacheAdapter::from_env_or_disabled();
|
||||
let kms_runtime = Arc::new(crate::app::context::handles::KmsRuntimeHandle::new(kms.handle()));
|
||||
// Let ecstore probe this cache inside get_object_reader, after
|
||||
// metadata resolution but before the erasure data read (backlog#802).
|
||||
crate::app::object_data_cache::register_object_data_cache_body_hook(Arc::clone(&object_data_cache));
|
||||
@@ -94,7 +94,7 @@ impl AppContext {
|
||||
iam,
|
||||
federated_identity: default_federated_identity_interface(),
|
||||
kms,
|
||||
kms_runtime: default_kms_runtime_interface(),
|
||||
kms_runtime,
|
||||
outbound_tls_runtime: default_outbound_tls_runtime_interface(),
|
||||
notify: default_notify_interface(),
|
||||
notification_system: default_notification_system_interface(),
|
||||
|
||||
@@ -128,12 +128,19 @@ impl KmsInterface for KmsHandle {
|
||||
}
|
||||
|
||||
/// Default KMS runtime interface adapter.
|
||||
#[derive(Default)]
|
||||
pub struct KmsRuntimeHandle;
|
||||
pub struct KmsRuntimeHandle {
|
||||
kms: Option<Arc<KmsServiceManager>>,
|
||||
}
|
||||
|
||||
impl KmsRuntimeHandle {
|
||||
pub fn new(kms: Arc<KmsServiceManager>) -> Self {
|
||||
Self { kms: Some(kms) }
|
||||
}
|
||||
}
|
||||
|
||||
impl KmsRuntimeInterface for KmsRuntimeHandle {
|
||||
fn service_manager(&self) -> Option<Arc<KmsServiceManager>> {
|
||||
runtime_sources::kms_service_manager()
|
||||
self.kms.clone()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -485,7 +492,9 @@ pub fn default_notification_system_interface() -> Arc<dyn NotificationSystemInte
|
||||
}
|
||||
|
||||
pub fn default_kms_runtime_interface() -> Arc<dyn KmsRuntimeInterface> {
|
||||
Arc::new(KmsRuntimeHandle)
|
||||
Arc::new(KmsRuntimeHandle {
|
||||
kms: runtime_sources::kms_service_manager(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn default_outbound_tls_runtime_interface() -> Arc<dyn OutboundTlsRuntimeInterface> {
|
||||
@@ -607,10 +616,10 @@ pub fn default_buffer_config_interface() -> Arc<dyn BufferConfigInterface> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ServerConfigHandle, default_federated_identity_interface, federated_identity_interface,
|
||||
publish_default_federated_identity_service, runtime_sources,
|
||||
KmsRuntimeHandle, KmsServiceManager, ServerConfigHandle, default_federated_identity_interface,
|
||||
federated_identity_interface, publish_default_federated_identity_service, runtime_sources,
|
||||
};
|
||||
use crate::app::context::interfaces::ServerConfigInterface;
|
||||
use crate::app::context::interfaces::{KmsRuntimeInterface, ServerConfigInterface};
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_iam::{
|
||||
federation::{FederatedIdentityRegistry, FederatedIdentityService, oidc::StandardOidcAdapter},
|
||||
@@ -733,4 +742,19 @@ mod tests {
|
||||
"handle B must serve its own credentials"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kms_runtime_handles_keep_injected_managers_isolated() {
|
||||
let manager_a = Arc::new(KmsServiceManager::new());
|
||||
let manager_b = Arc::new(KmsServiceManager::new());
|
||||
let handle_a = KmsRuntimeHandle::new(manager_a.clone());
|
||||
let handle_b = KmsRuntimeHandle::new(manager_b.clone());
|
||||
|
||||
assert!(Arc::ptr_eq(&handle_a.service_manager().expect("manager A"), &manager_a));
|
||||
assert!(Arc::ptr_eq(&handle_b.service_manager().expect("manager B"), &manager_b));
|
||||
assert!(!Arc::ptr_eq(
|
||||
&handle_a.service_manager().expect("manager A"),
|
||||
&handle_b.service_manager().expect("manager B")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
#![cfg(feature = "rio-v2")]
|
||||
|
||||
use std::fs;
|
||||
use std::io::Cursor;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::sse::SseObjectEncryptionResolver;
|
||||
use super::storage_api::ecstore_test_support::{
|
||||
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
|
||||
};
|
||||
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use temp_env::async_with_vars;
|
||||
use tokio::io::{AsyncReadExt, AsyncWrite};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManifestRecord {
|
||||
bucket: String,
|
||||
object: String,
|
||||
backend_files: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct VecAsyncWriter {
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for VecAsyncWriter {
|
||||
fn poll_write(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
_cx: &mut std::task::Context<'_>,
|
||||
buf: &[u8],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
self.bytes.extend_from_slice(buf);
|
||||
std::task::Poll::Ready(Ok(buf.len()))
|
||||
}
|
||||
|
||||
fn poll_flush(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<std::io::Result<()>> {
|
||||
std::task::Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_root() -> PathBuf {
|
||||
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../crates/rio-v2/tests/fixtures/minio-generated"))
|
||||
}
|
||||
|
||||
fn case_dir(case_id: &str) -> PathBuf {
|
||||
fixture_root().join("cases").join(case_id)
|
||||
}
|
||||
|
||||
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
|
||||
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
|
||||
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
|
||||
}
|
||||
|
||||
fn require_fixture_case(case_id: &str) -> PathBuf {
|
||||
let path = case_dir(case_id);
|
||||
assert!(
|
||||
path.is_dir(),
|
||||
"fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
|
||||
path.display()
|
||||
);
|
||||
path
|
||||
}
|
||||
|
||||
fn read_plaintext_sha256(case_dir: &Path) -> String {
|
||||
fs::read_to_string(case_dir.join("plaintext.sha256"))
|
||||
.unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
|
||||
.trim()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn minio_static_kms_key_b64() -> String {
|
||||
std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
|
||||
.unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
|
||||
}
|
||||
|
||||
fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
|
||||
let expected = format!("disk1/{}/{}/xl.meta", manifest.bucket, manifest.object);
|
||||
let relative = manifest
|
||||
.backend_files
|
||||
.iter()
|
||||
.find(|entry| entry.as_str() == expected)
|
||||
.unwrap_or_else(|| panic!("object xl.meta missing from manifest backend_files: {expected}"));
|
||||
case_dir.join("backend").join(relative)
|
||||
}
|
||||
|
||||
fn load_file_info(case_dir: &Path, manifest: &ManifestRecord) -> FileInfo {
|
||||
let xl_meta_path = object_xl_meta_path(case_dir, manifest);
|
||||
let xl_meta = fs::read(&xl_meta_path).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta_path.display()));
|
||||
get_file_info(
|
||||
&xl_meta,
|
||||
&manifest.bucket,
|
||||
&manifest.object,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: true,
|
||||
include_free_versions: true,
|
||||
},
|
||||
)
|
||||
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta_path.display()))
|
||||
}
|
||||
|
||||
fn load_object_info(file_info: &FileInfo, manifest: &ManifestRecord) -> ObjectInfo {
|
||||
ObjectInfo::from_file_info(file_info, &manifest.bucket, &manifest.object, false)
|
||||
}
|
||||
|
||||
fn sha256_hex(bytes: &[u8]) -> String {
|
||||
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String) {
|
||||
let case_dir = require_fixture_case(case_id);
|
||||
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
|
||||
let expected_sha256 = read_plaintext_sha256(&case_dir);
|
||||
let file_info = load_file_info(&case_dir, &manifest);
|
||||
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
|
||||
let object_info = load_object_info(&file_info, &manifest);
|
||||
|
||||
(object_info, encrypted, expected_sha256)
|
||||
}
|
||||
|
||||
async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms_key_b64: String) -> Result<Vec<u8>, String> {
|
||||
let object_size = object_info.size;
|
||||
|
||||
async_with_vars(
|
||||
[
|
||||
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
|
||||
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
|
||||
],
|
||||
async move {
|
||||
let resolver = SseObjectEncryptionResolver;
|
||||
let (mut reader, offset, length) = GetObjectReader::new_with_resolver(
|
||||
Box::new(Cursor::new(encrypted)),
|
||||
None,
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&http::HeaderMap::new(),
|
||||
Some(&resolver),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
|
||||
|
||||
if offset != 0 || length != object_size {
|
||||
return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}"));
|
||||
}
|
||||
|
||||
let mut plaintext = Vec::new();
|
||||
reader
|
||||
.read_to_end(&mut plaintext)
|
||||
.await
|
||||
.map_err(|err| format!("read plaintext from MinIO raw fixture: {err}"))?;
|
||||
|
||||
Ok(plaintext)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
|
||||
let mut disks = Vec::with_capacity(file_info.erasure.distribution.len());
|
||||
for disk_number in 1..=file_info.erasure.distribution.len() {
|
||||
let disk_root = case_dir.join("backend").join(format!("disk{disk_number}"));
|
||||
let disk_root_str = disk_root
|
||||
.to_str()
|
||||
.unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
|
||||
let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_number - 1);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("open fixture disk {disk_number}: {err}"));
|
||||
disks.push(disk);
|
||||
}
|
||||
let mut disk_order = vec![None; disks.len()];
|
||||
for (idx, disk) in disks.iter().enumerate() {
|
||||
let block_index = file_info.erasure.distribution[idx];
|
||||
disk_order[block_index - 1] = Some(disk);
|
||||
}
|
||||
let data_dir = file_info
|
||||
.data_dir
|
||||
.as_ref()
|
||||
.unwrap_or_else(|| panic!("fixture {} is missing data_dir", manifest.object));
|
||||
|
||||
let mut encrypted = Vec::new();
|
||||
for part in &file_info.parts {
|
||||
let checksum_info = file_info.erasure.get_checksum_info(part.number);
|
||||
let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
|
||||
let shard_read_len = file_info.erasure.shard_file_size(part.size as i64);
|
||||
let mut readers = Vec::with_capacity(disks.len());
|
||||
for (idx, disk) in disk_order.iter().enumerate() {
|
||||
let reader = create_bitrot_reader(
|
||||
None,
|
||||
*disk,
|
||||
&manifest.bucket,
|
||||
&path,
|
||||
0,
|
||||
shard_read_len as usize,
|
||||
file_info.erasure.shard_size(),
|
||||
checksum_info.algorithm.clone(),
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("create bitrot reader for disk{} {path}: {err:?}", idx + 1));
|
||||
readers.push(reader);
|
||||
}
|
||||
|
||||
let erasure = Erasure::try_new(
|
||||
file_info.erasure.data_blocks,
|
||||
file_info.erasure.parity_blocks,
|
||||
file_info.erasure.block_size,
|
||||
)
|
||||
.expect("fixture erasure geometry");
|
||||
let mut writer = VecAsyncWriter::default();
|
||||
let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
|
||||
if let Some(err) = err {
|
||||
panic!("decode erasure shards for {path}: {err}");
|
||||
}
|
||||
assert_eq!(written, part.size, "decoded part size should match xl.meta part size");
|
||||
encrypted.extend_from_slice(&writer.bytes);
|
||||
}
|
||||
for disk in disks {
|
||||
disk.close().await.expect("close fixture disk");
|
||||
}
|
||||
encrypted
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
|
||||
async fn reads_minio_generated_sse_s3_multipart_fixture() {
|
||||
assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
|
||||
async fn reads_minio_generated_sse_kms_multipart_fixture() {
|
||||
assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
|
||||
async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
|
||||
let (object_info, encrypted, _) = load_fixture_reader_input("sse-s3-multipart-8m").await;
|
||||
let wrong_key_b64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
|
||||
|
||||
let result = read_fixture_plaintext(encrypted, object_info, wrong_key_b64).await;
|
||||
|
||||
assert!(result.is_err(), "wrong KMS key must fail closed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
|
||||
async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {
|
||||
let (object_info, mut encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await;
|
||||
encrypted.truncate(encrypted.len() / 2);
|
||||
|
||||
let result = read_fixture_plaintext(encrypted, object_info, minio_static_kms_key_b64()).await;
|
||||
|
||||
if let Ok(plaintext) = result {
|
||||
assert_ne!(
|
||||
sha256_hex(&plaintext),
|
||||
expected_sha256,
|
||||
"truncated ciphertext must not restore the original plaintext"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
|
||||
let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
|
||||
// `ObjectInfo.size` is the on-disk size. For SSE objects that is the
|
||||
// DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is
|
||||
// deliberately larger than the logical object size. The size a client sees
|
||||
// (and what MinIO records via `x-*-internal-actual-size`) comes from
|
||||
// `decrypted_size()`/`get_actual_size()`, so assert against that — the raw
|
||||
// `size` field would never equal the plaintext length for encrypted objects.
|
||||
let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata");
|
||||
let kms_key_b64 = minio_static_kms_key_b64();
|
||||
|
||||
let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
|
||||
.await
|
||||
.expect("fixture must restore with the configured KMS key");
|
||||
|
||||
assert_eq!(decrypted_size, expected_size);
|
||||
assert_eq!(plaintext.len(), expected_size as usize);
|
||||
assert_eq!(sha256_hex(&plaintext), expected_sha256);
|
||||
}
|
||||
@@ -36,6 +36,8 @@ mod ecfs_extend;
|
||||
mod ecfs_test;
|
||||
pub(crate) mod head_prefix;
|
||||
#[cfg(test)]
|
||||
mod minio_generated_read_test;
|
||||
#[cfg(test)]
|
||||
mod multi_factor_scheduler_integration_test;
|
||||
pub(crate) mod runtime_sources;
|
||||
#[cfg(test)]
|
||||
|
||||
+339
-40
@@ -70,6 +70,10 @@
|
||||
//! ```
|
||||
|
||||
use super::StorageError;
|
||||
use super::storage_api::ecstore_object::{
|
||||
EncryptionResolutionError, EncryptionResolutionErrorKind, ObjectEncryptionResolver, ReadEncryptionMaterial,
|
||||
ReadEncryptionMode, ReadEncryptionRequest,
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use aes_gcm::aead::Payload;
|
||||
@@ -155,6 +159,7 @@ use rustfs_utils::http::headers::{
|
||||
};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId};
|
||||
use std::borrow::Cow;
|
||||
|
||||
// ============================================================================
|
||||
// High-Level SSE Configuration
|
||||
@@ -652,6 +657,23 @@ pub(crate) fn validate_sse_headers_for_read(metadata: &HashMap<String, String>,
|
||||
}
|
||||
|
||||
pub(crate) fn map_get_object_reader_error(err: StorageError) -> ApiError {
|
||||
if let StorageError::Io(io_error) = &err
|
||||
&& let Some(resolution_error) = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<EncryptionResolutionError>())
|
||||
{
|
||||
let code = match resolution_error.kind() {
|
||||
EncryptionResolutionErrorKind::InvalidRequest => S3ErrorCode::InvalidRequest,
|
||||
EncryptionResolutionErrorKind::ServiceUnavailable => S3ErrorCode::ServiceUnavailable,
|
||||
_ => S3ErrorCode::InternalError,
|
||||
};
|
||||
return ApiError {
|
||||
code,
|
||||
message: resolution_error.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
}
|
||||
|
||||
if let Some(message) = map_ssec_get_object_reader_error_message(&err) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
@@ -774,6 +796,117 @@ pub enum EncryptionKeyKind {
|
||||
Object,
|
||||
}
|
||||
|
||||
pub(crate) struct SseObjectEncryptionResolver;
|
||||
|
||||
#[async_trait]
|
||||
impl ObjectEncryptionResolver for SseObjectEncryptionResolver {
|
||||
async fn resolve_read_material(
|
||||
&self,
|
||||
request: ReadEncryptionRequest<'_>,
|
||||
) -> Result<Option<ReadEncryptionMaterial>, EncryptionResolutionError> {
|
||||
let metadata = normalize_encryption_metadata_case(request.metadata)?;
|
||||
let (customer_algorithm, customer_key, customer_key_md5) =
|
||||
extract_ssec_params_from_headers(request.headers).map_err(map_encryption_resolution_error)?;
|
||||
if let Some(stored_algorithm) = metadata.get("x-amz-server-side-encryption-customer-algorithm") {
|
||||
let request_algorithm = customer_algorithm.as_ref().ok_or_else(|| {
|
||||
map_encryption_resolution_error(ssec_invalid_request(
|
||||
"The object was stored using a form of Server Side Encryption. \
|
||||
The correct parameters must be provided to retrieve the object.",
|
||||
))
|
||||
})?;
|
||||
if stored_algorithm != request_algorithm.as_str() {
|
||||
return Err(map_encryption_resolution_error(ssec_invalid_request(
|
||||
"The provided encryption parameters did not match the ones used originally to encrypt the object.",
|
||||
)));
|
||||
}
|
||||
}
|
||||
let material = sse_decryption(DecryptionRequest {
|
||||
bucket: request.bucket,
|
||||
key: request.object,
|
||||
metadata: &metadata,
|
||||
sse_customer_key: customer_key.as_ref(),
|
||||
sse_customer_key_md5: customer_key_md5.as_ref(),
|
||||
})
|
||||
.await
|
||||
.map_err(map_encryption_resolution_error)?;
|
||||
|
||||
Ok(material.map(|material| ReadEncryptionMaterial {
|
||||
key_bytes: material.key_bytes,
|
||||
mode: match material.key_kind {
|
||||
EncryptionKeyKind::Direct => ReadEncryptionMode::Direct {
|
||||
base_nonce: material.base_nonce,
|
||||
},
|
||||
EncryptionKeyKind::Object => ReadEncryptionMode::Object,
|
||||
},
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_encryption_metadata_case(
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<Cow<'_, HashMap<String, String>>, EncryptionResolutionError> {
|
||||
const CANONICAL_KEYS: &[&str] = &[
|
||||
"x-amz-server-side-encryption",
|
||||
"x-amz-server-side-encryption-aws-kms-key-id",
|
||||
"x-amz-server-side-encryption-customer-algorithm",
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
SSEC_ORIGINAL_SIZE_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
"x-rustfs-encryption-context",
|
||||
"x-rustfs-encryption-tag",
|
||||
INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
];
|
||||
|
||||
let needs_normalization = metadata.keys().any(|key| {
|
||||
CANONICAL_KEYS
|
||||
.iter()
|
||||
.any(|canonical| key != canonical && key.eq_ignore_ascii_case(canonical))
|
||||
});
|
||||
if !needs_normalization {
|
||||
return Ok(Cow::Borrowed(metadata));
|
||||
}
|
||||
|
||||
let mut normalized = metadata.clone();
|
||||
for canonical in CANONICAL_KEYS {
|
||||
let mut matching_values = metadata
|
||||
.iter()
|
||||
.filter_map(|(key, value)| key.eq_ignore_ascii_case(canonical).then_some(value));
|
||||
let Some(value) = matching_values.next() else {
|
||||
continue;
|
||||
};
|
||||
if matching_values.any(|candidate| candidate != value) {
|
||||
return Err(EncryptionResolutionError::new(
|
||||
EncryptionResolutionErrorKind::InvalidMetadata,
|
||||
format!("conflicting object encryption metadata for {canonical}"),
|
||||
));
|
||||
}
|
||||
if !normalized.contains_key(*canonical) {
|
||||
normalized.insert((*canonical).to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Ok(Cow::Owned(normalized))
|
||||
}
|
||||
|
||||
fn map_encryption_resolution_error(error: ApiError) -> EncryptionResolutionError {
|
||||
let kind = match error.code {
|
||||
S3ErrorCode::InvalidArgument | S3ErrorCode::InvalidRequest => EncryptionResolutionErrorKind::InvalidRequest,
|
||||
S3ErrorCode::ServiceUnavailable => EncryptionResolutionErrorKind::ServiceUnavailable,
|
||||
_ => EncryptionResolutionErrorKind::DecryptionFailed,
|
||||
};
|
||||
EncryptionResolutionError::new(kind, error.message)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ManagedSealedKey {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -2642,19 +2775,20 @@ fn ssec_invalid_request(message: &str) -> ApiError {
|
||||
mod tests {
|
||||
use super::{
|
||||
ApiError, DataKey, DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest,
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_IV_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType,
|
||||
SseDekProvider, SsecParams, StorageError, TestSseDekProvider, apply_managed_decryption_material,
|
||||
apply_managed_encryption_material, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse,
|
||||
kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata, md5_base64,
|
||||
normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption, sse_encryption,
|
||||
sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, validate_sse_headers_for_write,
|
||||
validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match,
|
||||
EncryptionResolutionErrorKind, INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, KmsUnavailableError,
|
||||
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
|
||||
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
|
||||
ObjectEncryptionResolver, PrepareEncryptionRequest, ReadEncryptionMode, ReadEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER,
|
||||
SSEType, SseDekProvider, SseObjectEncryptionResolver, SsecParams, StorageError, TestSseDekProvider,
|
||||
apply_managed_decryption_material, apply_managed_encryption_material, encryption_material_to_metadata,
|
||||
extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers,
|
||||
generate_ssec_nonce, is_managed_sse, kms_operation_error, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
md5_base64, normalize_managed_metadata, reset_sse_dek_provider, resolve_effective_kms_key_id, sse_decryption,
|
||||
sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
|
||||
validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, verify_ssec_key_match,
|
||||
};
|
||||
#[cfg(feature = "rio-v2")]
|
||||
use super::{
|
||||
@@ -2709,11 +2843,154 @@ mod tests {
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
static SSE_TEST_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
static SSE_TEST_KMS_KEY_DIR: OnceLock<tempfile::TempDir> = OnceLock::new();
|
||||
|
||||
async fn lock_sse_test_state() -> tokio::sync::MutexGuard<'static, ()> {
|
||||
SSE_TEST_LOCK.get_or_init(|| Mutex::new(())).lock().await
|
||||
}
|
||||
|
||||
async fn configure_test_global_local_kms() -> Arc<rustfs_kms::KmsServiceManager> {
|
||||
let key_dir = SSE_TEST_KMS_KEY_DIR.get_or_init(|| tempfile::TempDir::new().expect("create KMS key directory"));
|
||||
let manager = rustfs_kms::init_global_kms_service_manager();
|
||||
manager
|
||||
.reconfigure(rustfs_kms::KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
|
||||
.await
|
||||
.expect("configure test KMS service");
|
||||
manager
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_encryption_resolver_returns_ssec_read_material() {
|
||||
let key = [0x31; 32];
|
||||
let key_b64 = BASE64_STANDARD.encode(key);
|
||||
let key_md5 = md5_base64(key);
|
||||
let nonce = [0x42; 12];
|
||||
let metadata = HashMap::from([
|
||||
("X-Amz-Server-Side-Encryption-Customer-Algorithm".to_string(), "AES256".to_string()),
|
||||
("X-Amz-Server-Side-Encryption-Customer-Key-Md5".to_string(), key_md5.clone()),
|
||||
("X-Rustfs-Encryption-Iv".to_string(), BASE64_STANDARD.encode(nonce)),
|
||||
]);
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"),
|
||||
);
|
||||
|
||||
let material = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &headers,
|
||||
})
|
||||
.await
|
||||
.expect("SSE-C material should resolve")
|
||||
.expect("SSE-C metadata should produce material");
|
||||
|
||||
assert_eq!(material.key_bytes, key);
|
||||
assert_eq!(material.mode, ReadEncryptionMode::Direct { base_nonce: nonce });
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_encryption_resolver_rejects_missing_or_invalid_ssec_algorithm() {
|
||||
let key = [0x31; 32];
|
||||
let key_b64 = BASE64_STANDARD.encode(key);
|
||||
let key_md5 = md5_base64(key);
|
||||
let metadata = HashMap::from([
|
||||
("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string()),
|
||||
("x-amz-server-side-encryption-customer-key-md5".to_string(), key_md5.clone()),
|
||||
]);
|
||||
|
||||
for algorithm in [None, Some("AES128")] {
|
||||
let mut headers = HeaderMap::new();
|
||||
if let Some(algorithm) = algorithm {
|
||||
headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static(algorithm));
|
||||
}
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_str(&key_b64).expect("base64 key is a valid header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
HeaderValue::from_str(&key_md5).expect("base64 MD5 is a valid header"),
|
||||
);
|
||||
|
||||
let result = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &headers,
|
||||
})
|
||||
.await;
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("missing or invalid SSE-C algorithm must fail closed"),
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_encryption_resolver_classifies_missing_ssec_key_as_invalid_request() {
|
||||
let metadata = HashMap::from([("x-amz-server-side-encryption-customer-algorithm".to_string(), "AES256".to_string())]);
|
||||
let result = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &HeaderMap::new(),
|
||||
})
|
||||
.await;
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("missing SSE-C key must fail closed"),
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidRequest);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_encryption_resolver_rejects_conflicting_metadata_case_variants() {
|
||||
let metadata = HashMap::from([
|
||||
("x-rustfs-encryption-key".to_string(), "first".to_string()),
|
||||
("X-Rustfs-Encryption-Key".to_string(), "second".to_string()),
|
||||
]);
|
||||
let result = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &HeaderMap::new(),
|
||||
})
|
||||
.await;
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("conflicting metadata aliases must fail closed"),
|
||||
};
|
||||
|
||||
assert_eq!(error.kind(), EncryptionResolutionErrorKind::InvalidMetadata);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_encryption_metadata_case_accepts_lowercase_minio_internal_keys() {
|
||||
let lowercase_key = MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase();
|
||||
let metadata = HashMap::from([(lowercase_key, "sealed-key".to_string())]);
|
||||
|
||||
let normalized = super::normalize_encryption_metadata_case(&metadata).expect("metadata aliases should normalize");
|
||||
|
||||
assert_eq!(
|
||||
normalized.get(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER),
|
||||
Some(&"sealed-key".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
struct UnavailableSseDekProvider;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -3433,18 +3710,11 @@ mod tests {
|
||||
#[cfg(feature = "rio-v2")]
|
||||
#[tokio::test]
|
||||
async fn test_sse_kms_roundtrip_persists_and_uses_minio_context() {
|
||||
use rustfs_kms::config::KmsConfig;
|
||||
use rustfs_kms::types::{CreateKeyRequest, KeyUsage};
|
||||
use tempfile::TempDir;
|
||||
let _guard = lock_sse_test_state().await;
|
||||
|
||||
reset_sse_dek_provider();
|
||||
let manager = rustfs_kms::init_global_kms_service_manager();
|
||||
let temp_dir = TempDir::new().expect("temp dir");
|
||||
manager
|
||||
.reconfigure(KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults())
|
||||
.await
|
||||
.expect("kms reconfigure should succeed");
|
||||
let manager = configure_test_global_local_kms().await;
|
||||
manager
|
||||
.get_encryption_service()
|
||||
.await
|
||||
@@ -3736,6 +4006,19 @@ mod tests {
|
||||
|
||||
assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
|
||||
assert_eq!(decrypted.key_bytes, material.key_bytes);
|
||||
|
||||
let resolved = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &HeaderMap::new(),
|
||||
})
|
||||
.await
|
||||
.expect("managed resolver")
|
||||
.expect("managed material");
|
||||
assert_eq!(resolved.mode, ReadEncryptionMode::Object);
|
||||
assert_eq!(resolved.key_bytes, material.key_bytes);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
@@ -3796,6 +4079,29 @@ mod tests {
|
||||
|
||||
assert_eq!(decrypted.key_kind, EncryptionKeyKind::Object);
|
||||
assert_eq!(decrypted.key_bytes, material.key_bytes);
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-amz-server-side-encryption-customer-algorithm", HeaderValue::from_static("AES256"));
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key",
|
||||
HeaderValue::from_str(&customer_key).expect("customer key header"),
|
||||
);
|
||||
headers.insert(
|
||||
"x-amz-server-side-encryption-customer-key-md5",
|
||||
HeaderValue::from_str(&customer_key_md5).expect("customer key MD5 header"),
|
||||
);
|
||||
let resolved = SseObjectEncryptionResolver
|
||||
.resolve_read_material(ReadEncryptionRequest {
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
metadata: &metadata,
|
||||
headers: &headers,
|
||||
})
|
||||
.await
|
||||
.expect("SSE-C resolver")
|
||||
.expect("SSE-C material");
|
||||
assert_eq!(resolved.mode, ReadEncryptionMode::Object);
|
||||
assert_eq!(resolved.key_bytes, material.key_bytes);
|
||||
}
|
||||
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -4270,17 +4576,9 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_managed_decryption_selects_provider_from_persisted_dek() {
|
||||
use rustfs_kms::config::KmsConfig;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let _guard = lock_sse_test_state().await;
|
||||
reset_sse_dek_provider();
|
||||
let manager = rustfs_kms::init_global_kms_service_manager();
|
||||
let key_dir = TempDir::new().expect("create KMS key directory");
|
||||
manager
|
||||
.reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
|
||||
.await
|
||||
.expect("start test KMS service");
|
||||
let manager = configure_test_global_local_kms().await;
|
||||
|
||||
let local_master_key = [7u8; 32];
|
||||
let local_provider = TestSseDekProvider::new_with_key(local_master_key);
|
||||
@@ -4348,9 +4646,6 @@ mod tests {
|
||||
/// the local-provider cache.
|
||||
#[tokio::test]
|
||||
async fn test_kms_envelope_never_routes_to_cached_local_provider() {
|
||||
use rustfs_kms::config::KmsConfig;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let _guard = lock_sse_test_state().await;
|
||||
reset_sse_dek_provider();
|
||||
|
||||
@@ -4362,12 +4657,7 @@ mod tests {
|
||||
.expect("write local provider into local cache") = Some(Arc::new(TestSseDekProvider::new_with_key(local_master_key)));
|
||||
|
||||
// 2. Start a KMS service (dynamic enable).
|
||||
let manager = rustfs_kms::init_global_kms_service_manager();
|
||||
let key_dir = TempDir::new().expect("create KMS key directory");
|
||||
manager
|
||||
.reconfigure(KmsConfig::local(key_dir.path().to_path_buf()).with_insecure_development_defaults())
|
||||
.await
|
||||
.expect("start test KMS service");
|
||||
let manager = configure_test_global_local_kms().await;
|
||||
|
||||
// 3. Construct a KMS JSON envelope — the persisted format of a KMS-wrapped DEK.
|
||||
// is_data_key_envelope() will return true for this payload.
|
||||
@@ -4458,7 +4748,7 @@ mod tests {
|
||||
use rustfs_kms::config::KmsConfig;
|
||||
let _guard = lock_sse_test_state().await;
|
||||
|
||||
let manager = rustfs_kms::init_global_kms_service_manager();
|
||||
let manager = Arc::new(rustfs_kms::KmsServiceManager::new());
|
||||
|
||||
manager
|
||||
.reconfigure(KmsConfig::static_kms("first-key".to_string(), BASE64_STANDARD.encode([0x11; 32])))
|
||||
@@ -4692,6 +4982,15 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_map_get_object_reader_error_preserves_typed_service_unavailable() {
|
||||
let resolution_error =
|
||||
super::EncryptionResolutionError::new(EncryptionResolutionErrorKind::ServiceUnavailable, "KMS unavailable");
|
||||
let err = map_get_object_reader_error(StorageError::other(resolution_error));
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.message, "KMS unavailable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_map_get_object_reader_error_leaves_non_ssec_errors_unchanged() {
|
||||
let err = map_get_object_reader_error(StorageError::other("plain io failure"));
|
||||
|
||||
@@ -509,12 +509,21 @@ pub(crate) mod ecstore_object {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource;
|
||||
pub(crate) use rustfs_ecstore::api::object::{
|
||||
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup,
|
||||
ObjectEncryptionResolver, ObjectMutationHook, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "rio-v2"))]
|
||||
pub(crate) mod ecstore_test_support {
|
||||
pub(crate) use rustfs_ecstore::api::bitrot::create_bitrot_reader;
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::erasure::Erasure;
|
||||
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_set_disk {
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class};
|
||||
}
|
||||
@@ -945,13 +954,21 @@ pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Res
|
||||
/// The process-level bootstrap instance context that single-instance startup
|
||||
/// threads through the storage foundation (Phase 5 follow-up, backlog#1052).
|
||||
pub(crate) fn bootstrap_instance_ctx() -> Arc<InstanceContext> {
|
||||
ecstore_runtime::bootstrap_ctx()
|
||||
let context = ecstore_runtime::bootstrap_ctx();
|
||||
configure_object_encryption_resolver(&context);
|
||||
context
|
||||
}
|
||||
|
||||
/// Construct a fresh per-server instance context (backlog#1052 S5): a second
|
||||
/// embedded server owns its own erasure/region/endpoint/deployment id cells.
|
||||
pub(crate) fn new_instance_ctx() -> Arc<InstanceContext> {
|
||||
Arc::new(InstanceContext::new())
|
||||
let context = Arc::new(InstanceContext::new());
|
||||
configure_object_encryption_resolver(&context);
|
||||
context
|
||||
}
|
||||
|
||||
fn configure_object_encryption_resolver(context: &InstanceContext) {
|
||||
let _ = context.set_object_encryption_resolver(Arc::new(super::sse::SseObjectEncryptionResolver));
|
||||
}
|
||||
|
||||
pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) {
|
||||
@@ -1746,7 +1763,7 @@ pub(crate) async fn init_compression_total_memory_from_backend(store: Arc<ECStor
|
||||
mod tests {
|
||||
use super::{
|
||||
apply_active_resync_intents, bucket_targets_metadata_lock_shard, ecstore_bucket, lock_bucket_targets_metadata,
|
||||
scanner_maintenance_config_file,
|
||||
new_instance_ctx, scanner_maintenance_config_file,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -1777,6 +1794,16 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fresh_instance_context_installs_object_encryption_resolver() {
|
||||
assert!(new_instance_ctx().object_encryption_resolver().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_instance_context_installs_object_encryption_resolver() {
|
||||
assert!(super::bootstrap_instance_ctx().object_encryption_resolver().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_maintenance_config_only_includes_scanner_owned_work() {
|
||||
assert!(scanner_maintenance_config_file(ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG));
|
||||
|
||||
Reference in New Issue
Block a user