gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-10 07:06:53 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
ce9f04f28e01f58a340e6734feb662f125ab8f2b
rustfs/rustfs/src/admin/handlers/kms_dynamic.rs
T
Zhengchao An 7528a0b916 feat(kms): accept the AWS backend through KMS configuration (#5592) 
* feat(kms): accept the AWS backend through KMS configuration

The AWS KMS backend could be constructed but not selected: the admin
configure API had no AWS variant and startup rejected the backend name.

The configure request pins the region rather than defaulting it, because
that configuration is persisted once and replayed on every node: leaving
the region to each node's ambient provider chain would let nodes address
different regions, and therefore different keys, while reporting an
identical configuration. The request accepts no credential fields, so
credentials stay with the aws-config provider chain on each node, and
`deny_unknown_fields` refuses attempts to submit them anyway.

* test(kms): cover AWS backend selection through the service manager

An end-to-end check that an admin configure request selects the AWS
backend, builds a client, and passes the startup health check. Marked
#[ignore]: it needs real AWS credentials, though it creates no key and
is therefore not billable on its own.
2026-08-02 03:58:20 +08:00

1571 lines
62 KiB
Rust
Raw Blame History

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! KMS dynamic configuration admin API handlers
use super::kms_audit::{KmsAdminAudit, KmsAdminOperation};
use crate::admin::auth::validate_admin_request;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::runtime_sources::{
current_app_context, current_kms_runtime_service_manager, current_notification_system_for_context,
current_object_store_handle_for_context, current_or_init_kms_runtime_service_manager,
};
use crate::admin::storage_api::config::{read_admin_config, save_admin_config};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, RemoteAddr};
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_kms::{
ConfigureKmsRequest, ConfigureKmsResponse, KmsConfig, KmsConfigSummary, KmsServiceStatus, KmsStatusResponse, StartKmsRequest,
StartKmsResponse, StopKmsResponse,
};
use rustfs_policy::policy::action::{Action, KmsAction};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use sha2::{Digest, Sha256};
use tracing::{error, info, instrument, warn};
/// Path to store KMS configuration in the cluster metadata
const KMS_CONFIG_PATH: &str = "config/kms_config.json";
const STATIC_KMS_LOCAL_CONFIG_REQUIRED: &str =
"Static KMS must be configured through RUSTFS_KMS_STATIC_SECRET_KEY or RUSTFS_KMS_STATIC_SECRET_KEY_FILE";
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_KMS: &str = "kms";
const EVENT_ADMIN_KMS_DYNAMIC_STATE: &str = "admin_kms_dynamic_state";
/// Substituted for every secret value before a configuration is fingerprinted,
/// so the digest depends on the shape of the configuration and never on key
/// material an operator could recover by replaying candidate secrets.
const REDACTED_CONFIG_VALUE: &str = "[redacted]";
/// Configuration field names carrying secrets. Matched at any depth so a
/// backend that nests its credentials cannot leak them into the fingerprint.
const REDACTED_CONFIG_FIELDS: [&str; 6] = ["token", "secret_key", "secret_id", "master_key", "password", "client_secret"];
fn kms_service_manager_from_context() -> std::sync::Arc<rustfs_kms::KmsServiceManager> {
current_kms_runtime_service_manager().unwrap_or_else(|| {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_manager_fallback",
result = "service_manager_fallback_initialized",
"admin kms dynamic state"
);
current_or_init_kms_runtime_service_manager()
})
}
fn token_is_blank(auth_method: &rustfs_kms::config::VaultAuthMethod) -> bool {
matches!(
auth_method,
rustfs_kms::config::VaultAuthMethod::Token { token } if token.trim().is_empty()
)
}
fn existing_vault_auth(config: &KmsConfig) -> Option<rustfs_kms::config::VaultAuthMethod> {
match &config.backend_config {
rustfs_kms::config::BackendConfig::VaultKv2(vault) => Some(vault.auth_method.clone()),
rustfs_kms::config::BackendConfig::VaultTransit(vault) => Some(vault.auth_method.clone()),
rustfs_kms::config::BackendConfig::Local(_) => None,
rustfs_kms::config::BackendConfig::Static(_) => None,
// AWS credentials come from the aws-config chain, not from KMS config.
rustfs_kms::config::BackendConfig::Aws(_) => None,
}
}
fn kms_configure_actions() -> Vec<Action> {
vec![Action::KmsAction(KmsAction::ConfigureAction)]
}
fn kms_service_control_actions() -> Vec<Action> {
vec![Action::KmsAction(KmsAction::ServiceControlAction)]
}
fn normalize_configure_request_secrets(
request: &mut ConfigureKmsRequest,
existing_config: Option<&KmsConfig>,
) -> Result<(), String> {
if existing_config.is_some_and(|config| matches!(&config.backend_config, rustfs_kms::BackendConfig::Local(_)))
&& !matches!(request, ConfigureKmsRequest::Local(_))
{
return Err("Changing from the Local KMS backend is not supported".to_string());
}
if let ConfigureKmsRequest::Local(request) = request
&& let Some(KmsConfig {
backend_config: rustfs_kms::BackendConfig::Local(existing),
allow_insecure_dev_defaults,
..
}) = existing_config
{
if request.key_dir != existing.key_dir {
return Err("Changing the Local KMS key directory is not supported".to_string());
}
match request.file_permissions {
Some(permissions) if Some(permissions) != existing.file_permissions => {
return Err("Changing Local KMS file permissions is not supported".to_string());
}
None => request.file_permissions = existing.file_permissions,
Some(_) => {}
}
if request.master_key.as_deref().is_some_and(|master_key| !master_key.is_empty()) {
return Err("Changing the Local KMS master key is not supported".to_string());
}
request.master_key.clone_from(&existing.master_key);
request.allow_insecure_dev_defaults = Some(*allow_insecure_dev_defaults);
}
let needs_existing_auth = match request {
ConfigureKmsRequest::VaultKv2(req) => token_is_blank(&req.auth_method),
ConfigureKmsRequest::VaultTransit(req) => token_is_blank(&req.auth_method),
ConfigureKmsRequest::Local(_) => false,
ConfigureKmsRequest::Static(_) => false,
// AWS credentials come from the aws-config chain, so there is nothing
// to carry over from the existing configuration.
ConfigureKmsRequest::Aws(_) => false,
};
if !needs_existing_auth {
return Ok(());
}
let existing_auth = existing_config
.and_then(existing_vault_auth)
.ok_or_else(|| "Vault token is required when no existing KMS credentials are available".to_string())?;
match request {
ConfigureKmsRequest::VaultKv2(req) => req.auth_method = existing_auth,
ConfigureKmsRequest::VaultTransit(req) => req.auth_method = existing_auth,
ConfigureKmsRequest::Local(_) => {}
ConfigureKmsRequest::Static(_) => {}
ConfigureKmsRequest::Aws(_) => {}
}
Ok(())
}
fn ensure_kms_config_persistable(config: &KmsConfig) -> Result<(), String> {
if matches!(&config.backend_config, rustfs_kms::BackendConfig::Static(_)) {
return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string());
}
Ok(())
}
fn ensure_kms_request_persistable(request: &ConfigureKmsRequest) -> Result<(), String> {
if matches!(request, ConfigureKmsRequest::Static(_)) {
return Err(STATIC_KMS_LOCAL_CONFIG_REQUIRED.to_string());
}
Ok(())
}
/// Save KMS configuration to cluster storage
#[instrument(skip(config))]
async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
ensure_kms_config_persistable(config)?;
let context = current_app_context();
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
return Err("Storage layer not initialized".to_string());
};
let data = serde_json::to_vec(config).map_err(|e| format!("Failed to serialize KMS config: {e}"))?;
save_admin_config(store, KMS_CONFIG_PATH, data)
.await
.map_err(|e| format!("Failed to save KMS config to storage: {e}"))?;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_persisted",
storage_path = KMS_CONFIG_PATH,
state = "config_persisted",
"admin kms dynamic state"
);
Ok(())
}
fn decode_persisted_kms_config(data: &[u8]) -> serde_json::Result<(KmsConfig, bool)> {
let mut config: KmsConfig = serde_json::from_slice(data)?;
// The immediate-deletion gate is per-server operator state, never stored,
// so a config loaded from cluster storage still has to pick it up here.
config.allow_immediate_deletion = rustfs_kms::config::allow_immediate_deletion_from_env();
let value: serde_json::Value = serde_json::from_slice(data)?;
let is_missing_development_flag = value
.as_object()
.is_some_and(|object| !object.contains_key("allow_insecure_dev_defaults"));
let mut uses_legacy_local_defaults = false;
if is_missing_development_flag
&& matches!(&config.backend_config, rustfs_kms::BackendConfig::Local(_))
&& config.validate().is_err()
{
// RUSTFS_COMPAT_TODO(rustfs-5063): Remove after pre-beta.9 configurations are rewritten with this field.
// Pre-beta.9 persisted Local KMS configurations predate the explicit
// development-default flag.
config.allow_insecure_dev_defaults = true;
if config.validate().is_ok() {
uses_legacy_local_defaults = true;
} else {
config.allow_insecure_dev_defaults = false;
}
}
Ok((config, uses_legacy_local_defaults))
}
/// Load KMS configuration from cluster storage
#[instrument]
pub async fn load_kms_config() -> Option<KmsConfig> {
let context = current_app_context();
let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_load_skipped",
reason = "storage_uninitialized",
result = "config_load_skipped",
"admin kms dynamic state"
);
return None;
};
match read_admin_config(store, KMS_CONFIG_PATH).await {
Ok(data) => match decode_persisted_kms_config(&data) {
Ok((config, is_legacy_local)) => {
if is_legacy_local {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_legacy_local_config_loaded",
storage_path = KMS_CONFIG_PATH,
state = "legacy_config_accepted",
"admin kms dynamic state"
);
}
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_loaded",
storage_path = KMS_CONFIG_PATH,
state = "config_loaded",
"admin kms dynamic state"
);
Some(config)
}
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_deserialize_failed",
storage_path = KMS_CONFIG_PATH,
result = "config_deserialize_failed",
error = %e,
"admin kms dynamic state"
);
None
}
},
Err(e) => {
// Config not found is normal on first run
if e.to_string().contains("ConfigNotFound") || e.to_string().contains("not found") {
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_loaded",
state = "not_found",
storage_path = KMS_CONFIG_PATH,
"admin kms dynamic state"
);
} else {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_config_load_failed",
storage_path = KMS_CONFIG_PATH,
result = "config_load_failed",
error = %e,
"admin kms dynamic state"
);
}
None
}
}
}
fn redact_config_secrets(value: &mut serde_json::Value) {
match value {
serde_json::Value::Object(object) => {
for (key, field) in object.iter_mut() {
if REDACTED_CONFIG_FIELDS.contains(&key.as_str()) {
*field = serde_json::Value::String(REDACTED_CONFIG_VALUE.to_string());
} else {
redact_config_secrets(field);
}
}
}
serde_json::Value::Array(items) => items.iter_mut().for_each(redact_config_secrets),
_ => {}
}
}
/// Serialize with object keys ordered so two nodes holding the same
/// configuration always hash the same bytes, whatever their map iteration order.
fn write_canonical_json(value: &serde_json::Value, out: &mut String) {
match value {
serde_json::Value::Object(object) => {
let mut keys = object.keys().collect::<Vec<_>>();
keys.sort_unstable();
out.push('{');
for (index, key) in keys.into_iter().enumerate() {
if index > 0 {
out.push(',');
}
out.push_str(&serde_json::Value::String(key.clone()).to_string());
out.push(':');
write_canonical_json(&object[key], out);
}
out.push('}');
}
serde_json::Value::Array(items) => {
out.push('[');
for (index, item) in items.iter().enumerate() {
if index > 0 {
out.push(',');
}
write_canonical_json(item, out);
}
out.push(']');
}
other => out.push_str(&other.to_string()),
}
}
fn redacted_canonical_config(config: &KmsConfig) -> Option<String> {
let mut value = serde_json::to_value(config).ok()?;
redact_config_secrets(&mut value);
let mut canonical = String::new();
write_canonical_json(&value, &mut canonical);
Some(canonical)
}
/// Fingerprint of a KMS configuration in its persisted form, with every secret
/// replaced before hashing.
///
/// Comparing fingerprints across nodes makes a configuration split visible
/// without exposing what any node is configured with. Redaction is deliberate:
/// two nodes holding the same backend with different credentials fingerprint
/// alike, which is the price of a digest that is safe to report.
pub fn kms_config_fingerprint(config: &KmsConfig) -> Option<String> {
let canonical = redacted_canonical_config(config)?;
Some(hex_simd::encode_to_string(
Sha256::digest(canonical.as_bytes()),
hex_simd::AsciiCase::Lower,
))
}
/// Fingerprint of the configuration this node is currently running.
///
/// `None` when KMS has never been configured on this node, which is itself a
/// divergence from a cluster that has.
pub async fn current_kms_config_fingerprint() -> Option<String> {
kms_config_fingerprint(&kms_service_manager_from_context().get_config().await?)
}
fn kms_config_is_unchanged(current: &KmsConfig, candidate: &KmsConfig) -> bool {
match (serde_json::to_vec(current), serde_json::to_vec(candidate)) {
(Ok(current), Ok(candidate)) => current == candidate,
_ => false,
}
}
/// Re-read the cluster-persisted KMS configuration and switch this node to it.
///
/// Invoked on peers by the reload signal that a configure or reconfigure
/// request broadcasts, so that a runtime change reaches every node instead of
/// only the one that served the admin request.
pub async fn reload_persisted_kms_config() -> Result<(), String> {
let Some(config) = load_kms_config().await else {
return Err("no persisted KMS configuration is available".to_string());
};
let service_manager = kms_service_manager_from_context();
if service_manager
.get_config()
.await
.is_some_and(|current| kms_config_is_unchanged(&current, &config))
{
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "peer_reload",
state = "already_current",
"admin kms dynamic state"
);
return Ok(());
}
service_manager.reconfigure(config).await.map_err(|err| {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "peer_reload",
state = "reload_failed",
error = %err,
"admin kms dynamic state"
);
format!("failed to apply persisted KMS configuration: {err}")
})?;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "peer_reload",
state = "reconfigured",
"admin kms dynamic state"
);
Ok(())
}
/// Ask every peer to adopt the configuration that was just persisted.
///
/// Returns the hosts that did not converge. Reporting is best effort by
/// design: this node has already switched and KMS configuration has no quorum
/// or authoritative holder to roll back to, so refusing the local change would
/// trade a bounded divergence window for an outage.
async fn broadcast_kms_config_reload() -> Vec<String> {
let context = current_app_context();
let Some(notification_sys) = current_notification_system_for_context(context.as_deref()) else {
return Vec::new();
};
let mut unconverged = Vec::new();
for outcome in notification_sys.reload_kms_config().await {
let Some(err) = outcome.err else {
continue;
};
let host = if outcome.host.is_empty() {
"<unknown>".to_string()
} else {
outcome.host
};
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_peer_config_reload_failed",
peer = %host,
result = "peer_reload_failed",
error = %err,
"admin kms dynamic state"
);
unconverged.push(host);
}
unconverged
}
/// Fold a best-effort peer reload result into the local outcome.
///
/// The local outcome stays successful whatever the peers reported: this node
/// has already switched, and a failed peer is named in the message so the
/// operator can act on the divergence instead of being told nothing happened.
fn local_success_with_peer_report(message: &str, unconverged: &[String]) -> (bool, String) {
if unconverged.is_empty() {
return (true, message.to_string());
}
(
true,
format!(
"{message}. {} peer(s) did not reload the new configuration and keep serving their previous KMS configuration until they do: {}",
unconverged.len(),
unconverged.join(", ")
),
)
}
pub fn register_kms_dynamic_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/kms/configure").as_str(),
AdminOperation(&ConfigureKmsHandler {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/kms/start").as_str(),
AdminOperation(&StartKmsHandler {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/kms/stop").as_str(),
AdminOperation(&StopKmsHandler {}),
)?;
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/kms/service-status").as_str(),
AdminOperation(&GetKmsStatusHandler {}),
)?;
r.insert(
Method::POST,
format!("{}{}", ADMIN_PREFIX, "/v3/kms/reconfigure").as_str(),
AdminOperation(&ReconfigureKmsHandler {}),
)?;
Ok(())
}
/// Configure KMS service handler
pub struct ConfigureKmsHandler;
#[async_trait::async_trait]
impl Operation for ConfigureKmsHandler {
async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
let audit = KmsAdminAudit::from_request(&req.extensions, &req.headers, &cred);
audit.gate_admin(
validate_admin_request(
&req.headers,
&cred,
owner,
false,
kms_configure_actions(),
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await,
KmsAdminOperation::Configure,
None,
)?;
let body = req
.input
.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
.await
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
let mut configure_request: ConfigureKmsRequest = if body.is_empty() {
return Ok(S3Response::new((
StatusCode::BAD_REQUEST,
Body::from("Request body is required".to_string()),
)));
} else {
match serde_json::from_slice(&body) {
Ok(req) => req,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed",
operation = "configure",
result = "request_decode_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
}
}
};
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "configure",
state = "requested",
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
let existing_config = service_manager.get_config().await;
if let Err(e) = normalize_configure_request_secrets(&mut configure_request, existing_config.as_ref()) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
if let Err(e) = ensure_kms_request_persistable(&configure_request) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
// Convert request to KmsConfig
let kms_config = configure_request.to_kms_config();
let persisted_config = kms_config.clone();
let (success, message, status) = match service_manager
.configure_with_persistence(kms_config, || async move {
save_kms_config(&persisted_config)
.await
.map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
})
.await
{
Ok(()) => {
let status = service_manager.get_status().await;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "configure",
state = "configured",
status = ?status,
"admin kms dynamic state"
);
let unconverged = broadcast_kms_config_reload().await;
let (success, message) = local_success_with_peer_report("KMS configured successfully", &unconverged);
audit.finish(KmsAdminOperation::Configure, None, None);
(success, message, status)
}
Err(e) => {
let error_msg = format!("Failed to configure KMS: {e}");
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "configure",
state = "configure_failed",
error = %e,
"admin kms dynamic state"
);
audit.finish(KmsAdminOperation::Configure, None, Some(&e));
let status = service_manager.get_status().await;
(false, error_msg, status)
}
};
let response = ConfigureKmsResponse {
success,
message,
status,
};
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "configure",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
)));
}
};
Ok(S3Response::new((StatusCode::OK, Body::from(json_response))))
}
}
/// Start KMS service handler
pub struct StartKmsHandler;
#[async_trait::async_trait]
impl Operation for StartKmsHandler {
async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
let audit = KmsAdminAudit::from_request(&req.extensions, &req.headers, &cred);
audit.gate_admin(
validate_admin_request(
&req.headers,
&cred,
owner,
false,
kms_service_control_actions(),
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await,
KmsAdminOperation::Start,
None,
)?;
let body = req
.input
.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
.await
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
let start_request: StartKmsRequest = if body.is_empty() {
StartKmsRequest { force: None }
} else {
match serde_json::from_slice(&body) {
Ok(req) => req,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed",
operation = "start",
result = "request_decode_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
}
}
};
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "start",
state = "requested",
force = start_request.force.unwrap_or(false),
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
let force = start_request.force.unwrap_or(false);
let (success, message, status) = match service_manager.start_or_restart(force).await {
Ok(rustfs_kms::KmsStartOutcome::Started) => {
let status = service_manager.get_status().await;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "start",
state = "running",
status = ?status,
"admin kms dynamic state"
);
audit.finish(KmsAdminOperation::Start, None, None);
(true, "KMS service started successfully".to_string(), status)
}
Ok(rustfs_kms::KmsStartOutcome::Restarted) => {
let status = service_manager.get_status().await;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "restart",
state = "running",
status = ?status,
"admin kms dynamic state"
);
audit.finish(KmsAdminOperation::Start, None, None);
(true, "KMS service restarted successfully".to_string(), status)
}
Ok(rustfs_kms::KmsStartOutcome::AlreadyRunning) => {
let status = service_manager.get_status().await;
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "start",
state = "already_running",
"admin kms dynamic state"
);
// A refusal, not an outage: recorded as a failed attempt so the
// trail shows the request without inventing a KMS error for it.
audit.finish_with_class(KmsAdminOperation::Start, Some("invalid_operation"));
(false, "KMS service is already running. Use force=true to restart.".to_string(), status)
}
Err(e) => {
let error_msg = format!("Failed to start or restart KMS service: {e}");
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "start",
state = "start_failed",
error = %e,
"admin kms dynamic state"
);
audit.finish(KmsAdminOperation::Start, None, Some(&e));
let status = service_manager.get_status().await;
(false, error_msg, status)
}
};
let response = StartKmsResponse {
success,
message,
status,
};
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "start",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
)));
}
};
Ok(S3Response::new((StatusCode::OK, Body::from(json_response))))
}
}
/// Stop KMS service handler
pub struct StopKmsHandler;
#[async_trait::async_trait]
impl Operation for StopKmsHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
let audit = KmsAdminAudit::from_request(&req.extensions, &req.headers, &cred);
audit.gate_admin(
validate_admin_request(
&req.headers,
&cred,
owner,
false,
kms_service_control_actions(),
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await,
KmsAdminOperation::Stop,
None,
)?;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "stop",
state = "requested",
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
let (success, message, status) = match service_manager.stop().await {
Ok(()) => {
let status = service_manager.get_status().await;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "stop",
state = "stopped",
status = ?status,
"admin kms dynamic state"
);
audit.finish(KmsAdminOperation::Stop, None, None);
(true, "KMS service stopped successfully".to_string(), status)
}
Err(e) => {
let error_msg = format!("Failed to stop KMS service: {e}");
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "stop",
state = "stop_failed",
error = %e,
"admin kms dynamic state"
);
audit.finish(KmsAdminOperation::Stop, None, Some(&e));
let status = service_manager.get_status().await;
(false, error_msg, status)
}
};
let response = StopKmsResponse {
success,
message,
status,
};
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "stop",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
)));
}
};
Ok(S3Response::new((StatusCode::OK, Body::from(json_response))))
}
}
/// Get KMS status handler
pub struct GetKmsStatusHandler;
#[async_trait::async_trait]
impl Operation for GetKmsStatusHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
validate_admin_request(
&req.headers,
&cred,
owner,
false,
kms_service_control_actions(),
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await?;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_status_requested",
state = "status_requested",
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
let (status, config) = service_manager.get_redacted_state().await;
// Get backend type and health status
let backend_type = config.as_ref().map(|c| c.backend.clone());
let healthy = if matches!(status, KmsServiceStatus::Running) {
match service_manager.health_check().await {
Ok(healthy) => Some(healthy),
Err(_) => Some(false),
}
} else {
None
};
// Create config summary (without sensitive data)
let config_summary = config.as_ref().map(KmsConfigSummary::from);
let response = KmsStatusResponse {
status,
backend_type,
healthy,
config_summary,
};
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_status_resolved",
status = ?response.status,
backend_type = ?response.backend_type,
healthy = response.healthy,
has_config_summary = response.config_summary.is_some(),
state = "status_resolved",
"admin kms dynamic state"
);
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "status",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
)));
}
};
Ok(S3Response::new((StatusCode::OK, Body::from(json_response))))
}
}
/// Reconfigure KMS service handler
pub struct ReconfigureKmsHandler;
#[async_trait::async_trait]
impl Operation for ReconfigureKmsHandler {
async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let Some(cred) = req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &cred.access_key).await?;
let audit = KmsAdminAudit::from_request(&req.extensions, &req.headers, &cred);
audit.gate_admin(
validate_admin_request(
&req.headers,
&cred,
owner,
false,
kms_configure_actions(),
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await,
KmsAdminOperation::Reconfigure,
None,
)?;
let body = req
.input
.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
.await
.map_err(|e| s3_error!(InvalidRequest, "failed to read request body: {}", e))?;
let mut configure_request: ConfigureKmsRequest = if body.is_empty() {
return Ok(S3Response::new((
StatusCode::BAD_REQUEST,
Body::from("Request body is required".to_string()),
)));
} else {
match serde_json::from_slice(&body) {
Ok(req) => req,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_request_decode_failed",
operation = "reconfigure",
result = "request_decode_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(format!("Invalid JSON: {e}")))));
}
}
};
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "reconfigure",
state = "requested",
"admin kms dynamic state"
);
let service_manager = kms_service_manager_from_context();
let existing_config = service_manager.get_config().await;
if let Err(e) = normalize_configure_request_secrets(&mut configure_request, existing_config.as_ref()) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
if let Err(e) = ensure_kms_request_persistable(&configure_request) {
return Ok(S3Response::new((StatusCode::BAD_REQUEST, Body::from(e))));
}
// Convert request to KmsConfig
let kms_config = configure_request.to_kms_config();
let persisted_config = kms_config.clone();
let (success, message, status) = match service_manager
.reconfigure_with_persistence(kms_config, || async move {
save_kms_config(&persisted_config)
.await
.map_err(|error| rustfs_kms::KmsError::backend_error(format!("Failed to persist KMS configuration: {error}")))
})
.await
{
Ok(()) => {
let status = service_manager.get_status().await;
info!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "reconfigure",
state = "reconfigured",
status = ?status,
"admin kms dynamic state"
);
let unconverged = broadcast_kms_config_reload().await;
let (success, message) =
local_success_with_peer_report("KMS reconfigured and restarted successfully", &unconverged);
audit.finish(KmsAdminOperation::Reconfigure, None, None);
(success, message, status)
}
Err(e) => {
let error_msg = format!("Failed to reconfigure KMS: {e}");
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = "kms_service_state",
operation = "reconfigure",
state = "reconfigure_failed",
error = %e,
"admin kms dynamic state"
);
audit.finish(KmsAdminOperation::Reconfigure, None, Some(&e));
let status = service_manager.get_status().await;
(false, error_msg, status)
}
};
let response = ConfigureKmsResponse {
success,
message,
status,
};
let json_response = match serde_json::to_string(&response) {
Ok(json) => json,
Err(e) => {
error!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_KMS,
event = EVENT_ADMIN_KMS_DYNAMIC_STATE,
operation = "reconfigure",
result = "response_serialize_failed",
error = %e,
"admin kms dynamic state"
);
return Ok(S3Response::new((
StatusCode::INTERNAL_SERVER_ERROR,
Body::from("Serialization error".to_string()),
)));
}
};
Ok(S3Response::new((StatusCode::OK, Body::from(json_response))))
}
}
#[cfg(test)]
mod tests {
use super::{
decode_persisted_kms_config, ensure_kms_config_persistable, ensure_kms_request_persistable, kms_config_fingerprint,
kms_config_is_unchanged, kms_configure_actions, kms_service_control_actions, local_success_with_peer_report,
normalize_configure_request_secrets, redacted_canonical_config,
};
use rustfs_policy::policy::action::{Action, AdminAction, KmsAction};
use std::path::PathBuf;
use tempfile::TempDir;
fn assert_has_action(actions: &[Action], action: Action) {
assert!(actions.contains(&action), "expected action list to contain {action:?}");
}
fn assert_lacks_action(actions: &[Action], action: Action) {
assert!(!actions.contains(&action), "expected action list not to contain {action:?}");
}
#[test]
fn kms_dynamic_auth_actions_use_dedicated_kms_actions() {
assert_has_action(&kms_configure_actions(), Action::KmsAction(KmsAction::ConfigureAction));
assert_has_action(&kms_service_control_actions(), Action::KmsAction(KmsAction::ServiceControlAction));
}
#[test]
fn kms_dynamic_actions_reject_server_info_fallback() {
assert_lacks_action(&kms_configure_actions(), Action::AdminAction(AdminAction::ServerInfoAdminAction));
assert_lacks_action(&kms_service_control_actions(), Action::AdminAction(AdminAction::ServerInfoAdminAction));
}
#[test]
fn persisted_beta5_local_config_retains_legacy_development_mode() {
let temp_dir = TempDir::new().expect("create legacy local KMS directory");
let config = rustfs_kms::KmsConfig::local(temp_dir.path().to_path_buf());
let mut value = serde_json::to_value(config).expect("serialize local KMS config");
value
.as_object_mut()
.expect("KMS config is a JSON object")
.remove("allow_insecure_dev_defaults");
let (config, migrated) = decode_persisted_kms_config(&serde_json::to_vec(&value).expect("serialize beta.5 config"))
.expect("decode beta.5 persisted config");
assert!(migrated);
assert!(config.allow_insecure_dev_defaults);
assert!(config.validate().is_ok());
}
#[test]
fn persisted_local_config_with_explicit_secure_mode_stays_secure() {
let temp_dir = TempDir::new().expect("create secure local KMS directory");
let config = rustfs_kms::KmsConfig::local(temp_dir.path().to_path_buf());
let (config, migrated) = decode_persisted_kms_config(&serde_json::to_vec(&config).expect("serialize current config"))
.expect("decode current persisted config");
assert!(!migrated);
assert!(!config.allow_insecure_dev_defaults);
assert!(config.validate().is_err());
}
#[test]
fn persisted_config_rejects_duplicate_security_field() {
let temp_dir = TempDir::new().expect("create local KMS directory");
let config = rustfs_kms::KmsConfig::local(temp_dir.path().to_path_buf());
let serialized = serde_json::to_string(&config).expect("serialize current config");
let duplicate = serialized.replacen('{', r#"{"allow_insecure_dev_defaults":true,"#, 1);
assert!(decode_persisted_kms_config(duplicate.as_bytes()).is_err());
}
#[test]
fn persisted_secure_local_config_without_legacy_field_stays_secure() {
#[cfg(unix)]
let key_dir = std::path::PathBuf::from("/var/lib/rustfs/kms");
#[cfg(windows)]
let key_dir = std::path::PathBuf::from(r"C:\rustfs-kms");
let mut config = rustfs_kms::KmsConfig::local(key_dir);
let rustfs_kms::BackendConfig::Local(local) = &mut config.backend_config else {
panic!("local constructor must create local backend config");
};
local.master_key = Some("configured-master-key".to_string());
let mut value = serde_json::to_value(config).expect("serialize secure local KMS config");
value
.as_object_mut()
.expect("KMS config is a JSON object")
.remove("allow_insecure_dev_defaults");
let (config, migrated) = decode_persisted_kms_config(&serde_json::to_vec(&value).expect("serialize old config"))
.expect("decode secure persisted config");
assert!(!migrated);
assert!(!config.allow_insecure_dev_defaults);
assert!(config.validate().is_ok());
}
#[test]
fn static_kms_config_is_not_persisted_with_cluster_configuration() {
use base64::Engine as _;
let config = rustfs_kms::KmsConfig::static_kms(
"static-key".to_string(),
base64::engine::general_purpose::STANDARD.encode([0x5au8; 32]),
);
assert!(ensure_kms_config_persistable(&config).is_err());
}
#[test]
fn local_reconfigure_preserves_hidden_master_key_for_same_directory() {
let key_dir = PathBuf::from("/var/lib/rustfs/kms");
let mut existing = rustfs_kms::KmsConfig::local(key_dir.clone());
let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
panic!("local constructor must create local backend config");
};
existing_local.master_key = Some("stored-master-key".to_string());
let mut request = rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
key_dir,
master_key: None,
file_permissions: Some(0o600),
default_key_id: Some("experience-key".to_string()),
timeout_seconds: Some(30),
retry_attempts: Some(3),
enable_cache: Some(true),
max_cached_keys: Some(1000),
cache_ttl_seconds: Some(3600),
allow_insecure_dev_defaults: Some(false),
});
normalize_configure_request_secrets(&mut request, Some(&existing)).expect("normalize local request");
let rustfs_kms::ConfigureKmsRequest::Local(request) = request else {
panic!("request must remain local");
};
assert_eq!(request.master_key.as_deref(), Some("stored-master-key"));
}
#[test]
fn local_reconfigure_preserves_unspecified_legacy_file_permissions() {
let key_dir = PathBuf::from("/var/lib/rustfs/kms");
let mut existing = rustfs_kms::KmsConfig::local(key_dir.clone());
let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
panic!("local constructor must create local backend config");
};
existing_local.master_key = Some("stored-master-key".to_string());
existing_local.file_permissions = None;
let mut request = rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
key_dir,
master_key: None,
file_permissions: None,
default_key_id: Some("experience-key".to_string()),
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
allow_insecure_dev_defaults: Some(false),
});
normalize_configure_request_secrets(&mut request, Some(&existing)).expect("normalize legacy local request");
let rustfs_kms::ConfigureKmsRequest::Local(request) = request else {
panic!("request must remain local");
};
assert!(request.file_permissions.is_none());
}
#[test]
fn local_reconfigure_does_not_reuse_master_key_for_different_directory() {
let mut existing = rustfs_kms::KmsConfig::local(PathBuf::from("/var/lib/rustfs/kms"));
let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
panic!("local constructor must create local backend config");
};
existing_local.master_key = Some("stored-master-key".to_string());
let mut request = rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
key_dir: PathBuf::from("/var/lib/rustfs/other-kms"),
master_key: None,
file_permissions: Some(0o600),
default_key_id: None,
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
allow_insecure_dev_defaults: Some(false),
});
let error = normalize_configure_request_secrets(&mut request, Some(&existing))
.expect_err("changing the local key directory must be rejected");
assert_eq!(error, "Changing the Local KMS key directory is not supported");
}
#[test]
fn local_reconfigure_rejects_file_permission_and_master_key_changes() {
let key_dir = PathBuf::from("/var/lib/rustfs/kms");
let mut existing = rustfs_kms::KmsConfig::local(key_dir.clone());
let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
panic!("local constructor must create local backend config");
};
existing_local.master_key = Some("stored-master-key".to_string());
existing_local.file_permissions = Some(0o600);
let request = |master_key, file_permissions| {
rustfs_kms::ConfigureKmsRequest::Local(rustfs_kms::ConfigureLocalKmsRequest {
key_dir: key_dir.clone(),
master_key,
file_permissions,
default_key_id: None,
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
allow_insecure_dev_defaults: Some(false),
})
};
let mut permissions_change = request(None, Some(0o666));
let permissions_error = normalize_configure_request_secrets(&mut permissions_change, Some(&existing))
.expect_err("changing file permissions must be rejected");
assert_eq!(permissions_error, "Changing Local KMS file permissions is not supported");
let mut master_key_change = request(Some("replacement-master-key".to_string()), Some(0o600));
let master_key_error = normalize_configure_request_secrets(&mut master_key_change, Some(&existing))
.expect_err("changing the master key must be rejected");
assert_eq!(master_key_error, "Changing the Local KMS master key is not supported");
let mut backend_change = rustfs_kms::ConfigureKmsRequest::Static(rustfs_kms::ConfigureStaticKmsRequest {
key_id: "static-key".to_string(),
secret_key: "not-used-by-normalization".to_string(),
default_key_id: None,
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
allow_insecure_dev_defaults: None,
});
let backend_error = normalize_configure_request_secrets(&mut backend_change, Some(&existing))
.expect_err("changing from the local backend must be rejected");
assert_eq!(backend_error, "Changing from the Local KMS backend is not supported");
}
fn aws_configure_request(region: &str) -> rustfs_kms::ConfigureKmsRequest {
rustfs_kms::ConfigureKmsRequest::Aws(rustfs_kms::ConfigureAwsKmsRequest {
region: region.to_string(),
endpoint_url: None,
default_key_id: Some("arn:aws:kms:us-east-1:111122223333:key/1234abcd".to_string()),
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
allow_insecure_dev_defaults: None,
})
}
/// The AWS backend holds no credential material of its own, so its
/// configuration is safe to persist cluster-wide and needs nothing carried
/// over from a previous configuration.
#[test]
fn aws_configure_request_is_persistable_and_needs_no_existing_credentials() {
let mut request = aws_configure_request("us-east-1");
normalize_configure_request_secrets(&mut request, None).expect("aws request needs no existing credentials");
assert!(ensure_kms_request_persistable(&request).is_ok());
let config = request.to_kms_config();
assert!(ensure_kms_config_persistable(&config).is_ok());
let canonical = redacted_canonical_config(&config).expect("aws configuration should serialize");
assert!(canonical.contains("us-east-1"), "the pinned region must drive the fingerprint");
for credential_field in ["access_key", "secret_access_key", "session_token"] {
assert!(
!canonical.contains(credential_field),
"aws configuration must carry no credential material: {canonical}"
);
}
}
/// Two nodes cannot be allowed to read the same AWS configuration as
/// different regions, so the pinned region has to be part of what a
/// fingerprint comparison would flag as a split.
#[test]
fn aws_config_fingerprint_tracks_the_pinned_region() {
let first = kms_config_fingerprint(&aws_configure_request("us-east-1").to_kms_config())
.expect("fingerprint should be computable");
let same = kms_config_fingerprint(&aws_configure_request("us-east-1").to_kms_config())
.expect("fingerprint should be computable");
let other = kms_config_fingerprint(&aws_configure_request("eu-central-1").to_kms_config())
.expect("fingerprint should be computable");
assert_eq!(first, same);
assert_ne!(first, other);
}
#[test]
fn local_backend_cannot_be_switched_to_aws() {
let mut existing = rustfs_kms::KmsConfig::local(PathBuf::from("/var/lib/rustfs/kms"));
let rustfs_kms::BackendConfig::Local(existing_local) = &mut existing.backend_config else {
panic!("local constructor must create local backend config");
};
existing_local.master_key = Some("stored-master-key".to_string());
let mut request = aws_configure_request("us-east-1");
let error = normalize_configure_request_secrets(&mut request, Some(&existing))
.expect_err("switching away from the local backend must be rejected");
assert_eq!(error, "Changing from the Local KMS backend is not supported");
}
const VAULT_TOKEN: &str = "hvs-super-secret-token";
fn vault_config(address: &str, token: &str) -> rustfs_kms::KmsConfig {
let vault = rustfs_kms::config::VaultConfig {
address: address.to_string(),
auth_method: rustfs_kms::config::VaultAuthMethod::Token {
token: token.to_string(),
},
..Default::default()
};
rustfs_kms::KmsConfig {
backend: rustfs_kms::KmsBackend::VaultKv2,
backend_config: rustfs_kms::BackendConfig::VaultKv2(Box::new(vault)),
..Default::default()
}
}
#[test]
fn config_fingerprint_input_carries_no_credentials() {
let canonical = redacted_canonical_config(&vault_config("https://vault.internal:8200", VAULT_TOKEN))
.expect("vault configuration should serialize");
assert!(!canonical.contains(VAULT_TOKEN), "fingerprint input must not carry the vault token");
assert!(canonical.contains(super::REDACTED_CONFIG_VALUE));
assert!(
canonical.contains("vault.internal"),
"non-secret configuration must still drive the fingerprint"
);
}
#[test]
fn config_fingerprint_agrees_across_nodes_holding_the_same_configuration() {
let first = kms_config_fingerprint(&vault_config("https://vault.internal:8200", VAULT_TOKEN))
.expect("fingerprint should be computable");
let second = kms_config_fingerprint(&vault_config("https://vault.internal:8200", VAULT_TOKEN))
.expect("fingerprint should be computable");
assert_eq!(first, second);
let other_backend = kms_config_fingerprint(&vault_config("https://vault.other:8200", VAULT_TOKEN))
.expect("fingerprint should be computable");
assert_ne!(first, other_backend, "a different backend address must be visible as a split");
// Credentials are redacted before hashing, so a rotated token is not a
// split. Documented here so the trade-off is not silently regressed.
let rotated_token = kms_config_fingerprint(&vault_config("https://vault.internal:8200", "hvs-rotated-token"))
.expect("fingerprint should be computable");
assert_eq!(first, rotated_token);
}
#[test]
fn peer_reload_replays_only_when_the_persisted_configuration_moved() {
let current = vault_config("https://vault.internal:8200", VAULT_TOKEN);
assert!(kms_config_is_unchanged(
&current,
&vault_config("https://vault.internal:8200", VAULT_TOKEN)
));
assert!(!kms_config_is_unchanged(&current, &vault_config("https://vault.other:8200", VAULT_TOKEN)));
assert!(
!kms_config_is_unchanged(&current, &vault_config("https://vault.internal:8200", "hvs-rotated-token")),
"a credential-only change must still be adopted, unlike the redacted fingerprint"
);
}
#[test]
fn peer_reload_failures_are_reported_without_failing_the_local_change() {
let (success, message) = local_success_with_peer_report("KMS configured successfully", &[]);
assert!(success);
assert_eq!(message, "KMS configured successfully");
let (success, message) = local_success_with_peer_report(
"KMS configured successfully",
&["10.0.0.2:9000".to_string(), "10.0.0.3:9000".to_string()],
);
assert!(success, "a peer that failed to reload must not fail the node that already switched");
assert!(message.contains("2 peer(s)"));
assert!(message.contains("10.0.0.2:9000"));
assert!(message.contains("10.0.0.3:9000"));
}
}
Reference in New Issue View Git Blame Copy Permalink